1 files changed, 66 insertions, 36 deletions

diff --git a/subcmds/sync.py b/subcmds/sync.py
index b50df099..43d450be 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -14,10 +14,10 @@
 # limitations under the License.
 
 from __future__ import print_function
+import json
 import netrc
 from optparse import SUPPRESS_HELP
 import os
-import pickle
 import re
 import shutil
 import socket
@@ -119,6 +119,11 @@ credentials.
 The -f/--force-broken option can be used to proceed with syncing
 other projects if a project sync fails.
 
+The --force-sync option can be used to overwrite existing git
+directories if they have previously been linked to a different
+object direcotry. WARNING: This may cause data to be lost since
+refs may be removed when overwriting.
+
 The --no-clone-bundle option disables any attempt to use
 $URL/clone.bundle to bootstrap a new Git repository from a
 resumeable bundle file on a content delivery network. This
@@ -128,6 +133,13 @@ HTTP client or proxy configuration, but the Git binary works.
 The --fetch-submodules option enables fetching Git submodules
 of a project from server.
 
+The -c/--current-branch option can be used to only fetch objects that
+are on the branch specified by a project's revision.
+
+The --optimized-fetch option can be used to only fetch projects that
+are fixed to a sha1 revision if the sha1 revision does not already
+exist locally.
+
 SSH Connections
 ---------------
 
@@ -167,6 +179,11 @@ later is required to fix a server side protocol bug.
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help="continue sync even if a project fails to sync")
+    p.add_option('--force-sync',
+                 dest='force_sync', action='store_true',
+                 help="overwrite an existing git directory if it needs to "
+                      "point to a different object directory. WARNING: this "
+                      "may cause loss of data")
     p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
@@ -203,6 +220,9 @@ later is required to fix a server side protocol bug.
     p.add_option('--no-tags',
                  dest='no_tags', action='store_true',
                  help="don't fetch tags")
+    p.add_option('--optimized-fetch',
+                 dest='optimized_fetch', action='store_true',
+                 help='only fetch projects fixed to sha1 if revision does not exist locally')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -271,8 +291,10 @@ later is required to fix a server side protocol bug.
         success = project.Sync_NetworkHalf(
           quiet=opt.quiet,
           current_branch_only=opt.current_branch_only,
+          force_sync=opt.force_sync,
           clone_bundle=not opt.no_clone_bundle,
-          no_tags=opt.no_tags, archive=self.manifest.IsArchive)
+          no_tags=opt.no_tags, archive=self.manifest.IsArchive,
+          optimized_fetch=opt.optimized_fetch)
         self._fetch_times.Set(project, time.time() - start)
 
         # Lock around all the rest of the code, since printing, updating a set
@@ -508,6 +530,9 @@ later is required to fix a server side protocol bug.
       self.manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
 
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
@@ -560,7 +585,10 @@ later is required to fix a server side protocol bug.
             branch = branch[len(R_HEADS):]
 
           env = os.environ.copy()
-          if 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
+          if 'SYNC_TARGET' in env:
+            target = env['SYNC_TARGET']
+            [success, manifest_str] = server.GetApprovedManifest(branch, target)
+          elif 'TARGET_PRODUCT' in env and 'TARGET_BUILD_VARIANT' in env:
             target = '%s-%s' % (env['TARGET_PRODUCT'],
                                 env['TARGET_BUILD_VARIANT'])
             [success, manifest_str] = server.GetApprovedManifest(branch, target)
@@ -571,17 +599,16 @@ later is required to fix a server side protocol bug.
           [success, manifest_str] = server.GetManifest(opt.smart_tag)
 
         if success:
-          manifest_name = "smart_sync_override.xml"
-          manifest_path = os.path.join(self.manifest.manifestProject.worktree,
-                                       manifest_name)
+          manifest_name = smart_sync_manifest_name
           try:
-            f = open(manifest_path, 'w')
+            f = open(smart_sync_manifest_path, 'w')
             try:
               f.write(manifest_str)
             finally:
               f.close()
-          except IOError:
-            print('error: cannot write manifest to %s' % manifest_path,
+          except IOError as e:
+            print('error: cannot write manifest to %s:\n%s'
+                  % (smart_sync_manifest_path, e),
                   file=sys.stderr)
             sys.exit(1)
           self._ReloadManifest(manifest_name)
@@ -598,6 +625,13 @@ later is required to fix a server side protocol bug.
               % (self.manifest.manifest_server, e.errcode, e.errmsg),
               file=sys.stderr)
         sys.exit(1)
+    else:  # Not smart sync or smart tag mode
+      if os.path.isfile(smart_sync_manifest_path):
+        try:
+          os.remove(smart_sync_manifest_path)
+        except OSError as e:
+          print('error: failed to remove existing smart sync override manifest: %s' %
+                e, file=sys.stderr)
 
     rp = self.manifest.repoProject
     rp.PreSync()
@@ -611,7 +645,8 @@ later is required to fix a server side protocol bug.
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
                           current_branch_only=opt.current_branch_only,
-                          no_tags=opt.no_tags)
+                          no_tags=opt.no_tags,
+                          optimized_fetch=opt.optimized_fetch)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)
@@ -674,7 +709,7 @@ later is required to fix a server side protocol bug.
     for project in all_projects:
       pm.update()
       if project.worktree:
-        project.Sync_LocalHalf(syncbuf)
+        project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
     pm.end()
     print(file=sys.stderr)
     if not syncbuf.Finish():
@@ -762,7 +797,7 @@ class _FetchTimes(object):
   _ALPHA = 0.5
 
   def __init__(self, manifest):
-    self._path = os.path.join(manifest.repodir, '.repopickle_fetchtimes')
+    self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
     self._times = None
     self._seen = set()
 
@@ -781,22 +816,17 @@ class _FetchTimes(object):
   def _Load(self):
     if self._times is None:
       try:
-        f = open(self._path, 'rb')
-      except IOError:
-        self._times = {}
-        return self._times
-      try:
+        f = open(self._path)
         try:
-          self._times = pickle.load(f)
-        except IOError:
-          try:
-            os.remove(self._path)
-          except OSError:
-            pass
-          self._times = {}
-      finally:
-        f.close()
-    return self._times
+          self._times = json.load(f)
+        finally:
+          f.close()
+      except (IOError, ValueError):
+        try:
+          os.remove(self._path)
+        except OSError:
+          pass
+        self._times = {}
 
   def Save(self):
     if self._times is None:
@@ -810,13 +840,13 @@ class _FetchTimes(object):
       del self._times[name]
 
     try:
-      f = open(self._path, 'wb')
+      f = open(self._path, 'w')
       try:
-        pickle.dump(self._times, f)
-      except (IOError, OSError, pickle.PickleError):
-        try:
-          os.remove(self._path)
-        except OSError:
-          pass
-    finally:
-      f.close()
+        json.dump(self._times, f, indent=2)
+      finally:
+        f.close()
+    except (IOError, TypeError):
+      try:
+        os.remove(self._path)
+      except OSError:
+        pass