1 files changed, 181 insertions, 142 deletions

diff --git a/subcmds/sync.py b/subcmds/sync.py
index d4637d0c..228a279a 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from __future__ import print_function
 import netrc
 from optparse import SUPPRESS_HELP
 import os
@@ -44,13 +45,13 @@ try:
 except ImportError:
   multiprocessing = None
 
-from git_command import GIT
+from git_command import GIT, git_require
 from git_refs import R_HEADS, HEAD
 from main import WrapperModule
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
-from error import RepoChangedException, GitError
+from error import RepoChangedException, GitError, ManifestParseError
 from project import SyncBuffer
 from progress import Progress
 
@@ -113,6 +114,9 @@ resumeable bundle file on a content delivery network. This
 may be necessary if there are problems with the local Python
 HTTP client or proxy configuration, but the Git binary works.
 
+The --fetch-submodules option enables fetching Git submodules
+of a project from server.
+
 SSH Connections
 ---------------
 
@@ -144,27 +148,30 @@ later is required to fix a server side protocol bug.
 """
 
   def _Options(self, p, show_smart=True):
-    self.jobs = self.manifest.default.sync_j
+    try:
+      self.jobs = self.manifest.default.sync_j
+    except ManifestParseError:
+      self.jobs = 1
 
     p.add_option('-f', '--force-broken',
                  dest='force_broken', action='store_true',
                  help="continue sync even if a project fails to sync")
-    p.add_option('-l','--local-only',
+    p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
-    p.add_option('-n','--network-only',
+    p.add_option('-n', '--network-only',
                  dest='network_only', action='store_true',
                  help="fetch only, don't update working tree")
-    p.add_option('-d','--detach',
+    p.add_option('-d', '--detach',
                  dest='detach_head', action='store_true',
                  help='detach projects back to manifest revision')
-    p.add_option('-c','--current-branch',
+    p.add_option('-c', '--current-branch',
                  dest='current_branch_only', action='store_true',
                  help='fetch only current branch from server')
-    p.add_option('-q','--quiet',
+    p.add_option('-q', '--quiet',
                  dest='quiet', action='store_true',
                  help='be more quiet')
-    p.add_option('-j','--jobs',
+    p.add_option('-j', '--jobs',
                  dest='jobs', action='store', type='int',
                  help="projects to fetch simultaneously (default %d)" % self.jobs)
     p.add_option('-m', '--manifest-name',
@@ -173,6 +180,15 @@ later is required to fix a server side protocol bug.
     p.add_option('--no-clone-bundle',
                  dest='no_clone_bundle', action='store_true',
                  help='disable use of /clone.bundle on HTTP/HTTPS')
+    p.add_option('-u', '--manifest-server-username', action='store',
+                 dest='manifest_server_username',
+                 help='username to authenticate with the manifest server')
+    p.add_option('-p', '--manifest-server-password', action='store',
+                 dest='manifest_server_password',
+                 help='password to authenticate with the manifest server')
+    p.add_option('--fetch-submodules',
+                 dest='fetch_submodules', action='store_true',
+                 help='fetch submodules from server')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -180,12 +196,6 @@ later is required to fix a server side protocol bug.
       p.add_option('-t', '--smart-tag',
                    dest='smart_tag', action='store',
                    help='smart sync using manifest from a known tag')
-      p.add_option('-u', '--manifest-server-username', action='store',
-                   dest='manifest_server_username',
-                   help='username to authenticate with the manifest server')
-      p.add_option('-p', '--manifest-server-password', action='store',
-                   dest='manifest_server_password',
-                   help='password to authenticate with the manifest server')
 
     g = p.add_option_group('repo Version options')
     g.add_option('--no-repo-verify',
@@ -196,61 +206,62 @@ later is required to fix a server side protocol bug.
                  help=SUPPRESS_HELP)
 
   def _FetchHelper(self, opt, project, lock, fetched, pm, sem, err_event):
-      """Main function of the fetch threads when jobs are > 1.
-
-      Args:
-        opt: Program options returned from optparse.  See _Options().
-        project: Project object for the project to fetch.
-        lock: Lock for accessing objects that are shared amongst multiple
-            _FetchHelper() threads.
-        fetched: set object that we will add project.gitdir to when we're done
-            (with our lock held).
-        pm: Instance of a Project object.  We will call pm.update() (with our
-            lock held).
-        sem: We'll release() this semaphore when we exit so that another thread
-            can be started up.
-        err_event: We'll set this event in the case of an error (after printing
-            out info about the error).
-      """
-      # We'll set to true once we've locked the lock.
-      did_lock = False
-
-      # Encapsulate everything in a try/except/finally so that:
-      # - We always set err_event in the case of an exception.
-      # - We always make sure we call sem.release().
-      # - We always make sure we unlock the lock if we locked it.
+    """Main function of the fetch threads when jobs are > 1.
+
+    Args:
+      opt: Program options returned from optparse.  See _Options().
+      project: Project object for the project to fetch.
+      lock: Lock for accessing objects that are shared amongst multiple
+          _FetchHelper() threads.
+      fetched: set object that we will add project.gitdir to when we're done
+          (with our lock held).
+      pm: Instance of a Project object.  We will call pm.update() (with our
+          lock held).
+      sem: We'll release() this semaphore when we exit so that another thread
+          can be started up.
+      err_event: We'll set this event in the case of an error (after printing
+          out info about the error).
+    """
+    # We'll set to true once we've locked the lock.
+    did_lock = False
+
+    # Encapsulate everything in a try/except/finally so that:
+    # - We always set err_event in the case of an exception.
+    # - We always make sure we call sem.release().
+    # - We always make sure we unlock the lock if we locked it.
+    try:
       try:
-        try:
-          start = time.time()
-          success = project.Sync_NetworkHalf(
-            quiet=opt.quiet,
-            current_branch_only=opt.current_branch_only,
-            clone_bundle=not opt.no_clone_bundle)
-          self._fetch_times.Set(project, time.time() - start)
-
-          # Lock around all the rest of the code, since printing, updating a set
-          # and Progress.update() are not thread safe.
-          lock.acquire()
-          did_lock = True
-
-          if not success:
-            print >>sys.stderr, 'error: Cannot fetch %s' % project.name
-            if opt.force_broken:
-              print >>sys.stderr, 'warn: --force-broken, continuing to sync'
-            else:
-              raise _FetchError()
+        start = time.time()
+        success = project.Sync_NetworkHalf(
+          quiet=opt.quiet,
+          current_branch_only=opt.current_branch_only,
+          clone_bundle=not opt.no_clone_bundle)
+        self._fetch_times.Set(project, time.time() - start)
+
+        # Lock around all the rest of the code, since printing, updating a set
+        # and Progress.update() are not thread safe.
+        lock.acquire()
+        did_lock = True
+
+        if not success:
+          print('error: Cannot fetch %s' % project.name, file=sys.stderr)
+          if opt.force_broken:
+            print('warn: --force-broken, continuing to sync',
+                  file=sys.stderr)
+          else:
+            raise _FetchError()
 
-          fetched.add(project.gitdir)
-          pm.update()
-        except _FetchError:
-          err_event.set()
-        except:
-          err_event.set()
-          raise
-      finally:
-        if did_lock:
-          lock.release()
-        sem.release()
+        fetched.add(project.gitdir)
+        pm.update()
+      except _FetchError:
+        err_event.set()
+      except:
+        err_event.set()
+        raise
+    finally:
+      if did_lock:
+        lock.release()
+      sem.release()
 
   def _Fetch(self, projects, opt):
     fetched = set()
@@ -265,9 +276,9 @@ later is required to fix a server side protocol bug.
             clone_bundle=not opt.no_clone_bundle):
           fetched.add(project.gitdir)
         else:
-          print >>sys.stderr, 'error: Cannot fetch %s' % project.name
+          print('error: Cannot fetch %s' % project.name, file=sys.stderr)
           if opt.force_broken:
-            print >>sys.stderr, 'warn: --force-broken, continuing to sync'
+            print('warn: --force-broken, continuing to sync', file=sys.stderr)
           else:
             sys.exit(1)
     else:
@@ -300,7 +311,7 @@ later is required to fix a server side protocol bug.
 
       # If we saw an error, exit with code 1 so that other scripts can check.
       if err_event.isSet():
-        print >>sys.stderr, '\nerror: Exited sync due to fetch errors'
+        print('\nerror: Exited sync due to fetch errors', file=sys.stderr)
         sys.exit(1)
 
     pm.end()
@@ -310,7 +321,8 @@ later is required to fix a server side protocol bug.
     return fetched
 
   def _GCProjects(self, projects):
-    if multiprocessing:
+    has_dash_c = git_require((1, 7, 2))
+    if multiprocessing and has_dash_c:
       cpu_count = multiprocessing.cpu_count()
     else:
       cpu_count = 1
@@ -352,7 +364,7 @@ later is required to fix a server side protocol bug.
       t.join()
 
     if err_event.isSet():
-      print >>sys.stderr, '\nerror: Exited sync due to gc errors'
+      print('\nerror: Exited sync due to gc errors', file=sys.stderr)
       sys.exit(1)
 
   def UpdateProjectList(self):
@@ -376,34 +388,36 @@ later is required to fix a server side protocol bug.
         if path not in new_project_paths:
           # If the path has already been deleted, we don't need to do it
           if os.path.exists(self.manifest.topdir + '/' + path):
-              project = Project(
-                             manifest = self.manifest,
-                             name = path,
-                             remote = RemoteSpec('origin'),
-                             gitdir = os.path.join(self.manifest.topdir,
-                                                   path, '.git'),
-                             worktree = os.path.join(self.manifest.topdir, path),
-                             relpath = path,
-                             revisionExpr = 'HEAD',
-                             revisionId = None,
-                             groups = None)
-
-              if project.IsDirty():
-                print >>sys.stderr, 'error: Cannot remove project "%s": \
-uncommitted changes are present' % project.relpath
-                print >>sys.stderr, '       commit changes, then run sync again'
-                return -1
-              else:
-                print >>sys.stderr, 'Deleting obsolete path %s' % project.worktree
-                shutil.rmtree(project.worktree)
-                # Try deleting parent subdirs if they are empty
-                project_dir = os.path.dirname(project.worktree)
-                while project_dir != self.manifest.topdir:
-                  try:
-                    os.rmdir(project_dir)
-                  except OSError:
-                    break
-                  project_dir = os.path.dirname(project_dir)
+            project = Project(
+                           manifest = self.manifest,
+                           name = path,
+                           remote = RemoteSpec('origin'),
+                           gitdir = os.path.join(self.manifest.topdir,
+                                                 path, '.git'),
+                           worktree = os.path.join(self.manifest.topdir, path),
+                           relpath = path,
+                           revisionExpr = 'HEAD',
+                           revisionId = None,
+                           groups = None)
+
+            if project.IsDirty():
+              print('error: Cannot remove project "%s": uncommitted changes'
+                    'are present' % project.relpath, file=sys.stderr)
+              print('       commit changes, then run sync again',
+                    file=sys.stderr)
+              return -1
+            else:
+              print('Deleting obsolete path %s' % project.worktree,
+                    file=sys.stderr)
+              shutil.rmtree(project.worktree)
+              # Try deleting parent subdirs if they are empty
+              project_dir = os.path.dirname(project.worktree)
+              while project_dir != self.manifest.topdir:
+                try:
+                  os.rmdir(project_dir)
+                except OSError:
+                  break
+                project_dir = os.path.dirname(project_dir)
 
     new_project_paths.sort()
     fd = open(file_path, 'w')
@@ -422,24 +436,24 @@ uncommitted changes are present' % project.relpath
       self.jobs = min(self.jobs, (soft_limit - 5) / 3)
 
     if opt.network_only and opt.detach_head:
-      print >>sys.stderr, 'error: cannot combine -n and -d'
+      print('error: cannot combine -n and -d', file=sys.stderr)
       sys.exit(1)
     if opt.network_only and opt.local_only:
-      print >>sys.stderr, 'error: cannot combine -n and -l'
+      print('error: cannot combine -n and -l', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_name and opt.smart_sync:
-      print >>sys.stderr, 'error: cannot combine -m and -s'
+      print('error: cannot combine -m and -s', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_name and opt.smart_tag:
-      print >>sys.stderr, 'error: cannot combine -m and -t'
+      print('error: cannot combine -m and -t', file=sys.stderr)
       sys.exit(1)
     if opt.manifest_server_username or opt.manifest_server_password:
       if not (opt.smart_sync or opt.smart_tag):
-        print >>sys.stderr, 'error: -u and -p may only be combined with ' \
-                            '-s or -t'
+        print('error: -u and -p may only be combined with -s or -t',
+              file=sys.stderr)
         sys.exit(1)
       if None in [opt.manifest_server_username, opt.manifest_server_password]:
-        print >>sys.stderr, 'error: both -u and -p must be given'
+        print('error: both -u and -p must be given', file=sys.stderr)
         sys.exit(1)
 
     if opt.manifest_name:
@@ -447,8 +461,8 @@ uncommitted changes are present' % project.relpath
 
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
-        print >>sys.stderr, \
-            'error: cannot smart sync: no manifest server defined in manifest'
+        print('error: cannot smart sync: no manifest server defined in'
+              'manifest', file=sys.stderr)
         sys.exit(1)
 
       manifest_server = self.manifest.manifest_server
@@ -463,7 +477,8 @@ uncommitted changes are present' % project.relpath
           try:
             info = netrc.netrc()
           except IOError:
-            print >>sys.stderr, '.netrc file does not exist or could not be opened'
+            print('.netrc file does not exist or could not be opened',
+                  file=sys.stderr)
           else:
             try:
               parse_result = urlparse.urlparse(manifest_server)
@@ -473,10 +488,10 @@ uncommitted changes are present' % project.relpath
             except TypeError:
               # TypeError is raised when the given hostname is not present
               # in the .netrc file.
-              print >>sys.stderr, 'No credentials found for %s in .netrc' % \
-                                  parse_result.hostname
+              print('No credentials found for %s in .netrc'
+                    % parse_result.hostname, file=sys.stderr)
             except netrc.NetrcParseError as e:
-              print >>sys.stderr, 'Error parsing .netrc file: %s' % e
+              print('Error parsing .netrc file: %s' % e, file=sys.stderr)
 
         if (username and password):
           manifest_server = manifest_server.replace('://', '://%s:%s@' %
@@ -515,20 +530,21 @@ uncommitted changes are present' % project.relpath
             finally:
               f.close()
           except IOError:
-            print >>sys.stderr, 'error: cannot write manifest to %s' % \
-                manifest_path
+            print('error: cannot write manifest to %s' % manifest_path,
+                  file=sys.stderr)
             sys.exit(1)
           self.manifest.Override(manifest_name)
         else:
-          print >>sys.stderr, 'error: %s' % manifest_str
+          print('error: %s' % manifest_str, file=sys.stderr)
           sys.exit(1)
       except (socket.error, IOError, xmlrpclib.Fault) as e:
-        print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%s' % (
-            self.manifest.manifest_server, e)
+        print('error: cannot connect to manifest server %s:\n%s'
+              % (self.manifest.manifest_server, e), file=sys.stderr)
         sys.exit(1)
       except xmlrpclib.ProtocolError as e:
-        print >>sys.stderr, 'error: cannot connect to manifest server %s:\n%d %s' % (
-            self.manifest.manifest_server, e.errcode, e.errmsg)
+        print('error: cannot connect to manifest server %s:\n%d %s'
+              % (self.manifest.manifest_server, e.errcode, e.errmsg),
+              file=sys.stderr)
         sys.exit(1)
 
     rp = self.manifest.repoProject
@@ -552,7 +568,9 @@ uncommitted changes are present' % project.relpath
       self.manifest._Unload()
       if opt.jobs is None:
         self.jobs = self.manifest.default.sync_j
-    all_projects = self.GetProjects(args, missing_ok=True)
+    all_projects = self.GetProjects(args,
+                                    missing_ok=True,
+                                    submodules_ok=opt.fetch_submodules)
 
     self._fetch_times = _FetchTimes(self.manifest)
     if not opt.local_only:
@@ -563,12 +581,33 @@ uncommitted changes are present' % project.relpath
       to_fetch.extend(all_projects)
       to_fetch.sort(key=self._fetch_times.Get, reverse=True)
 
-      self._Fetch(to_fetch, opt)
+      fetched = self._Fetch(to_fetch, opt)
       _PostRepoFetch(rp, opt.no_repo_verify)
       if opt.network_only:
         # bail out now; the rest touches the working tree
         return
 
+      # Iteratively fetch missing and/or nested unregistered submodules
+      previously_missing_set = set()
+      while True:
+        self.manifest._Unload()
+        all_projects = self.GetProjects(args,
+                                        missing_ok=True,
+                                        submodules_ok=opt.fetch_submodules)
+        missing = []
+        for project in all_projects:
+          if project.gitdir not in fetched:
+            missing.append(project)
+        if not missing:
+          break
+        # Stop us from non-stopped fetching actually-missing repos: If set of
+        # missing repos has not been changed from last fetch, we break.
+        missing_set = set(p.name for p in missing)
+        if previously_missing_set == missing_set:
+          break
+        previously_missing_set = missing_set
+        fetched.update(self._Fetch(missing, opt))
+
     if self.manifest.IsMirror:
       # bail out now, we have no working tree
       return
@@ -584,14 +623,14 @@ uncommitted changes are present' % project.relpath
       if project.worktree:
         project.Sync_LocalHalf(syncbuf)
     pm.end()
-    print >>sys.stderr
+    print(file=sys.stderr)
     if not syncbuf.Finish():
       sys.exit(1)
 
     # If there's a notice that's supposed to print at the end of the sync, print
     # it now...
     if self.manifest.notice:
-      print self.manifest.notice
+      print(self.manifest.notice)
 
 def _PostRepoUpgrade(manifest, quiet=False):
   wrapper = WrapperModule()
@@ -603,27 +642,28 @@ def _PostRepoUpgrade(manifest, quiet=False):
 
 def _PostRepoFetch(rp, no_repo_verify=False, verbose=False):
   if rp.HasChanges:
-    print >>sys.stderr, 'info: A new version of repo is available'
-    print >>sys.stderr, ''
+    print('info: A new version of repo is available', file=sys.stderr)
+    print(file=sys.stderr)
     if no_repo_verify or _VerifyTag(rp):
       syncbuf = SyncBuffer(rp.config)
       rp.Sync_LocalHalf(syncbuf)
       if not syncbuf.Finish():
         sys.exit(1)
-      print >>sys.stderr, 'info: Restarting repo with latest version'
+      print('info: Restarting repo with latest version', file=sys.stderr)
       raise RepoChangedException(['--repo-upgraded'])
     else:
-      print >>sys.stderr, 'warning: Skipped upgrade to unverified version'
+      print('warning: Skipped upgrade to unverified version', file=sys.stderr)
   else:
     if verbose:
-      print >>sys.stderr, 'repo version %s is current' % rp.work_git.describe(HEAD)
+      print('repo version %s is current' % rp.work_git.describe(HEAD),
+            file=sys.stderr)
 
 def _VerifyTag(project):
   gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
   if not os.path.exists(gpg_dir):
-    print >>sys.stderr,\
-"""warning: GnuPG was not available during last "repo init"
-warning: Cannot automatically authenticate repo."""
+    print('warning: GnuPG was not available during last "repo init"\n'
+          'warning: Cannot automatically authenticate repo."""',
+          file=sys.stderr)
     return True
 
   try:
@@ -637,10 +677,9 @@ warning: Cannot automatically authenticate repo."""
     if rev.startswith(R_HEADS):
       rev = rev[len(R_HEADS):]
 
-    print >>sys.stderr
-    print >>sys.stderr,\
-      "warning: project '%s' branch '%s' is not signed" \
-      % (project.name, rev)
+    print(file=sys.stderr)
+    print("warning: project '%s' branch '%s' is not signed"
+          % (project.name, rev), file=sys.stderr)
     return False
 
   env = os.environ.copy()
@@ -659,10 +698,10 @@ warning: Cannot automatically authenticate repo."""
   proc.stderr.close()
 
   if proc.wait() != 0:
-    print >>sys.stderr
-    print >>sys.stderr, out
-    print >>sys.stderr, err
-    print >>sys.stderr
+    print(file=sys.stderr)
+    print(out, file=sys.stderr)
+    print(err, file=sys.stderr)
+    print(file=sys.stderr)
     return False
   return True
 
@@ -696,7 +735,7 @@ class _FetchTimes(object):
       try:
         try:
           self._times = pickle.load(f)
-        except:
+        except IOError:
           try:
             os.remove(self._path)
           except OSError: