4 files changed, 63 insertions, 16 deletions

diff --git a/subcmds/cherry_pick.py b/subcmds/cherry_pick.py
index 520e4c32..1f7dffdc 100644
--- a/subcmds/cherry_pick.py
+++ b/subcmds/cherry_pick.py
@@ -76,6 +76,7 @@ change id will be added.
                      capture_stdout = True,
                      capture_stderr = True)
       p.stdin.write(new_msg)
+      p.stdin.close()
       if p.Wait() != 0:
         print("error: Failed to update commit message", file=sys.stderr)
         sys.exit(1)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index 88b23fbd..b93cd6d0 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -20,6 +20,7 @@ import multiprocessing
 import re
 import os
 import select
+import signal
 import sys
 import subprocess
 
@@ -150,11 +151,15 @@ without iterating through the remaining projects.
     attributes that we need.
 
     """
+    if not self.manifest.IsMirror:
+      lrev = project.GetRevisionId()
+    else:
+      lrev = None
     return {
       'name': project.name,
       'relpath': project.relpath,
       'remote_name': project.remote.name,
-      'lrev': project.GetRevisionId(),
+      'lrev': lrev,
       'rrev': project.revisionExpr,
       'annotations': dict((a.name, a.value) for a in project.annotations),
       'gitdir': project.gitdir,
@@ -200,6 +205,13 @@ without iterating through the remaining projects.
     mirror = self.manifest.IsMirror
     rc = 0
 
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
+
+    if os.path.isfile(smart_sync_manifest_path):
+      self.manifest.Override(smart_sync_manifest_path)
+
     if not opt.regex:
       projects = self.GetProjects(args)
     else:
@@ -207,14 +219,12 @@ without iterating through the remaining projects.
 
     os.environ['REPO_COUNT'] = str(len(projects))
 
-    pool = multiprocessing.Pool(opt.jobs)
+    pool = multiprocessing.Pool(opt.jobs, InitWorker)
     try:
       config = self.manifest.manifestProject.config
       results_it = pool.imap(
          DoWorkWrapper,
-         ([mirror, opt, cmd, shell, cnt, config, self._SerializeProject(p)]
-          for cnt, p in enumerate(projects))
-      )
+         self.ProjectArgs(projects, mirror, opt, cmd, shell, config))
       pool.close()
       for r in results_it:
         rc = rc or r
@@ -236,12 +246,28 @@ without iterating through the remaining projects.
     if rc != 0:
       sys.exit(rc)
 
+  def ProjectArgs(self, projects, mirror, opt, cmd, shell, config):
+    for cnt, p in enumerate(projects):
+      try:
+        project = self._SerializeProject(p)
+      except Exception as e:
+        print('Project list error: %r' % e,
+              file=sys.stderr)
+        return
+      except KeyboardInterrupt:
+        print('Project list interrupted',
+              file=sys.stderr)
+        return
+      yield [mirror, opt, cmd, shell, cnt, config, project]
 
 class WorkerKeyboardInterrupt(Exception):
   """ Keyboard interrupt exception for worker processes. """
   pass
 
 
+def InitWorker():
+  signal.signal(signal.SIGINT, signal.SIG_IGN)
+
 def DoWorkWrapper(args):
   """ A wrapper around the DoWork() method.
 
@@ -263,7 +289,9 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config):
   def setenv(name, val):
     if val is None:
       val = ''
-    env[name] = val.encode()
+    if hasattr(val, 'encode'):
+      val = val.encode()
+    env[name] = val
 
   setenv('REPO_PROJECT', project['name'])
   setenv('REPO_PATH', project['relpath'])
diff --git a/subcmds/init.py b/subcmds/init.py
index b73de71c..dbb6ddda 100644
--- a/subcmds/init.py
+++ b/subcmds/init.py
@@ -27,7 +27,7 @@ else:
   import imp
   import urlparse
   urllib = imp.new_module('urllib')
-  urllib.parse = urlparse.urlparse
+  urllib.parse = urlparse
 
 from color import Coloring
 from command import InteractiveCommand, MirrorSafeCommand
@@ -153,7 +153,7 @@ to update the working directory files.
       # server where this git is located, so let's save that here.
       mirrored_manifest_git = None
       if opt.reference:
-        manifest_git_path = urllib.parse(opt.manifest_url).path[1:]
+        manifest_git_path = urllib.parse.urlparse(opt.manifest_url).path[1:]
         mirrored_manifest_git = os.path.join(opt.reference, manifest_git_path)
         if not mirrored_manifest_git.endswith(".git"):
           mirrored_manifest_git += ".git"
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 2bdab3a6..ec333ae7 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -131,6 +131,10 @@ of a project from server.
 The -c/--current-branch option can be used to only fetch objects that
 are on the branch specified by a project's revision.
 
+The --optimized-fetch option can be used to only fetch projects that
+are fixed to a sha1 revision if the sha1 revision does not already
+exist locally.
+
 SSH Connections
 ---------------
 
@@ -206,6 +210,9 @@ later is required to fix a server side protocol bug.
     p.add_option('--no-tags',
                  dest='no_tags', action='store_true',
                  help="don't fetch tags")
+    p.add_option('--optimized-fetch',
+                 dest='optimized_fetch', action='store_true',
+                 help='only fetch projects fixed to sha1 if revision does not exist locally')
     if show_smart:
       p.add_option('-s', '--smart-sync',
                    dest='smart_sync', action='store_true',
@@ -275,7 +282,8 @@ later is required to fix a server side protocol bug.
           quiet=opt.quiet,
           current_branch_only=opt.current_branch_only,
           clone_bundle=not opt.no_clone_bundle,
-          no_tags=opt.no_tags, archive=self.manifest.IsArchive)
+          no_tags=opt.no_tags, archive=self.manifest.IsArchive,
+          optimized_fetch=opt.optimized_fetch)
         self._fetch_times.Set(project, time.time() - start)
 
         # Lock around all the rest of the code, since printing, updating a set
@@ -509,6 +517,9 @@ later is required to fix a server side protocol bug.
       self.manifest.Override(opt.manifest_name)
 
     manifest_name = opt.manifest_name
+    smart_sync_manifest_name = "smart_sync_override.xml"
+    smart_sync_manifest_path = os.path.join(
+      self.manifest.manifestProject.worktree, smart_sync_manifest_name)
 
     if opt.smart_sync or opt.smart_tag:
       if not self.manifest.manifest_server:
@@ -575,17 +586,16 @@ later is required to fix a server side protocol bug.
           [success, manifest_str] = server.GetManifest(opt.smart_tag)
 
         if success:
-          manifest_name = "smart_sync_override.xml"
-          manifest_path = os.path.join(self.manifest.manifestProject.worktree,
-                                       manifest_name)
+          manifest_name = smart_sync_manifest_name
           try:
-            f = open(manifest_path, 'w')
+            f = open(smart_sync_manifest_path, 'w')
             try:
               f.write(manifest_str)
             finally:
               f.close()
-          except IOError:
-            print('error: cannot write manifest to %s' % manifest_path,
+          except IOError as e:
+            print('error: cannot write manifest to %s:\n%s'
+                  % (smart_sync_manifest_path, e),
                   file=sys.stderr)
             sys.exit(1)
           self._ReloadManifest(manifest_name)
@@ -602,6 +612,13 @@ later is required to fix a server side protocol bug.
               % (self.manifest.manifest_server, e.errcode, e.errmsg),
               file=sys.stderr)
         sys.exit(1)
+    else:  # Not smart sync or smart tag mode
+      if os.path.isfile(smart_sync_manifest_path):
+        try:
+          os.remove(smart_sync_manifest_path)
+        except OSError as e:
+          print('error: failed to remove existing smart sync override manifest: %s' %
+                e, file=sys.stderr)
 
     rp = self.manifest.repoProject
     rp.PreSync()
@@ -615,7 +632,8 @@ later is required to fix a server side protocol bug.
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
                           current_branch_only=opt.current_branch_only,
-                          no_tags=opt.no_tags)
+                          no_tags=opt.no_tags,
+                          optimized_fetch=opt.optimized_fetch)
 
     if mp.HasChanges:
       syncbuf = SyncBuffer(mp.config)