1 files changed, 128 insertions, 163 deletions

diff --git a/git_config.py b/git_config.py
index fcd0446c..3cd09391 100644
--- a/git_config.py
+++ b/git_config.py
@@ -13,32 +13,28 @@
 # limitations under the License.
 
 import contextlib
+import datetime
 import errno
 from http.client import HTTPException
 import json
 import os
 import re
-import signal
 import ssl
 import subprocess
 import sys
-try:
-  import threading as _threading
-except ImportError:
-  import dummy_threading as _threading
-import time
 import urllib.error
 import urllib.request
 
 from error import GitError, UploadError
 import platform_utils
 from repo_trace import Trace
-
 from git_command import GitCommand
-from git_command import ssh_sock
-from git_command import terminate_ssh_clients
 from git_refs import R_CHANGES, R_HEADS, R_TAGS
 
+# Prefix that is prepended to all the keys of SyncAnalysisState's data
+# that is saved in the config.
+SYNC_STATE_PREFIX = 'repo.syncstate.'
+
 ID_RE = re.compile(r'^[0-9a-f]{40}$')
 
 REVIEW_CACHE = dict()
@@ -74,6 +70,15 @@ class GitConfig(object):
 
   _USER_CONFIG = '~/.gitconfig'
 
+  _ForSystem = None
+  _SYSTEM_CONFIG = '/etc/gitconfig'
+
+  @classmethod
+  def ForSystem(cls):
+    if cls._ForSystem is None:
+      cls._ForSystem = cls(configfile=cls._SYSTEM_CONFIG)
+    return cls._ForSystem
+
   @classmethod
   def ForUser(cls):
     if cls._ForUser is None:
@@ -99,6 +104,10 @@ class GitConfig(object):
           os.path.dirname(self.file),
           '.repo_' + os.path.basename(self.file) + '.json')
 
+  def ClearCache(self):
+    """Clear the in-memory cache of config."""
+    self._cache_dict = None
+
   def Has(self, name, include_defaults=True):
     """Return true if this configuration file has the key.
     """
@@ -262,6 +271,22 @@ class GitConfig(object):
       self._branches[b.name] = b
     return b
 
+  def GetSyncAnalysisStateData(self):
+    """Returns data to be logged for the analysis of sync performance."""
+    return {k: v for k, v in self.DumpConfigDict().items() if k.startswith(SYNC_STATE_PREFIX)}
+
+  def UpdateSyncAnalysisState(self, options, superproject_logging_data):
+    """Update Config's SYNC_STATE_PREFIX* data with the latest sync data.
+
+    Args:
+      options: Options passed to sync returned from optparse. See _Options().
+      superproject_logging_data: A dictionary of superproject data that is to be logged.
+
+    Returns:
+      SyncAnalysisState object.
+    """
+    return SyncAnalysisState(self, options, superproject_logging_data)
+
   def GetSubSections(self, section):
     """List all subsection names matching $section.*.*
     """
@@ -327,8 +352,8 @@ class GitConfig(object):
       Trace(': parsing %s', self.file)
       with open(self._json) as fd:
         return json.load(fd)
-    except (IOError, ValueError):
-      platform_utils.remove(self._json)
+    except (IOError, ValueErrorl):
+      platform_utils.remove(self._json, missing_ok=True)
       return None
 
   def _SaveJson(self, cache):
@@ -336,8 +361,7 @@ class GitConfig(object):
       with open(self._json, 'w') as fd:
         json.dump(cache, fd, indent=2)
     except (IOError, TypeError):
-      if os.path.exists(self._json):
-        platform_utils.remove(self._json)
+      platform_utils.remove(self._json, missing_ok=True)
 
   def _ReadGit(self):
     """
@@ -347,9 +371,10 @@ class GitConfig(object):
 
     """
     c = {}
-    d = self._do('--null', '--list')
-    if d is None:
+    if not os.path.exists(self.file):
       return c
+
+    d = self._do('--null', '--list')
     for line in d.rstrip('\0').split('\0'):
       if '\n' in line:
         key, val = line.split('\n', 1)
@@ -365,7 +390,10 @@ class GitConfig(object):
     return c
 
   def _do(self, *args):
-    command = ['config', '--file', self.file, '--includes']
+    if self.file == self._SYSTEM_CONFIG:
+      command = ['config', '--system', '--includes']
+    else:
+      command = ['config', '--file', self.file, '--includes']
     command.extend(args)
 
     p = GitCommand(None,
@@ -375,7 +403,7 @@ class GitConfig(object):
     if p.Wait() == 0:
       return p.stdout
     else:
-      GitError('git config %s: %s' % (str(args), p.stderr))
+      raise GitError('git config %s: %s' % (str(args), p.stderr))
 
 
 class RepoConfig(GitConfig):
@@ -440,129 +468,6 @@ class RefSpec(object):
     return s
 
 
-_master_processes = []
-_master_keys = set()
-_ssh_master = True
-_master_keys_lock = None
-
-
-def init_ssh():
-  """Should be called once at the start of repo to init ssh master handling.
-
-  At the moment, all we do is to create our lock.
-  """
-  global _master_keys_lock
-  assert _master_keys_lock is None, "Should only call init_ssh once"
-  _master_keys_lock = _threading.Lock()
-
-
-def _open_ssh(host, port=None):
-  global _ssh_master
-
-  # Bail before grabbing the lock if we already know that we aren't going to
-  # try creating new masters below.
-  if sys.platform in ('win32', 'cygwin'):
-    return False
-
-  # Acquire the lock.  This is needed to prevent opening multiple masters for
-  # the same host when we're running "repo sync -jN" (for N > 1) _and_ the
-  # manifest <remote fetch="ssh://xyz"> specifies a different host from the
-  # one that was passed to repo init.
-  _master_keys_lock.acquire()
-  try:
-
-    # Check to see whether we already think that the master is running; if we
-    # think it's already running, return right away.
-    if port is not None:
-      key = '%s:%s' % (host, port)
-    else:
-      key = host
-
-    if key in _master_keys:
-      return True
-
-    if not _ssh_master or 'GIT_SSH' in os.environ:
-      # Failed earlier, so don't retry.
-      return False
-
-    # We will make two calls to ssh; this is the common part of both calls.
-    command_base = ['ssh',
-                    '-o', 'ControlPath %s' % ssh_sock(),
-                    host]
-    if port is not None:
-      command_base[1:1] = ['-p', str(port)]
-
-    # Since the key wasn't in _master_keys, we think that master isn't running.
-    # ...but before actually starting a master, we'll double-check.  This can
-    # be important because we can't tell that that 'git@myhost.com' is the same
-    # as 'myhost.com' where "User git" is setup in the user's ~/.ssh/config file.
-    check_command = command_base + ['-O', 'check']
-    try:
-      Trace(': %s', ' '.join(check_command))
-      check_process = subprocess.Popen(check_command,
-                                       stdout=subprocess.PIPE,
-                                       stderr=subprocess.PIPE)
-      check_process.communicate()  # read output, but ignore it...
-      isnt_running = check_process.wait()
-
-      if not isnt_running:
-        # Our double-check found that the master _was_ infact running.  Add to
-        # the list of keys.
-        _master_keys.add(key)
-        return True
-    except Exception:
-      # Ignore excpetions.  We we will fall back to the normal command and print
-      # to the log there.
-      pass
-
-    command = command_base[:1] + ['-M', '-N'] + command_base[1:]
-    try:
-      Trace(': %s', ' '.join(command))
-      p = subprocess.Popen(command)
-    except Exception as e:
-      _ssh_master = False
-      print('\nwarn: cannot enable ssh control master for %s:%s\n%s'
-            % (host, port, str(e)), file=sys.stderr)
-      return False
-
-    time.sleep(1)
-    ssh_died = (p.poll() is not None)
-    if ssh_died:
-      return False
-
-    _master_processes.append(p)
-    _master_keys.add(key)
-    return True
-  finally:
-    _master_keys_lock.release()
-
-
-def close_ssh():
-  global _master_keys_lock
-
-  terminate_ssh_clients()
-
-  for p in _master_processes:
-    try:
-      os.kill(p.pid, signal.SIGTERM)
-      p.wait()
-    except OSError:
-      pass
-  del _master_processes[:]
-  _master_keys.clear()
-
-  d = ssh_sock(create=False)
-  if d:
-    try:
-      platform_utils.rmdir(os.path.dirname(d))
-    except OSError:
-      pass
-
-  # We're done with the lock, so we can delete it.
-  _master_keys_lock = None
-
-
-URI_SCP = re.compile(r'^([^@:]*@?[^:/]{1,}):')
 URI_ALL = re.compile(r'^([a-z][a-z+-]*)://([^@/]*@?[^/]*)/')
 
 
@@ -614,27 +519,6 @@ def GetUrlCookieFile(url, quiet):
   yield cookiefile, None
 
 
-def _preconnect(url):
-  m = URI_ALL.match(url)
-  if m:
-    scheme = m.group(1)
-    host = m.group(2)
-    if ':' in host:
-      host, port = host.split(':')
-    else:
-      port = None
-    if scheme in ('ssh', 'git+ssh', 'ssh+git'):
-      return _open_ssh(host, port)
-    return False
-
-  m = URI_SCP.match(url)
-  if m:
-    host = m.group(1)
-    return _open_ssh(host)
-
-  return False
-
-
 class Remote(object):
   """Configuration options related to a remote.
   """
@@ -671,9 +555,23 @@ class Remote(object):
 
     return self.url.replace(longest, longestUrl, 1)
 
-  def PreConnectFetch(self):
+  def PreConnectFetch(self, ssh_proxy):
+    """Run any setup for this remote before we connect to it.
+
+    In practice, if the remote is using SSH, we'll attempt to create a new
+    SSH master session to it for reuse across projects.
+
+    Args:
+      ssh_proxy: The SSH settings for managing master sessions.
+
+    Returns:
+      Whether the preconnect phase for this remote was successful.
+    """
+    if not ssh_proxy:
+      return True
+
     connectionUrl = self._InsteadOf()
-    return _preconnect(connectionUrl)
+    return ssh_proxy.preconnect(connectionUrl)
 
   def ReviewUrl(self, userEmail, validate_certs):
     if self._review_url is None:
@@ -844,3 +742,70 @@ class Branch(object):
   def _Get(self, key, all_keys=False):
     key = 'branch.%s.%s' % (self.name, key)
     return self._config.GetString(key, all_keys=all_keys)
+
+
+class SyncAnalysisState:
+  """Configuration options related to logging of sync state for analysis.
+
+  This object is versioned.
+  """
+  def __init__(self, config, options, superproject_logging_data):
+    """Initializes SyncAnalysisState.
+
+    Saves the following data into the |config| object.
+    - sys.argv, options, superproject's logging data.
+    - repo.*, branch.* and remote.* parameters from config object.
+    - Current time as synctime.
+    - Version number of the object.
+
+    All the keys saved by this object are prepended with SYNC_STATE_PREFIX.
+
+    Args:
+      config: GitConfig object to store all options.
+      options: Options passed to sync returned from optparse. See _Options().
+      superproject_logging_data: A dictionary of superproject data that is to be logged.
+    """
+    self._config = config
+    now = datetime.datetime.utcnow()
+    self._Set('main.synctime', now.isoformat() + 'Z')
+    self._Set('main.version', '1')
+    self._Set('sys.argv', sys.argv)
+    for key, value in superproject_logging_data.items():
+      self._Set(f'superproject.{key}', value)
+    for key, value in options.__dict__.items():
+      self._Set(f'options.{key}', value)
+    config_items = config.DumpConfigDict().items()
+    EXTRACT_NAMESPACES = {'repo', 'branch', 'remote'}
+    self._SetDictionary({k: v for k, v in config_items
+                         if not k.startswith(SYNC_STATE_PREFIX) and
+                         k.split('.', 1)[0] in EXTRACT_NAMESPACES})
+
+  def _SetDictionary(self, data):
+    """Save all key/value pairs of |data| dictionary.
+
+    Args:
+      data: A dictionary whose key/value are to be saved.
+    """
+    for key, value in data.items():
+      self._Set(key, value)
+
+  def _Set(self, key, value):
+    """Set the |value| for a |key| in the |_config| member.
+
+    |key| is prepended with the value of SYNC_STATE_PREFIX constant.
+
+    Args:
+      key: Name of the key.
+      value: |value| could be of any type. If it is 'bool', it will be saved
+             as a Boolean and for all other types, it will be saved as a String.
+    """
+    if value is None:
+      return
+    sync_key = f'{SYNC_STATE_PREFIX}{key}'
+    sync_key = sync_key.replace('_', '')
+    if isinstance(value, str):
+      self._config.SetString(sync_key, value)
+    elif isinstance(value, bool):
+      self._config.SetBoolean(sync_key, value)
+    else:
+      self._config.SetString(sync_key, str(value))