9 files changed, 223 insertions, 17 deletions

diff --git a/command.py b/command.py
index 207ef46b..38cacd3b 100644
--- a/command.py
+++ b/command.py
@@ -126,7 +126,7 @@ class Command(object):
         pass
     return project
 
-  def GetProjects(self, args, missing_ok=False, submodules_ok=False):
+  def GetProjects(self, args, groups='', missing_ok=False, submodules_ok=False):
     """A list of projects that match the arguments.
     """
     all_projects_list = self.manifest.projects
@@ -134,7 +134,8 @@ class Command(object):
 
     mp = self.manifest.manifestProject
 
-    groups = mp.config.GetString('manifest.groups')
+    if not groups:
+        groups = mp.config.GetString('manifest.groups')
     if not groups:
       groups = 'default,platform-' + platform.system().lower()
     groups = [x for x in re.split(r'[,\s]+', groups) if x]
diff --git a/gitc_utils.py b/gitc_utils.py
new file mode 100644
index 00000000..bf79bd28
--- /dev/null
+++ b/gitc_utils.py
@@ -0,0 +1,69 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import shutil
+
+import git_command
+import git_config
+
+
+# TODO (sbasi) - Remove this constant and fetch manifest dir from /gitc/.config
+GITC_MANIFEST_DIR = '/usr/local/google/gitc/'
+GITC_FS_ROOT_DIR = '/gitc/manifest-rw/'
+NUM_BATCH_RETRIEVE_REVISIONID = 300
+
+def _set_project_revisions(projects):
+  """Sets the revisionExpr for a list of projects.
+
+  Because of the limit of open file descriptors allowed, length of projects
+  should not be overly large. Recommend calling this function multiple times
+  with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects.
+
+  @param projects: List of project objects to set the revionExpr for.
+  """
+  # Retrieve the commit id for each project based off of it's current
+  # revisionExpr and it is not already a commit id.
+  project_gitcmds = [(
+      project, git_command.GitCommand(None,
+                                      ['ls-remote',
+                                       project.remote.url,
+                                       project.revisionExpr],
+                                      capture_stdout=True, cwd='/tmp'))
+      for project in projects if not git_config.IsId(project.revisionExpr)]
+  for proj, gitcmd in project_gitcmds:
+    if gitcmd.Wait():
+      print('FATAL: Failed to retrieve revisionExpr for %s' % project)
+      sys.exit(1)
+    proj.revisionExpr = gitcmd.stdout.split('\t')[0]
+
+def generate_gitc_manifest(client_dir, manifest):
+  """Generate a manifest for shafsd to use for this GITC client.
+
+  @param client_dir: GITC client directory to install the .manifest file in.
+  @param manifest: XmlManifest object representing the repo manifest.
+  """
+  print('Generating GITC Manifest by fetching revision SHAs for each '
+        'project.')
+  project_gitcmd_dict = {}
+  index = 0
+  while index < len(manifest.projects):
+    _set_project_revisions(
+        manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)])
+    index += NUM_BATCH_RETRIEVE_REVISIONID
+  # Save the manifest.
+  with open(os.path.join(client_dir, '.manifest'), 'w') as f:
+    manifest.Save(f)
diff --git a/manifest_xml.py b/manifest_xml.py
index 7e719600..6dc01a47 100644
--- a/manifest_xml.py
+++ b/manifest_xml.py
@@ -303,6 +303,9 @@ class XmlManifest(object):
       if p.sync_s:
         e.setAttribute('sync-s', 'true')
 
+      if p.clone_depth:
+        e.setAttribute('clone-depth', str(p.clone_depth))
+
       if p.subprojects:
         subprojects = set(subp.name for subp in p.subprojects)
         output_projects(p, e, list(sorted(subprojects)))
diff --git a/project.py b/project.py
index fb9df2aa..82fa8487 100644
--- a/project.py
+++ b/project.py
@@ -1877,6 +1877,13 @@ class Project(object):
 
     if depth:
       cmd.append('--depth=%s' % depth)
+    else:
+      # If this repo has shallow objects, then we don't know which refs have
+      # shallow objects or not. Tell git to unshallow all fetched refs.  Don't
+      # do this with projects that don't have shallow objects, since it is less
+      # efficient.
+      if os.path.exists(os.path.join(self.gitdir, 'shallow')):
+        cmd.append('--depth=2147483647')
 
     if quiet:
       cmd.append('--quiet')
@@ -1914,16 +1921,6 @@ class Project(object):
           spec.append(str((u'+%s:' % branch) + remote.ToLocal(branch)))
     cmd.extend(spec)
 
-    shallowfetch = self.config.GetString('repo.shallowfetch')
-    if shallowfetch and shallowfetch != ' '.join(spec):
-      GitCommand(self, ['fetch', '--depth=2147483647', name]
-                 + shallowfetch.split(),
-                 bare=True, ssh_proxy=ssh_proxy).Wait()
-    if depth:
-      self.config.SetString('repo.shallowfetch', ' '.join(spec))
-    else:
-      self.config.SetString('repo.shallowfetch', None)
-
     ok = False
     for _i in range(2):
       gitcmd = GitCommand(self, cmd, bare=True, ssh_proxy=ssh_proxy)
diff --git a/repo b/repo
index f12354a4..bf8fa3dc 100755
--- a/repo
+++ b/repo
@@ -108,6 +108,7 @@ S_repo = 'repo'                 # special repo repository
 S_manifests = 'manifests'       # special manifest repository
 REPO_MAIN = S_repo + '/main.py' # main script
 MIN_PYTHON_VERSION = (2, 6)     # minimum supported python version
+GITC_MANIFEST_DIR = '/usr/local/google/gitc'
 
 
 import errno
@@ -212,14 +213,25 @@ group.add_option('--config-name',
                  dest='config_name', action="store_true", default=False,
                  help='Always prompt for name/e-mail')
 
+def _GitcInitOptions(init_optparse):
+  g = init_optparse.add_option_group('GITC options')
+  g.add_option('-f', '--manifest-file',
+               dest='manifest_file',
+               help='Optional manifest file to use for this GITC client.')
+  g.add_option('-c', '--gitc-client',
+               dest='gitc_client',
+               help='The name for the new gitc_client instance.')
+
 class CloneFailure(Exception):
   """Indicate the remote clone of repo itself failed.
   """
 
 
-def _Init(args):
+def _Init(args, gitc_init=False):
   """Installs repo by cloning it over the network.
   """
+  if gitc_init:
+    _GitcInitOptions(init_optparse)
   opt, args = init_optparse.parse_args(args)
   if args:
     init_optparse.print_usage()
@@ -242,6 +254,15 @@ def _Init(args):
     raise CloneFailure()
 
   try:
+    if gitc_init:
+      client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client)
+      if not os.path.exists(client_dir):
+        os.makedirs(client_dir)
+      os.chdir(client_dir)
+      if os.path.exists(repodir):
+        # This GITC Client has already initialized repo so continue.
+        return
+
     os.mkdir(repodir)
   except OSError as e:
     if e.errno != errno.EEXIST:
@@ -732,11 +753,11 @@ def main(orig_args):
       _Help(args)
     if not cmd:
       _NotInstalled()
-    if cmd == 'init':
+    if cmd == 'init' or cmd == 'gitc-init':
       if my_git:
         _SetDefaultsTo(my_git)
       try:
-        _Init(args)
+        _Init(args, gitc_init=(cmd == 'gitc-init'))
       except CloneFailure:
         shutil.rmtree(os.path.join(repodir, S_repo), ignore_errors=True)
         sys.exit(1)
diff --git a/subcmds/forall.py b/subcmds/forall.py
index b93cd6d0..96dc99d1 100644
--- a/subcmds/forall.py
+++ b/subcmds/forall.py
@@ -120,6 +120,9 @@ without iterating through the remaining projects.
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
                  help="Execute the command only on projects matching regex or wildcard expression")
+    p.add_option('-g', '--groups',
+                 dest='groups',
+                 help="Execute the command only on projects matching the specified groups")
     p.add_option('-c', '--command',
                  help='Command (and arguments) to execute',
                  dest='command',
@@ -213,7 +216,7 @@ without iterating through the remaining projects.
       self.manifest.Override(smart_sync_manifest_path)
 
     if not opt.regex:
-      projects = self.GetProjects(args)
+      projects = self.GetProjects(args, groups=opt.groups)
     else:
       projects = self.FindProjects(args)
 
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py
new file mode 100644
index 00000000..03d8cc30
--- /dev/null
+++ b/subcmds/gitc_init.py
@@ -0,0 +1,80 @@
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import print_function
+import os
+import shutil
+import sys
+
+import gitc_utils
+from subcmds import init
+
+
+class GitcInit(init.Init):
+  common = True
+  helpSummary = "Initialize a GITC Client."
+  helpUsage = """
+%prog [options] [client name]
+"""
+  helpDescription = """
+The '%prog' command is ran to initialize a new GITC client for use
+with the GITC file system.
+
+This command will setup the client directory, initialize repo, just
+like repo init does, and then downloads the manifest collection
+and installs in in the .repo/directory of the GITC client.
+
+Once this is done, a GITC manifest is generated by pulling the HEAD
+SHA for each project and generates the properly formatted XML file
+and installs it as .manifest in the GITC client directory.
+
+The -c argument is required to specify the GITC client name.
+
+The optional -f argument can be used to specify the manifest file to
+use for this GITC client.
+"""
+
+  def _Options(self, p):
+    super(GitcInit, self)._Options(p)
+    g = p.add_option_group('GITC options')
+    g.add_option('-f', '--manifest-file',
+                 dest='manifest_file',
+                 help='Optional manifest file to use for this GITC client.')
+    g.add_option('-c', '--gitc-client',
+                 dest='gitc_client',
+                 help='The name for the new gitc_client instance.')
+
+  def Execute(self, opt, args):
+    if not opt.gitc_client:
+      print('fatal: gitc client (-c) is required', file=sys.stderr)
+      sys.exit(1)
+    self.client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
+                                   opt.gitc_client)
+    if not os.path.exists(gitc_utils.GITC_MANIFEST_DIR):
+      os.makedirs(gitc_utils.GITC_MANIFEST_DIR)
+    if not os.path.exists(self.client_dir):
+      os.mkdir(self.client_dir)
+    super(GitcInit, self).Execute(opt, args)
+    # Make the destination manifest file a symlink to repo's so both repo and
+    # GITC refer to the same manifest.
+    if opt.manifest_file:
+      if not os.path.exists(opt.manifest_file):
+        print('fatal: Specified manifest file %s does not exist.' %
+              opt.manifest_file)
+        sys.exit(1)
+      self.manifest.Override(opt.manifest_file)
+    gitc_utils.generate_gitc_manifest(self.client_dir, self.manifest)
+    print('Please run `cd %s` to view your GITC client.' %
+          os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client))
\ No newline at end of file
diff --git a/subcmds/list.py b/subcmds/list.py
index 945c28d8..ca51c5f7 100644
--- a/subcmds/list.py
+++ b/subcmds/list.py
@@ -35,6 +35,9 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
     p.add_option('-r', '--regex',
                  dest='regex', action='store_true',
                  help="Filter the project list based on regex or wildcard matching of strings")
+    p.add_option('-g', '--groups',
+                 dest='groups',
+                 help="Filter the project list based on the groups the project is in")
     p.add_option('-f', '--fullpath',
                  dest='fullpath', action='store_true',
                  help="Display the full work tree path instead of the relative path")
@@ -62,7 +65,7 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'.
       sys.exit(1)
 
     if not opt.regex:
-      projects = self.GetProjects(args)
+      projects = self.GetProjects(args, groups=opt.groups)
     else:
       projects = self.FindProjects(args)
 
diff --git a/subcmds/sync.py b/subcmds/sync.py
index 43d450be..652a0c0d 100644
--- a/subcmds/sync.py
+++ b/subcmds/sync.py
@@ -58,6 +58,7 @@ except ImportError:
 
 from git_command import GIT, git_require
 from git_refs import R_HEADS, HEAD
+import gitc_utils
 from project import Project
 from project import RemoteSpec
 from command import Command, MirrorSafeCommand
@@ -184,6 +185,9 @@ later is required to fix a server side protocol bug.
                  help="overwrite an existing git directory if it needs to "
                       "point to a different object directory. WARNING: this "
                       "may cause loss of data")
+    p.add_option('--force-gitc',
+                 dest='force_gitc', action='store_true',
+                 help="actually sync sources in the gitc client directory.")
     p.add_option('-l', '--local-only',
                  dest='local_only', action='store_true',
                  help="only update working tree, don't fetch")
@@ -526,6 +530,25 @@ later is required to fix a server side protocol bug.
         print('error: both -u and -p must be given', file=sys.stderr)
         sys.exit(1)
 
+    cwd = os.getcwd()
+    if cwd.startswith(gitc_utils.GITC_MANIFEST_DIR) and not opt.force_gitc:
+      print('WARNING this will pull all the sources like a normal repo sync.\n'
+            '\nIf you want to update your GITC Client View please rerun this '
+            'command in \n%s%s.\nOr if you actually want to pull the sources, '
+            'rerun with --force-gitc.' %
+            (gitc_utils.GITC_FS_ROOT_DIR,
+             cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1]))
+      sys.exit(1)
+
+    self._gitc_sync = False
+    if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR):
+      self._gitc_sync = True
+      self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split(
+          '/')[0]
+      self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR,
+                                      self._client_name)
+      print('Updating GITC client: %s' % self._client_name)
+
     if opt.manifest_name:
       self.manifest.Override(opt.manifest_name)
 
@@ -642,6 +665,12 @@ later is required to fix a server side protocol bug.
     if opt.repo_upgraded:
       _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
 
+    if self._gitc_sync:
+      gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest)
+      print('GITC client successfully synced.')
+      return
+
+
     if not opt.local_only:
       mp.Sync_NetworkHalf(quiet=opt.quiet,
                           current_branch_only=opt.current_branch_only,