diff options
| -rw-r--r-- | gitc_utils.py | 69 | ||||
| -rw-r--r-- | subcmds/gitc_init.py | 63 | ||||
| -rw-r--r-- | subcmds/sync.py | 29 |
3 files changed, 108 insertions, 53 deletions
diff --git a/gitc_utils.py b/gitc_utils.py new file mode 100644 index 00000000..bf79bd28 --- /dev/null +++ b/gitc_utils.py | |||
| @@ -0,0 +1,69 @@ | |||
| 1 | # | ||
| 2 | # Copyright (C) 2015 The Android Open Source Project | ||
| 3 | # | ||
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 5 | # you may not use this file except in compliance with the License. | ||
| 6 | # You may obtain a copy of the License at | ||
| 7 | # | ||
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
| 9 | # | ||
| 10 | # Unless required by applicable law or agreed to in writing, software | ||
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 13 | # See the License for the specific language governing permissions and | ||
| 14 | # limitations under the License. | ||
| 15 | |||
| 16 | from __future__ import print_function | ||
| 17 | import os | ||
| 18 | import shutil | ||
| 19 | |||
| 20 | import git_command | ||
| 21 | import git_config | ||
| 22 | |||
| 23 | |||
| 24 | # TODO (sbasi) - Remove this constant and fetch manifest dir from /gitc/.config | ||
| 25 | GITC_MANIFEST_DIR = '/usr/local/google/gitc/' | ||
| 26 | GITC_FS_ROOT_DIR = '/gitc/manifest-rw/' | ||
| 27 | NUM_BATCH_RETRIEVE_REVISIONID = 300 | ||
| 28 | |||
| 29 | def _set_project_revisions(projects): | ||
| 30 | """Sets the revisionExpr for a list of projects. | ||
| 31 | |||
| 32 | Because of the limit of open file descriptors allowed, length of projects | ||
| 33 | should not be overly large. Recommend calling this function multiple times | ||
| 34 | with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. | ||
| 35 | |||
| 36 | @param projects: List of project objects to set the revionExpr for. | ||
| 37 | """ | ||
| 38 | # Retrieve the commit id for each project based off of it's current | ||
| 39 | # revisionExpr and it is not already a commit id. | ||
| 40 | project_gitcmds = [( | ||
| 41 | project, git_command.GitCommand(None, | ||
| 42 | ['ls-remote', | ||
| 43 | project.remote.url, | ||
| 44 | project.revisionExpr], | ||
| 45 | capture_stdout=True, cwd='/tmp')) | ||
| 46 | for project in projects if not git_config.IsId(project.revisionExpr)] | ||
| 47 | for proj, gitcmd in project_gitcmds: | ||
| 48 | if gitcmd.Wait(): | ||
| 49 | print('FATAL: Failed to retrieve revisionExpr for %s' % project) | ||
| 50 | sys.exit(1) | ||
| 51 | proj.revisionExpr = gitcmd.stdout.split('\t')[0] | ||
| 52 | |||
| 53 | def generate_gitc_manifest(client_dir, manifest): | ||
| 54 | """Generate a manifest for shafsd to use for this GITC client. | ||
| 55 | |||
| 56 | @param client_dir: GITC client directory to install the .manifest file in. | ||
| 57 | @param manifest: XmlManifest object representing the repo manifest. | ||
| 58 | """ | ||
| 59 | print('Generating GITC Manifest by fetching revision SHAs for each ' | ||
| 60 | 'project.') | ||
| 61 | project_gitcmd_dict = {} | ||
| 62 | index = 0 | ||
| 63 | while index < len(manifest.projects): | ||
| 64 | _set_project_revisions( | ||
| 65 | manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)]) | ||
| 66 | index += NUM_BATCH_RETRIEVE_REVISIONID | ||
| 67 | # Save the manifest. | ||
| 68 | with open(os.path.join(client_dir, '.manifest'), 'w') as f: | ||
| 69 | manifest.Save(f) | ||
diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py index 9b9cefda..03d8cc30 100644 --- a/subcmds/gitc_init.py +++ b/subcmds/gitc_init.py | |||
| @@ -18,15 +18,10 @@ import os | |||
| 18 | import shutil | 18 | import shutil |
| 19 | import sys | 19 | import sys |
| 20 | 20 | ||
| 21 | import git_command | 21 | import gitc_utils |
| 22 | from subcmds import init | 22 | from subcmds import init |
| 23 | 23 | ||
| 24 | 24 | ||
| 25 | GITC_MANIFEST_DIR = '/usr/local/google/gitc' | ||
| 26 | GITC_FS_ROOT_DIR = '/gitc/sha/rw' | ||
| 27 | NUM_BATCH_RETRIEVE_REVISIONID = 300 | ||
| 28 | |||
| 29 | |||
| 30 | class GitcInit(init.Init): | 25 | class GitcInit(init.Init): |
| 31 | common = True | 26 | common = True |
| 32 | helpSummary = "Initialize a GITC Client." | 27 | helpSummary = "Initialize a GITC Client." |
| @@ -65,59 +60,21 @@ use for this GITC client. | |||
| 65 | if not opt.gitc_client: | 60 | if not opt.gitc_client: |
| 66 | print('fatal: gitc client (-c) is required', file=sys.stderr) | 61 | print('fatal: gitc client (-c) is required', file=sys.stderr) |
| 67 | sys.exit(1) | 62 | sys.exit(1) |
| 68 | self.client_dir = os.path.join(GITC_MANIFEST_DIR, opt.gitc_client) | 63 | self.client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, |
| 69 | if not os.path.exists(GITC_MANIFEST_DIR): | 64 | opt.gitc_client) |
| 70 | os.makedirs(GITC_MANIFEST_DIR) | 65 | if not os.path.exists(gitc_utils.GITC_MANIFEST_DIR): |
| 66 | os.makedirs(gitc_utils.GITC_MANIFEST_DIR) | ||
| 71 | if not os.path.exists(self.client_dir): | 67 | if not os.path.exists(self.client_dir): |
| 72 | os.mkdir(self.client_dir) | 68 | os.mkdir(self.client_dir) |
| 73 | super(GitcInit, self).Execute(opt, args) | 69 | super(GitcInit, self).Execute(opt, args) |
| 70 | # Make the destination manifest file a symlink to repo's so both repo and | ||
| 71 | # GITC refer to the same manifest. | ||
| 74 | if opt.manifest_file: | 72 | if opt.manifest_file: |
| 75 | if not os.path.exists(opt.manifest_file): | 73 | if not os.path.exists(opt.manifest_file): |
| 76 | print('fatal: Specified manifest file %s does not exist.' % | 74 | print('fatal: Specified manifest file %s does not exist.' % |
| 77 | opt.manifest_file) | 75 | opt.manifest_file) |
| 78 | sys.exit(1) | 76 | sys.exit(1) |
| 79 | shutil.copyfile(opt.manifest_file, | 77 | self.manifest.Override(opt.manifest_file) |
| 80 | os.path.join(self.client_dir, '.manifest')) | 78 | gitc_utils.generate_gitc_manifest(self.client_dir, self.manifest) |
| 81 | else: | ||
| 82 | self._GenerateGITCManifest() | ||
| 83 | print('Please run `cd %s` to view your GITC client.' % | 79 | print('Please run `cd %s` to view your GITC client.' % |
| 84 | os.path.join(GITC_FS_ROOT_DIR, opt.gitc_client)) | 80 | os.path.join(gitc_utils.GITC_FS_ROOT_DIR, opt.gitc_client)) \ No newline at end of file |
| 85 | |||
| 86 | def _SetProjectRevisions(self, projects, branch): | ||
| 87 | """Sets the revisionExpr for a list of projects. | ||
| 88 | |||
| 89 | Because of the limit of open file descriptors allowed, length of projects | ||
| 90 | should not be overly large. Recommend calling this function multiple times | ||
| 91 | with each call not exceeding NUM_BATCH_RETRIEVE_REVISIONID projects. | ||
| 92 | |||
| 93 | @param projects: List of project objects to set the revionExpr for. | ||
| 94 | @param branch: The remote branch to retrieve the SHA from. If branch is | ||
| 95 | None, 'HEAD' is used. | ||
| 96 | """ | ||
| 97 | project_gitcmds = [( | ||
| 98 | project, git_command.GitCommand(None, | ||
| 99 | ['ls-remote', | ||
| 100 | project.remote.url, | ||
| 101 | branch], capture_stdout=True)) | ||
| 102 | for project in projects] | ||
| 103 | for proj, gitcmd in project_gitcmds: | ||
| 104 | if gitcmd.Wait(): | ||
| 105 | print('FATAL: Failed to retrieve revisionID for %s' % project) | ||
| 106 | sys.exit(1) | ||
| 107 | proj.revisionExpr = gitcmd.stdout.split('\t')[0] | ||
| 108 | |||
| 109 | def _GenerateGITCManifest(self): | ||
| 110 | """Generate a manifest for shafsd to use for this GITC client.""" | ||
| 111 | print('Generating GITC Manifest by fetching revision SHAs for each ' | ||
| 112 | 'project.') | ||
| 113 | manifest = self.manifest | ||
| 114 | project_gitcmd_dict = {} | ||
| 115 | index = 0 | ||
| 116 | while index < len(manifest.projects): | ||
| 117 | self._SetProjectRevisions( | ||
| 118 | manifest.projects[index:(index+NUM_BATCH_RETRIEVE_REVISIONID)], | ||
| 119 | manifest.default.revisionExpr) | ||
| 120 | index += NUM_BATCH_RETRIEVE_REVISIONID | ||
| 121 | # Save the manifest. | ||
| 122 | with open(os.path.join(self.client_dir, '.manifest'), 'w') as f: | ||
| 123 | manifest.Save(f) | ||
diff --git a/subcmds/sync.py b/subcmds/sync.py index 43d450be..652a0c0d 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
| @@ -58,6 +58,7 @@ except ImportError: | |||
| 58 | 58 | ||
| 59 | from git_command import GIT, git_require | 59 | from git_command import GIT, git_require |
| 60 | from git_refs import R_HEADS, HEAD | 60 | from git_refs import R_HEADS, HEAD |
| 61 | import gitc_utils | ||
| 61 | from project import Project | 62 | from project import Project |
| 62 | from project import RemoteSpec | 63 | from project import RemoteSpec |
| 63 | from command import Command, MirrorSafeCommand | 64 | from command import Command, MirrorSafeCommand |
| @@ -184,6 +185,9 @@ later is required to fix a server side protocol bug. | |||
| 184 | help="overwrite an existing git directory if it needs to " | 185 | help="overwrite an existing git directory if it needs to " |
| 185 | "point to a different object directory. WARNING: this " | 186 | "point to a different object directory. WARNING: this " |
| 186 | "may cause loss of data") | 187 | "may cause loss of data") |
| 188 | p.add_option('--force-gitc', | ||
| 189 | dest='force_gitc', action='store_true', | ||
| 190 | help="actually sync sources in the gitc client directory.") | ||
| 187 | p.add_option('-l', '--local-only', | 191 | p.add_option('-l', '--local-only', |
| 188 | dest='local_only', action='store_true', | 192 | dest='local_only', action='store_true', |
| 189 | help="only update working tree, don't fetch") | 193 | help="only update working tree, don't fetch") |
| @@ -526,6 +530,25 @@ later is required to fix a server side protocol bug. | |||
| 526 | print('error: both -u and -p must be given', file=sys.stderr) | 530 | print('error: both -u and -p must be given', file=sys.stderr) |
| 527 | sys.exit(1) | 531 | sys.exit(1) |
| 528 | 532 | ||
| 533 | cwd = os.getcwd() | ||
| 534 | if cwd.startswith(gitc_utils.GITC_MANIFEST_DIR) and not opt.force_gitc: | ||
| 535 | print('WARNING this will pull all the sources like a normal repo sync.\n' | ||
| 536 | '\nIf you want to update your GITC Client View please rerun this ' | ||
| 537 | 'command in \n%s%s.\nOr if you actually want to pull the sources, ' | ||
| 538 | 'rerun with --force-gitc.' % | ||
| 539 | (gitc_utils.GITC_FS_ROOT_DIR, | ||
| 540 | cwd.split(gitc_utils.GITC_MANIFEST_DIR)[1])) | ||
| 541 | sys.exit(1) | ||
| 542 | |||
| 543 | self._gitc_sync = False | ||
| 544 | if cwd.startswith(gitc_utils.GITC_FS_ROOT_DIR): | ||
| 545 | self._gitc_sync = True | ||
| 546 | self._client_name = cwd.split(gitc_utils.GITC_FS_ROOT_DIR)[1].split( | ||
| 547 | '/')[0] | ||
| 548 | self._client_dir = os.path.join(gitc_utils.GITC_MANIFEST_DIR, | ||
| 549 | self._client_name) | ||
| 550 | print('Updating GITC client: %s' % self._client_name) | ||
| 551 | |||
| 529 | if opt.manifest_name: | 552 | if opt.manifest_name: |
| 530 | self.manifest.Override(opt.manifest_name) | 553 | self.manifest.Override(opt.manifest_name) |
| 531 | 554 | ||
| @@ -642,6 +665,12 @@ later is required to fix a server side protocol bug. | |||
| 642 | if opt.repo_upgraded: | 665 | if opt.repo_upgraded: |
| 643 | _PostRepoUpgrade(self.manifest, quiet=opt.quiet) | 666 | _PostRepoUpgrade(self.manifest, quiet=opt.quiet) |
| 644 | 667 | ||
| 668 | if self._gitc_sync: | ||
| 669 | gitc_utils.generate_gitc_manifest(self._client_dir, self.manifest) | ||
| 670 | print('GITC client successfully synced.') | ||
| 671 | return | ||
| 672 | |||
| 673 | |||
| 645 | if not opt.local_only: | 674 | if not opt.local_only: |
| 646 | mp.Sync_NetworkHalf(quiet=opt.quiet, | 675 | mp.Sync_NetworkHalf(quiet=opt.quiet, |
| 647 | current_branch_only=opt.current_branch_only, | 676 | current_branch_only=opt.current_branch_only, |
