diff options
| -rw-r--r-- | docs/internal-fs-layout.md | 1 | ||||
| -rw-r--r-- | fetch.py | 38 | ||||
| -rw-r--r-- | git_config.py | 6 | ||||
| -rw-r--r-- | man/repo-gitc-init.1 | 6 | ||||
| -rw-r--r-- | man/repo-init.1 | 12 | ||||
| -rwxr-xr-x | repo | 4 | ||||
| -rw-r--r-- | subcmds/init.py | 83 |
7 files changed, 132 insertions, 18 deletions
diff --git a/docs/internal-fs-layout.md b/docs/internal-fs-layout.md index e3be1731..af6a4523 100644 --- a/docs/internal-fs-layout.md +++ b/docs/internal-fs-layout.md | |||
| @@ -157,6 +157,7 @@ User controlled settings are initialized when running `repo init`. | |||
| 157 | | Setting | `repo init` Option | Use/Meaning | | 157 | | Setting | `repo init` Option | Use/Meaning | |
| 158 | |------------------- |---------------------------|-------------| | 158 | |------------------- |---------------------------|-------------| |
| 159 | | manifest.groups | `--groups` & `--platform` | The manifest groups to sync | | 159 | | manifest.groups | `--groups` & `--platform` | The manifest groups to sync | |
| 160 | | manifest.standalone | `--standalone-manifest` | Download manifest as static file instead of creating checkout | | ||
| 160 | | repo.archive | `--archive` | Use `git archive` for checkouts | | 161 | | repo.archive | `--archive` | Use `git archive` for checkouts | |
| 161 | | repo.clonebundle | `--clone-bundle` | Whether the initial sync used clone.bundle explicitly | | 162 | | repo.clonebundle | `--clone-bundle` | Whether the initial sync used clone.bundle explicitly | |
| 162 | | repo.clonefilter | `--clone-filter` | Filter setting when using [partial git clones] | | 163 | | repo.clonefilter | `--clone-filter` | Filter setting when using [partial git clones] | |
diff --git a/fetch.py b/fetch.py new file mode 100644 index 00000000..5b9997a8 --- /dev/null +++ b/fetch.py | |||
| @@ -0,0 +1,38 @@ | |||
| 1 | # Copyright (C) 2021 The Android Open Source Project | ||
| 2 | # | ||
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
| 4 | # you may not use this file except in compliance with the License. | ||
| 5 | # You may obtain a copy of the License at | ||
| 6 | # | ||
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
| 8 | # | ||
| 9 | # Unless required by applicable law or agreed to in writing, software | ||
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
| 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| 12 | # See the License for the specific language governing permissions and | ||
| 13 | # limitations under the License. | ||
| 14 | |||
| 15 | """This module contains functions used to fetch files from various sources.""" | ||
| 16 | |||
| 17 | import subprocess | ||
| 18 | import sys | ||
| 19 | from urllib.parse import urlparse | ||
| 20 | |||
| 21 | def fetch_file(url): | ||
| 22 | """Fetch a file from the specified source using the appropriate protocol. | ||
| 23 | |||
| 24 | Returns: | ||
| 25 | The contents of the file as bytes. | ||
| 26 | """ | ||
| 27 | scheme = urlparse(url).scheme | ||
| 28 | if scheme == 'gs': | ||
| 29 | cmd = ['gsutil', 'cat', url] | ||
| 30 | try: | ||
| 31 | result = subprocess.run( | ||
| 32 | cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | ||
| 33 | return result.stdout | ||
| 34 | except subprocess.CalledProcessError as e: | ||
| 35 | print('fatal: error running "gsutil": %s' % e.output, | ||
| 36 | file=sys.stderr) | ||
| 37 | sys.exit(1) | ||
| 38 | raise ValueError('unsupported url %s' % url) | ||
diff --git a/git_config.py b/git_config.py index d882239b..778e81a4 100644 --- a/git_config.py +++ b/git_config.py | |||
| @@ -104,6 +104,10 @@ class GitConfig(object): | |||
| 104 | os.path.dirname(self.file), | 104 | os.path.dirname(self.file), |
| 105 | '.repo_' + os.path.basename(self.file) + '.json') | 105 | '.repo_' + os.path.basename(self.file) + '.json') |
| 106 | 106 | ||
| 107 | def ClearCache(self): | ||
| 108 | """Clear the in-memory cache of config.""" | ||
| 109 | self._cache_dict = None | ||
| 110 | |||
| 107 | def Has(self, name, include_defaults=True): | 111 | def Has(self, name, include_defaults=True): |
| 108 | """Return true if this configuration file has the key. | 112 | """Return true if this configuration file has the key. |
| 109 | """ | 113 | """ |
| @@ -399,7 +403,7 @@ class GitConfig(object): | |||
| 399 | if p.Wait() == 0: | 403 | if p.Wait() == 0: |
| 400 | return p.stdout | 404 | return p.stdout |
| 401 | else: | 405 | else: |
| 402 | GitError('git config %s: %s' % (str(args), p.stderr)) | 406 | raise GitError('git config %s: %s' % (str(args), p.stderr)) |
| 403 | 407 | ||
| 404 | 408 | ||
| 405 | class RepoConfig(GitConfig): | 409 | class RepoConfig(GitConfig): |
diff --git a/man/repo-gitc-init.1 b/man/repo-gitc-init.1 index 1d1b23a8..9b61866e 100644 --- a/man/repo-gitc-init.1 +++ b/man/repo-gitc-init.1 | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | .\" DO NOT MODIFY THIS FILE! It was generated by help2man. | 1 | .\" DO NOT MODIFY THIS FILE! It was generated by help2man. |
| 2 | .TH REPO "1" "July 2021" "repo gitc-init" "Repo Manual" | 2 | .TH REPO "1" "September 2021" "repo gitc-init" "Repo Manual" |
| 3 | .SH NAME | 3 | .SH NAME |
| 4 | repo \- repo gitc-init - manual page for repo gitc-init | 4 | repo \- repo gitc-init - manual page for repo gitc-init |
| 5 | .SH SYNOPSIS | 5 | .SH SYNOPSIS |
| @@ -31,6 +31,10 @@ manifest branch or revision (use HEAD for default) | |||
| 31 | \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml | 31 | \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml |
| 32 | initial manifest file | 32 | initial manifest file |
| 33 | .TP | 33 | .TP |
| 34 | \fB\-\-standalone\-manifest\fR | ||
| 35 | download the manifest as a static file rather then | ||
| 36 | create a git checkout of the manifest repo | ||
| 37 | .TP | ||
| 34 | \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR | 38 | \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR |
| 35 | restrict manifest projects to ones with specified | 39 | restrict manifest projects to ones with specified |
| 36 | group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6] | 40 | group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6] |
diff --git a/man/repo-init.1 b/man/repo-init.1 index e860f95d..9957b64d 100644 --- a/man/repo-init.1 +++ b/man/repo-init.1 | |||
| @@ -1,5 +1,5 @@ | |||
| 1 | .\" DO NOT MODIFY THIS FILE! It was generated by help2man. | 1 | .\" DO NOT MODIFY THIS FILE! It was generated by help2man. |
| 2 | .TH REPO "1" "July 2021" "repo init" "Repo Manual" | 2 | .TH REPO "1" "September 2021" "repo init" "Repo Manual" |
| 3 | .SH NAME | 3 | .SH NAME |
| 4 | repo \- repo init - manual page for repo init | 4 | repo \- repo init - manual page for repo init |
| 5 | .SH SYNOPSIS | 5 | .SH SYNOPSIS |
| @@ -31,6 +31,10 @@ manifest branch or revision (use HEAD for default) | |||
| 31 | \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml | 31 | \fB\-m\fR NAME.xml, \fB\-\-manifest\-name\fR=\fI\,NAME\/\fR.xml |
| 32 | initial manifest file | 32 | initial manifest file |
| 33 | .TP | 33 | .TP |
| 34 | \fB\-\-standalone\-manifest\fR | ||
| 35 | download the manifest as a static file rather then | ||
| 36 | create a git checkout of the manifest repo | ||
| 37 | .TP | ||
| 34 | \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR | 38 | \fB\-g\fR GROUP, \fB\-\-groups\fR=\fI\,GROUP\/\fR |
| 35 | restrict manifest projects to ones with specified | 39 | restrict manifest projects to ones with specified |
| 36 | group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6] | 40 | group(s) [default|all|G1,G2,G3|G4,\-G5,\-G6] |
| @@ -137,6 +141,12 @@ equivalent to using \fB\-b\fR HEAD. | |||
| 137 | The optional \fB\-m\fR argument can be used to specify an alternate manifest to be | 141 | The optional \fB\-m\fR argument can be used to specify an alternate manifest to be |
| 138 | used. If no manifest is specified, the manifest default.xml will be used. | 142 | used. If no manifest is specified, the manifest default.xml will be used. |
| 139 | .PP | 143 | .PP |
| 144 | If the \fB\-\-standalone\-manifest\fR argument is set, the manifest will be downloaded | ||
| 145 | directly from the specified \fB\-\-manifest\-url\fR as a static file (rather than setting | ||
| 146 | up a manifest git checkout). With \fB\-\-standalone\-manifest\fR, the manifest will be | ||
| 147 | fully static and will not be re\-downloaded during subsesquent `repo init` and | ||
| 148 | `repo sync` calls. | ||
| 149 | .PP | ||
| 140 | The \fB\-\-reference\fR option can be used to point to a directory that has the content | 150 | The \fB\-\-reference\fR option can be used to point to a directory that has the content |
| 141 | of a \fB\-\-mirror\fR sync. This will make the working directory use as much data as | 151 | of a \fB\-\-mirror\fR sync. This will make the working directory use as much data as |
| 142 | possible from the local reference directory when fetching from the server. This | 152 | possible from the local reference directory when fetching from the server. This |
| @@ -312,6 +312,10 @@ def InitParser(parser, gitc_init=False): | |||
| 312 | metavar='PLATFORM') | 312 | metavar='PLATFORM') |
| 313 | group.add_option('--submodules', action='store_true', | 313 | group.add_option('--submodules', action='store_true', |
| 314 | help='sync any submodules associated with the manifest repo') | 314 | help='sync any submodules associated with the manifest repo') |
| 315 | group.add_option('--standalone-manifest', action='store_true', | ||
| 316 | help='download the manifest as a static file ' | ||
| 317 | 'rather then create a git checkout of ' | ||
| 318 | 'the manifest repo') | ||
| 315 | 319 | ||
| 316 | # Options that only affect manifest project, and not any of the projects | 320 | # Options that only affect manifest project, and not any of the projects |
| 317 | # specified in the manifest itself. | 321 | # specified in the manifest itself. |
diff --git a/subcmds/init.py b/subcmds/init.py index 5671fc24..9c6b2ad9 100644 --- a/subcmds/init.py +++ b/subcmds/init.py | |||
| @@ -15,6 +15,7 @@ | |||
| 15 | import os | 15 | import os |
| 16 | import platform | 16 | import platform |
| 17 | import re | 17 | import re |
| 18 | import subprocess | ||
| 18 | import sys | 19 | import sys |
| 19 | import urllib.parse | 20 | import urllib.parse |
| 20 | 21 | ||
| @@ -24,6 +25,7 @@ from error import ManifestParseError | |||
| 24 | from project import SyncBuffer | 25 | from project import SyncBuffer |
| 25 | from git_config import GitConfig | 26 | from git_config import GitConfig |
| 26 | from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD | 27 | from git_command import git_require, MIN_GIT_VERSION_SOFT, MIN_GIT_VERSION_HARD |
| 28 | import fetch | ||
| 27 | import git_superproject | 29 | import git_superproject |
| 28 | import platform_utils | 30 | import platform_utils |
| 29 | from wrapper import Wrapper | 31 | from wrapper import Wrapper |
| @@ -53,6 +55,12 @@ The optional -m argument can be used to specify an alternate manifest | |||
| 53 | to be used. If no manifest is specified, the manifest default.xml | 55 | to be used. If no manifest is specified, the manifest default.xml |
| 54 | will be used. | 56 | will be used. |
| 55 | 57 | ||
| 58 | If the --standalone-manifest argument is set, the manifest will be downloaded | ||
| 59 | directly from the specified --manifest-url as a static file (rather than | ||
| 60 | setting up a manifest git checkout). With --standalone-manifest, the manifest | ||
| 61 | will be fully static and will not be re-downloaded during subsesquent | ||
| 62 | `repo init` and `repo sync` calls. | ||
| 63 | |||
| 56 | The --reference option can be used to point to a directory that | 64 | The --reference option can be used to point to a directory that |
| 57 | has the content of a --mirror sync. This will make the working | 65 | has the content of a --mirror sync. This will make the working |
| 58 | directory use as much data as possible from the local reference | 66 | directory use as much data as possible from the local reference |
| @@ -112,6 +120,22 @@ to update the working directory files. | |||
| 112 | m = self.manifest.manifestProject | 120 | m = self.manifest.manifestProject |
| 113 | is_new = not m.Exists | 121 | is_new = not m.Exists |
| 114 | 122 | ||
| 123 | # If repo has already been initialized, we take -u with the absence of | ||
| 124 | # --standalone-manifest to mean "transition to a standard repo set up", | ||
| 125 | # which necessitates starting fresh. | ||
| 126 | # If --standalone-manifest is set, we always tear everything down and start | ||
| 127 | # anew. | ||
| 128 | if not is_new: | ||
| 129 | was_standalone_manifest = m.config.GetString('manifest.standalone') | ||
| 130 | if opt.standalone_manifest or ( | ||
| 131 | was_standalone_manifest and opt.manifest_url): | ||
| 132 | m.config.ClearCache() | ||
| 133 | if m.gitdir and os.path.exists(m.gitdir): | ||
| 134 | platform_utils.rmtree(m.gitdir) | ||
| 135 | if m.worktree and os.path.exists(m.worktree): | ||
| 136 | platform_utils.rmtree(m.worktree) | ||
| 137 | |||
| 138 | is_new = not m.Exists | ||
| 115 | if is_new: | 139 | if is_new: |
| 116 | if not opt.manifest_url: | 140 | if not opt.manifest_url: |
| 117 | print('fatal: manifest url is required.', file=sys.stderr) | 141 | print('fatal: manifest url is required.', file=sys.stderr) |
| @@ -136,6 +160,19 @@ to update the working directory files. | |||
| 136 | 160 | ||
| 137 | m._InitGitDir(mirror_git=mirrored_manifest_git) | 161 | m._InitGitDir(mirror_git=mirrored_manifest_git) |
| 138 | 162 | ||
| 163 | # If standalone_manifest is set, mark the project as "standalone" -- we'll | ||
| 164 | # still do much of the manifests.git set up, but will avoid actual syncs to | ||
| 165 | # a remote. | ||
| 166 | standalone_manifest = False | ||
| 167 | if opt.standalone_manifest: | ||
| 168 | standalone_manifest = True | ||
| 169 | elif not opt.manifest_url: | ||
| 170 | # If -u is set and --standalone-manifest is not, then we're not in | ||
| 171 | # standalone mode. Otherwise, use config to infer what we were in the last | ||
| 172 | # init. | ||
| 173 | standalone_manifest = bool(m.config.GetString('manifest.standalone')) | ||
| 174 | m.config.SetString('manifest.standalone', opt.manifest_url) | ||
| 175 | |||
| 139 | self._ConfigureDepth(opt) | 176 | self._ConfigureDepth(opt) |
| 140 | 177 | ||
| 141 | # Set the remote URL before the remote branch as we might need it below. | 178 | # Set the remote URL before the remote branch as we might need it below. |
| @@ -145,22 +182,23 @@ to update the working directory files. | |||
| 145 | r.ResetFetch() | 182 | r.ResetFetch() |
| 146 | r.Save() | 183 | r.Save() |
| 147 | 184 | ||
| 148 | if opt.manifest_branch: | 185 | if not standalone_manifest: |
| 149 | if opt.manifest_branch == 'HEAD': | 186 | if opt.manifest_branch: |
| 150 | opt.manifest_branch = m.ResolveRemoteHead() | 187 | if opt.manifest_branch == 'HEAD': |
| 151 | if opt.manifest_branch is None: | 188 | opt.manifest_branch = m.ResolveRemoteHead() |
| 152 | print('fatal: unable to resolve HEAD', file=sys.stderr) | 189 | if opt.manifest_branch is None: |
| 153 | sys.exit(1) | 190 | print('fatal: unable to resolve HEAD', file=sys.stderr) |
| 154 | m.revisionExpr = opt.manifest_branch | 191 | sys.exit(1) |
| 155 | else: | 192 | m.revisionExpr = opt.manifest_branch |
| 156 | if is_new: | ||
| 157 | default_branch = m.ResolveRemoteHead() | ||
| 158 | if default_branch is None: | ||
| 159 | # If the remote doesn't have HEAD configured, default to master. | ||
| 160 | default_branch = 'refs/heads/master' | ||
| 161 | m.revisionExpr = default_branch | ||
| 162 | else: | 193 | else: |
| 163 | m.PreSync() | 194 | if is_new: |
| 195 | default_branch = m.ResolveRemoteHead() | ||
| 196 | if default_branch is None: | ||
| 197 | # If the remote doesn't have HEAD configured, default to master. | ||
| 198 | default_branch = 'refs/heads/master' | ||
| 199 | m.revisionExpr = default_branch | ||
| 200 | else: | ||
| 201 | m.PreSync() | ||
| 164 | 202 | ||
| 165 | groups = re.split(r'[,\s]+', opt.groups) | 203 | groups = re.split(r'[,\s]+', opt.groups) |
| 166 | all_platforms = ['linux', 'darwin', 'windows'] | 204 | all_platforms = ['linux', 'darwin', 'windows'] |
| @@ -250,6 +288,16 @@ to update the working directory files. | |||
| 250 | if opt.use_superproject is not None: | 288 | if opt.use_superproject is not None: |
| 251 | m.config.SetBoolean('repo.superproject', opt.use_superproject) | 289 | m.config.SetBoolean('repo.superproject', opt.use_superproject) |
| 252 | 290 | ||
| 291 | if standalone_manifest: | ||
| 292 | if is_new: | ||
| 293 | manifest_name = 'default.xml' | ||
| 294 | manifest_data = fetch.fetch_file(opt.manifest_url) | ||
| 295 | dest = os.path.join(m.worktree, manifest_name) | ||
| 296 | os.makedirs(os.path.dirname(dest), exist_ok=True) | ||
| 297 | with open(dest, 'wb') as f: | ||
| 298 | f.write(manifest_data) | ||
| 299 | return | ||
| 300 | |||
| 253 | if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose, | 301 | if not m.Sync_NetworkHalf(is_new=is_new, quiet=opt.quiet, verbose=opt.verbose, |
| 254 | clone_bundle=opt.clone_bundle, | 302 | clone_bundle=opt.clone_bundle, |
| 255 | current_branch_only=opt.current_branch_only, | 303 | current_branch_only=opt.current_branch_only, |
| @@ -426,6 +474,11 @@ to update the working directory files. | |||
| 426 | if opt.archive and opt.mirror: | 474 | if opt.archive and opt.mirror: |
| 427 | self.OptionParser.error('--mirror and --archive cannot be used together.') | 475 | self.OptionParser.error('--mirror and --archive cannot be used together.') |
| 428 | 476 | ||
| 477 | if opt.standalone_manifest and ( | ||
| 478 | opt.manifest_branch or opt.manifest_name != 'default.xml'): | ||
| 479 | self.OptionParser.error('--manifest-branch and --manifest-name cannot' | ||
| 480 | ' be used with --standalone-manifest.') | ||
| 481 | |||
| 429 | if args: | 482 | if args: |
| 430 | if opt.manifest_url: | 483 | if opt.manifest_url: |
| 431 | self.OptionParser.error( | 484 | self.OptionParser.error( |
