From cc879a97c3e2614d19b15b4661c3cab4d33139c9 Mon Sep 17 00:00:00 2001 From: LaMont Jones Date: Thu, 18 Nov 2021 22:40:18 +0000 Subject: Add multi-manifest support with element To be addressed in another change: - a partial `repo sync` (with a list of projects/paths to sync) requires `--this-tree-only`. Change-Id: I6c7400bf001540e9d7694fa70934f8f204cb5f57 Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/322657 Tested-by: LaMont Jones Reviewed-by: Mike Frysinger --- command.py | 84 +++++++-- docs/internal-fs-layout.md | 4 + docs/manifest-format.md | 66 ++++++- git_superproject.py | 3 +- main.py | 43 ++++- manifest_xml.py | 454 ++++++++++++++++++++++++++++++++++++++++++--- project.py | 41 ++-- subcmds/abandon.py | 7 +- subcmds/branches.py | 14 +- subcmds/checkout.py | 2 +- subcmds/diff.py | 2 +- subcmds/diffmanifests.py | 3 + subcmds/download.py | 10 +- subcmds/forall.py | 12 +- subcmds/gitc_init.py | 1 + subcmds/grep.py | 15 +- subcmds/info.py | 16 +- subcmds/init.py | 12 ++ subcmds/list.py | 7 +- subcmds/manifest.py | 72 +++---- subcmds/overview.py | 4 +- subcmds/prune.py | 4 +- subcmds/rebase.py | 9 +- subcmds/stage.py | 11 +- subcmds/start.py | 5 +- subcmds/status.py | 9 +- subcmds/sync.py | 8 +- subcmds/upload.py | 44 +++-- 28 files changed, 797 insertions(+), 165 deletions(-) diff --git a/command.py b/command.py index b972a0be..12fe4172 100644 --- a/command.py +++ b/command.py @@ -61,13 +61,21 @@ class Command(object): # it is the number of parallel jobs to default to. PARALLEL_JOBS = None + # Whether this command supports Multi-manifest. If False, then main.py will + # iterate over the manifests and invoke the command once per (sub)manifest. + # This is only checked after calling ValidateOptions, so that partially + # migrated subcommands can set it to False. + MULTI_MANIFEST_SUPPORT = True + def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None, - git_event_log=None): + git_event_log=None, outer_client=None, outer_manifest=None): self.repodir = repodir self.client = client + self.outer_client = outer_client or client self.manifest = manifest self.gitc_manifest = gitc_manifest self.git_event_log = git_event_log + self.outer_manifest = outer_manifest # Cache for the OptionParser property. self._optparse = None @@ -135,6 +143,18 @@ class Command(object): type=int, default=self.PARALLEL_JOBS, help=f'number of jobs to run in parallel (default: {default})') + m = p.add_option_group('Multi-manifest options') + m.add_option('--outer-manifest', action='store_true', + help='operate starting at the outermost manifest') + m.add_option('--no-outer-manifest', dest='outer_manifest', + action='store_false', default=None, + help='do not operate on outer manifests') + m.add_option('--this-manifest-only', action='store_true', default=None, + help='only operate on this (sub)manifest') + m.add_option('--no-this-manifest-only', '--all-manifests', + dest='this_manifest_only', action='store_false', + help='operate on this manifest and its submanifests') + def _Options(self, p): """Initialize the option parser with subcommand-specific options.""" @@ -252,16 +272,19 @@ class Command(object): return project def GetProjects(self, args, manifest=None, groups='', missing_ok=False, - submodules_ok=False): + submodules_ok=False, all_manifests=False): """A list of projects that match the arguments. """ - if not manifest: - manifest = self.manifest - all_projects_list = manifest.projects + if all_manifests: + if not manifest: + manifest = self.manifest.outer_client + all_projects_list = manifest.all_projects + else: + if not manifest: + manifest = self.manifest + all_projects_list = manifest.projects result = [] - mp = manifest.manifestProject - if not groups: groups = manifest.GetGroupsStr() groups = [x for x in re.split(r'[,\s]+', groups) if x] @@ -282,12 +305,19 @@ class Command(object): for arg in args: # We have to filter by manifest groups in case the requested project is # checked out multiple times or differently based on them. - projects = [project for project in manifest.GetProjectsWithName(arg) + projects = [project for project in manifest.GetProjectsWithName( + arg, all_manifests=all_manifests) if project.MatchesGroups(groups)] if not projects: path = os.path.abspath(arg).replace('\\', '/') - project = self._GetProjectByPath(manifest, path) + tree = manifest + if all_manifests: + # Look for the deepest matching submanifest. + for tree in reversed(list(manifest.all_manifests)): + if path.startswith(tree.topdir): + break + project = self._GetProjectByPath(tree, path) # If it's not a derived project, update path->project mapping and # search again, as arg might actually point to a derived subproject. @@ -308,7 +338,8 @@ class Command(object): for project in projects: if not missing_ok and not project.Exists: - raise NoSuchProjectError('%s (%s)' % (arg, project.relpath)) + raise NoSuchProjectError('%s (%s)' % ( + arg, project.RelPath(local=not all_manifests))) if not project.MatchesGroups(groups): raise InvalidProjectGroupsError(arg) @@ -319,12 +350,22 @@ class Command(object): result.sort(key=_getpath) return result - def FindProjects(self, args, inverse=False): + def FindProjects(self, args, inverse=False, all_manifests=False): + """Find projects from command line arguments. + + Args: + args: a list of (case-insensitive) strings, projects to search for. + inverse: a boolean, if True, then projects not matching any |args| are + returned. + all_manifests: a boolean, if True then all manifests and submanifests are + used. If False, then only the local (sub)manifest is used. + """ result = [] patterns = [re.compile(r'%s' % a, re.IGNORECASE) for a in args] - for project in self.GetProjects(''): + for project in self.GetProjects('', all_manifests=all_manifests): + paths = [project.name, project.RelPath(local=not all_manifests)] for pattern in patterns: - match = pattern.search(project.name) or pattern.search(project.relpath) + match = any(pattern.search(x) for x in paths) if not inverse and match: result.append(project) break @@ -333,9 +374,24 @@ class Command(object): else: if inverse: result.append(project) - result.sort(key=lambda project: project.relpath) + result.sort(key=lambda project: (project.manifest.path_prefix, + project.relpath)) return result + def ManifestList(self, opt): + """Yields all of the manifests to traverse. + + Args: + opt: The command options. + """ + top = self.outer_manifest + if opt.outer_manifest is False or opt.this_manifest_only: + top = self.manifest + yield top + if not opt.this_manifest_only: + for child in top.all_children: + yield child + class InteractiveCommand(Command): """Command which requires user interaction on the tty and diff --git a/docs/internal-fs-layout.md b/docs/internal-fs-layout.md index 0e830510..a9bd1d26 100644 --- a/docs/internal-fs-layout.md +++ b/docs/internal-fs-layout.md @@ -50,6 +50,10 @@ For example, if you want to change the manifest branch, you can simply run For more documentation on the manifest format, including the local_manifests support, see the [manifest-format.md] file. +* `submanifests/{submanifest.path}/`: The path prefix to the manifest state of + a submanifest included in a multi-manifest checkout. The outermost manifest + manifest state is found adjacent to `submanifests/`. + * `manifests/`: A git checkout of the manifest project. Its `.git/` state points to the `manifest.git` bare checkout (see below). It tracks the git branch specified at `repo init` time via `--manifest-branch`. diff --git a/docs/manifest-format.md b/docs/manifest-format.md index 8e0049b3..7c0a7da9 100644 --- a/docs/manifest-format.md +++ b/docs/manifest-format.md @@ -26,6 +26,7 @@ following DTD: remote*, default?, manifest-server?, + submanifest*?, remove-project*, project*, extend-project*, @@ -57,6 +58,15 @@ following DTD: + + + + + + + + + element specified in the manifest. + + Attributes: + name: a string, the name for this submanifest. + remote: a string, the remote.name for this submanifest. + project: a string, the name of the manifest project. + revision: a string, the commitish. + manifestName: a string, the submanifest file name. + groups: a list of strings, the groups to add to all projects in the submanifest. + path: a string, the relative path for the submanifest checkout. + annotations: (derived) a list of annotations. + present: (derived) a boolean, whether the submanifest's manifest file is present. + """ + def __init__(self, + name, + remote=None, + project=None, + revision=None, + manifestName=None, + groups=None, + path=None, + parent=None): + self.name = name + self.remote = remote + self.project = project + self.revision = revision + self.manifestName = manifestName + self.groups = groups + self.path = path + self.annotations = [] + outer_client = parent._outer_client or parent + if self.remote and not self.project: + raise ManifestParseError( + f'Submanifest {name}: must specify project when remote is given.') + rc = self.repo_client = RepoClient( + parent.repodir, manifestName, parent_groups=','.join(groups) or '', + submanifest_path=self.relpath, outer_client=outer_client) + + self.present = os.path.exists(os.path.join(self.repo_client.subdir, + MANIFEST_FILE_NAME)) + + def __eq__(self, other): + if not isinstance(other, _XmlSubmanifest): + return False + return ( + self.name == other.name and + self.remote == other.remote and + self.project == other.project and + self.revision == other.revision and + self.manifestName == other.manifestName and + self.groups == other.groups and + self.path == other.path and + sorted(self.annotations) == sorted(other.annotations)) + + def __ne__(self, other): + return not self.__eq__(other) + + def ToSubmanifestSpec(self, root): + """Return a SubmanifestSpec object, populating attributes""" + mp = root.manifestProject + remote = root.remotes[self.remote or root.default.remote.name] + # If a project was given, generate the url from the remote and project. + # If not, use this manifestProject's url. + if self.project: + manifestUrl = remote.ToRemoteSpec(self.project).url + else: + manifestUrl = mp.GetRemote(mp.remote.name).url + manifestName = self.manifestName or 'default.xml' + revision = self.revision or self.name + path = self.path or revision.split('/')[-1] + groups = self.groups or [] + + return SubmanifestSpec(self.name, manifestUrl, manifestName, revision, path, + groups) + + @property + def relpath(self): + """The path of this submanifest relative to the parent manifest.""" + revision = self.revision or self.name + return self.path or revision.split('/')[-1] + + def GetGroupsStr(self): + """Returns the `groups` given for this submanifest.""" + if self.groups: + return ','.join(self.groups) + return '' + + def AddAnnotation(self, name, value, keep): + """Add annotations to the submanifest.""" + self.annotations.append(Annotation(name, value, keep)) + + +class SubmanifestSpec: + """The submanifest element, with all fields expanded.""" + + def __init__(self, + name, + manifestUrl, + manifestName, + revision, + path, + groups): + self.name = name + self.manifestUrl = manifestUrl + self.manifestName = manifestName + self.revision = revision + self.path = path + self.groups = groups or [] + + class XmlManifest(object): """manages the repo configuration file""" - def __init__(self, repodir, manifest_file, local_manifests=None): + def __init__(self, repodir, manifest_file, local_manifests=None, + outer_client=None, parent_groups='', submanifest_path=''): """Initialize. Args: @@ -210,23 +325,37 @@ class XmlManifest(object): be |repodir|/|MANIFEST_FILE_NAME|. local_manifests: Full path to the directory of local override manifests. This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|. + outer_client: RepoClient of the outertree. + parent_groups: a string, the groups to apply to this projects. + submanifest_path: The submanifest root relative to the repo root. """ # TODO(vapier): Move this out of this class. self.globalConfig = GitConfig.ForUser() self.repodir = os.path.abspath(repodir) - self.topdir = os.path.dirname(self.repodir) + self._CheckLocalPath(submanifest_path) + self.topdir = os.path.join(os.path.dirname(self.repodir), submanifest_path) self.manifestFile = manifest_file self.local_manifests = local_manifests self._load_local_manifests = True + self.parent_groups = parent_groups + + if outer_client and self.isGitcClient: + raise ManifestParseError('Multi-manifest is incompatible with `gitc-init`') + + if submanifest_path and not outer_client: + # If passing a submanifest_path, there must be an outer_client. + raise ManifestParseError(f'Bad call to {self.__class__.__name__}') + + # If self._outer_client is None, this is not a checkout that supports + # multi-tree. + self._outer_client = outer_client or self self.repoProject = MetaProject(self, 'repo', gitdir=os.path.join(repodir, 'repo/.git'), worktree=os.path.join(repodir, 'repo')) - mp = MetaProject(self, 'manifests', - gitdir=os.path.join(repodir, 'manifests.git'), - worktree=os.path.join(repodir, 'manifests')) + mp = self.SubmanifestProject(self.path_prefix) self.manifestProject = mp # This is a bit hacky, but we're in a chicken & egg situation: all the @@ -311,6 +440,31 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md ae.setAttribute('value', a.value) e.appendChild(ae) + def _SubmanifestToXml(self, r, doc, root): + """Generate XML node.""" + e = doc.createElement('submanifest') + root.appendChild(e) + e.setAttribute('name', r.name) + if r.remote is not None: + e.setAttribute('remote', r.remote) + if r.project is not None: + e.setAttribute('project', r.project) + if r.manifestName is not None: + e.setAttribute('manifest-name', r.manifestName) + if r.revision is not None: + e.setAttribute('revision', r.revision) + if r.path is not None: + e.setAttribute('path', r.path) + if r.groups: + e.setAttribute('groups', r.GetGroupsStr()) + + for a in r.annotations: + if a.keep == 'true': + ae = doc.createElement('annotation') + ae.setAttribute('name', a.name) + ae.setAttribute('value', a.value) + e.appendChild(ae) + def _ParseList(self, field): """Parse fields that contain flattened lists. @@ -329,6 +483,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md doc = xml.dom.minidom.Document() root = doc.createElement('manifest') + if self.is_submanifest: + root.setAttribute('path', self.path_prefix) doc.appendChild(root) # Save out the notice. There's a little bit of work here to give it the @@ -383,6 +539,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md root.appendChild(e) root.appendChild(doc.createTextNode('')) + for r in sorted(self.submanifests): + self._SubmanifestToXml(self.submanifests[r], doc, root) + if self.submanifests: + root.appendChild(doc.createTextNode('')) + def output_projects(parent, parent_node, projects): for project_name in projects: for project in self._projects[project_name]: @@ -537,6 +698,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md 'project', 'extend-project', 'include', + 'submanifest', # These are children of 'project' nodes. 'annotation', 'project', @@ -574,13 +736,75 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md def _output_manifest_project_extras(self, p, e): """Manifests can modify e if they support extra project attributes.""" + @property + def is_multimanifest(self): + """Whether this is a multimanifest checkout""" + return bool(self.outer_client.submanifests) + + @property + def is_submanifest(self): + """Whether this manifest is a submanifest""" + return self._outer_client and self._outer_client != self + + @property + def outer_client(self): + """The instance of the outermost manifest client""" + self._Load() + return self._outer_client + + @property + def all_manifests(self): + """Generator yielding all (sub)manifests.""" + self._Load() + outer = self._outer_client + yield outer + for tree in outer.all_children: + yield tree + + @property + def all_children(self): + """Generator yielding all child submanifests.""" + self._Load() + for child in self._submanifests.values(): + if child.repo_client: + yield child.repo_client + for tree in child.repo_client.all_children: + yield tree + + @property + def path_prefix(self): + """The path of this submanifest, relative to the outermost manifest.""" + if not self._outer_client or self == self._outer_client: + return '' + return os.path.relpath(self.topdir, self._outer_client.topdir) + + @property + def all_paths(self): + """All project paths for all (sub)manifests. See `paths`.""" + ret = {} + for tree in self.all_manifests: + prefix = tree.path_prefix + ret.update({os.path.join(prefix, k): v for k, v in tree.paths.items()}) + return ret + + @property + def all_projects(self): + """All projects for all (sub)manifests. See `projects`.""" + return list(itertools.chain.from_iterable(x._paths.values() for x in self.all_manifests)) + @property def paths(self): + """Return all paths for this manifest. + + Return: + A dictionary of {path: Project()}. `path` is relative to this manifest. + """ self._Load() return self._paths @property def projects(self): + """Return a list of all Projects in this manifest.""" self._Load() return list(self._paths.values()) @@ -594,6 +818,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._Load() return self._default + @property + def submanifests(self): + """All submanifests in this manifest.""" + self._Load() + return self._submanifests + @property def repo_hooks_project(self): self._Load() @@ -651,8 +881,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md return self._load_local_manifests and self.local_manifests def IsFromLocalManifest(self, project): - """Is the project from a local manifest? - """ + """Is the project from a local manifest?""" return any(x.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for x in project.groups) @@ -676,6 +905,50 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md def EnableGitLfs(self): return self.manifestProject.config.GetBoolean('repo.git-lfs') + def FindManifestByPath(self, path): + """Returns the manifest containing path.""" + path = os.path.abspath(path) + manifest = self._outer_client or self + old = None + while manifest._submanifests and manifest != old: + old = manifest + for name in manifest._submanifests: + tree = manifest._submanifests[name] + if path.startswith(tree.repo_client.manifest.topdir): + manifest = tree.repo_client + break + return manifest + + @property + def subdir(self): + """Returns the path for per-submanifest objects for this manifest.""" + return self.SubmanifestInfoDir(self.path_prefix) + + def SubmanifestInfoDir(self, submanifest_path, object_path=''): + """Return the path to submanifest-specific info for a submanifest. + + Return the full path of the directory in which to put per-manifest objects. + + Args: + submanifest_path: a string, the path of the submanifest, relative to the + outermost topdir. If empty, then repodir is returned. + object_path: a string, relative path to append to the submanifest info + directory path. + """ + if submanifest_path: + return os.path.join(self.repodir, SUBMANIFEST_DIR, submanifest_path, + object_path) + else: + return os.path.join(self.repodir, object_path) + + def SubmanifestProject(self, submanifest_path): + """Return a manifestProject for a submanifest.""" + subdir = self.SubmanifestInfoDir(submanifest_path) + mp = MetaProject(self, 'manifests', + gitdir=os.path.join(subdir, 'manifests.git'), + worktree=os.path.join(subdir, 'manifests')) + return mp + def GetDefaultGroupsStr(self): """Returns the default group string for the platform.""" return 'default,platform-' + platform.system().lower() @@ -693,6 +966,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._paths = {} self._remotes = {} self._default = None + self._submanifests = {} self._repo_hooks_project = None self._superproject = {} self._contactinfo = ContactInfo(Wrapper().BUG_URL) @@ -700,20 +974,29 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self.branch = None self._manifest_server = None - def _Load(self): + def _Load(self, initial_client=None, submanifest_depth=0): + if submanifest_depth > MAX_SUBMANIFEST_DEPTH: + raise ManifestParseError('maximum submanifest depth %d exceeded.' % + MAX_SUBMANIFEST_DEPTH) if not self._loaded: + if self._outer_client and self._outer_client != self: + # This will load all clients. + self._outer_client._Load(initial_client=self) + m = self.manifestProject b = m.GetBranch(m.CurrentBranch).merge if b is not None and b.startswith(R_HEADS): b = b[len(R_HEADS):] self.branch = b + parent_groups = self.parent_groups + # The manifestFile was specified by the user which is why we allow include # paths to point anywhere. nodes = [] nodes.append(self._ParseManifestXml( self.manifestFile, self.manifestProject.worktree, - restrict_includes=False)) + parent_groups=parent_groups, restrict_includes=False)) if self._load_local_manifests and self.local_manifests: try: @@ -722,9 +1005,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md local = os.path.join(self.local_manifests, local_file) # Since local manifests are entirely managed by the user, allow # them to point anywhere the user wants. + local_group = f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}' nodes.append(self._ParseManifestXml( - local, self.repodir, - parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}', + local, self.subdir, + parent_groups=f'{local_group},{parent_groups}', restrict_includes=False)) except OSError: pass @@ -743,6 +1027,23 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md self._loaded = True + # Now that we have loaded this manifest, load any submanifest manifests + # as well. We need to do this after self._loaded is set to avoid looping. + if self._outer_client: + for name in self._submanifests: + tree = self._submanifests[name] + spec = tree.ToSubmanifestSpec(self) + present = os.path.exists(os.path.join(self.subdir, MANIFEST_FILE_NAME)) + if present and tree.present and not tree.repo_client: + if initial_client and initial_client.topdir == self.topdir: + tree.repo_client = self + tree.present = present + elif not os.path.exists(self.subdir): + tree.present = False + if tree.present: + tree.repo_client._Load(initial_client=initial_client, + submanifest_depth=submanifest_depth + 1) + def _ParseManifestXml(self, path, include_root, parent_groups='', restrict_includes=True): """Parse a manifest XML and return the computed nodes. @@ -832,6 +1133,20 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if self._default is None: self._default = _Default() + submanifest_paths = set() + for node in itertools.chain(*node_list): + if node.nodeName == 'submanifest': + submanifest = self._ParseSubmanifest(node) + if submanifest: + if submanifest.name in self._submanifests: + if submanifest != self._submanifests[submanifest.name]: + raise ManifestParseError( + 'submanifest %s already exists with different attributes' % + (submanifest.name)) + else: + self._submanifests[submanifest.name] = submanifest + submanifest_paths.add(submanifest.relpath) + for node in itertools.chain(*node_list): if node.nodeName == 'notice': if self._notice is not None: @@ -859,6 +1174,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md raise ManifestParseError( 'duplicate path %s in %s' % (project.relpath, self.manifestFile)) + for tree in submanifest_paths: + if project.relpath.startswith(tree): + raise ManifestParseError( + 'project %s conflicts with submanifest path %s' % + (project.relpath, tree)) self._paths[project.relpath] = project projects.append(project) for subproject in project.subprojects: @@ -883,8 +1203,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if groups: groups = self._ParseList(groups) revision = node.getAttribute('revision') - remote = node.getAttribute('remote') - if remote: + remote_name = node.getAttribute('remote') + if not remote_name: + remote = self._default.remote + else: remote = self._get_remote(node) named_projects = self._projects[name] @@ -899,12 +1221,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if revision: p.SetRevision(revision) - if remote: + if remote_name: p.remote = remote.ToRemoteSpec(name) if dest_path: del self._paths[p.relpath] - relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(name, dest_path) + relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths( + name, dest_path, remote.name) p.UpdatePaths(relpath, worktree, gitdir, objdir) self._paths[p.relpath] = p @@ -1109,6 +1432,53 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md return '\n'.join(cleanLines) + def _ParseSubmanifest(self, node): + """Reads a element from the manifest file.""" + name = self._reqatt(node, 'name') + remote = node.getAttribute('remote') + if remote == '': + remote = None + project = node.getAttribute('project') + if project == '': + project = None + revision = node.getAttribute('revision') + if revision == '': + revision = None + manifestName = node.getAttribute('manifest-name') + if manifestName == '': + manifestName = None + groups = '' + if node.hasAttribute('groups'): + groups = node.getAttribute('groups') + groups = self._ParseList(groups) + path = node.getAttribute('path') + if path == '': + path = None + if revision: + msg = self._CheckLocalPath(revision.split('/')[-1]) + if msg: + raise ManifestInvalidPathError( + ' invalid "revision": %s: %s' % (revision, msg)) + else: + msg = self._CheckLocalPath(name) + if msg: + raise ManifestInvalidPathError( + ' invalid "name": %s: %s' % (name, msg)) + else: + msg = self._CheckLocalPath(path) + if msg: + raise ManifestInvalidPathError( + ' invalid "path": %s: %s' % (path, msg)) + + submanifest = _XmlSubmanifest(name, remote, project, revision, manifestName, + groups, path, self) + + for n in node.childNodes: + if n.nodeName == 'annotation': + self._ParseAnnotation(submanifest, n) + + return submanifest + def _JoinName(self, parent_name, name): return os.path.join(parent_name, name) @@ -1172,7 +1542,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md if parent is None: relpath, worktree, gitdir, objdir, use_git_worktrees = \ - self.GetProjectPaths(name, path) + self.GetProjectPaths(name, path, remote.name) else: use_git_worktrees = False relpath, worktree, gitdir, objdir = \ @@ -1218,31 +1588,54 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md return project - def GetProjectPaths(self, name, path): + def GetProjectPaths(self, name, path, remote): + """Return the paths for a project. + + Args: + name: a string, the name of the project. + path: a string, the path of the project. + remote: a string, the remote.name of the project. + """ # The manifest entries might have trailing slashes. Normalize them to avoid # unexpected filesystem behavior since we do string concatenation below. path = path.rstrip('/') name = name.rstrip('/') + remote = remote.rstrip('/') use_git_worktrees = False + use_remote_name = bool(self._outer_client._submanifests) relpath = path if self.IsMirror: worktree = None gitdir = os.path.join(self.topdir, '%s.git' % name) objdir = gitdir else: + if use_remote_name: + namepath = os.path.join(remote, f'{name}.git') + else: + namepath = f'{name}.git' worktree = os.path.join(self.topdir, path).replace('\\', '/') - gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path) + gitdir = os.path.join(self.subdir, 'projects', '%s.git' % path) # We allow people to mix git worktrees & non-git worktrees for now. # This allows for in situ migration of repo clients. if os.path.exists(gitdir) or not self.UseGitWorktrees: - objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name) + objdir = os.path.join(self.subdir, 'project-objects', namepath) else: use_git_worktrees = True - gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name) + gitdir = os.path.join(self.repodir, 'worktrees', namepath) objdir = gitdir return relpath, worktree, gitdir, objdir, use_git_worktrees - def GetProjectsWithName(self, name): + def GetProjectsWithName(self, name, all_manifests=False): + """All projects with |name|. + + Args: + name: a string, the name of the project. + all_manifests: a boolean, if True, then all manifests are searched. If + False, then only this manifest is searched. + """ + if all_manifests: + return list(itertools.chain.from_iterable( + x._projects.get(name, []) for x in self.all_manifests)) return self._projects.get(name, []) def GetSubprojectName(self, parent, submodule_path): @@ -1498,19 +1891,26 @@ class GitcManifest(XmlManifest): class RepoClient(XmlManifest): """Manages a repo client checkout.""" - def __init__(self, repodir, manifest_file=None): + def __init__(self, repodir, manifest_file=None, submanifest_path='', **kwargs): self.isGitcClient = False + submanifest_path = submanifest_path or '' + if submanifest_path: + self._CheckLocalPath(submanifest_path) + prefix = os.path.join(repodir, SUBMANIFEST_DIR, submanifest_path) + else: + prefix = repodir - if os.path.exists(os.path.join(repodir, LOCAL_MANIFEST_NAME)): + if os.path.exists(os.path.join(prefix, LOCAL_MANIFEST_NAME)): print('error: %s is not supported; put local manifests in `%s` instead' % - (LOCAL_MANIFEST_NAME, os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)), + (LOCAL_MANIFEST_NAME, os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)), file=sys.stderr) sys.exit(1) if manifest_file is None: - manifest_file = os.path.join(repodir, MANIFEST_FILE_NAME) - local_manifests = os.path.abspath(os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)) - super().__init__(repodir, manifest_file, local_manifests) + manifest_file = os.path.join(prefix, MANIFEST_FILE_NAME) + local_manifests = os.path.abspath(os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)) + super().__init__(repodir, manifest_file, local_manifests, + submanifest_path=submanifest_path, **kwargs) # TODO: Completely separate manifest logic out of the client. self.manifest = self diff --git a/project.py b/project.py index ff91d189..480dac63 100644 --- a/project.py +++ b/project.py @@ -546,6 +546,18 @@ class Project(object): # project containing repo hooks. self.enabled_repo_hooks = [] + def RelPath(self, local=True): + """Return the path for the project relative to a manifest. + + Args: + local: a boolean, if True, the path is relative to the local + (sub)manifest. If false, the path is relative to the + outermost manifest. + """ + if local: + return self.relpath + return os.path.join(self.manifest.path_prefix, self.relpath) + def SetRevision(self, revisionExpr, revisionId=None): """Set revisionId based on revision expression and id""" self.revisionExpr = revisionExpr @@ -2503,22 +2515,21 @@ class Project(object): mp = self.manifest.manifestProject ref_dir = mp.config.GetString('repo.reference') or '' + def _expanded_ref_dirs(): + """Iterate through the possible git reference directory paths.""" + name = self.name + '.git' + yield mirror_git or os.path.join(ref_dir, name) + for prefix in '', self.remote.name: + yield os.path.join(ref_dir, '.repo', 'project-objects', prefix, name) + yield os.path.join(ref_dir, '.repo', 'worktrees', prefix, name) + if ref_dir or mirror_git: - if not mirror_git: - mirror_git = os.path.join(ref_dir, self.name + '.git') - repo_git = os.path.join(ref_dir, '.repo', 'project-objects', - self.name + '.git') - worktrees_git = os.path.join(ref_dir, '.repo', 'worktrees', - self.name + '.git') - - if os.path.exists(mirror_git): - ref_dir = mirror_git - elif os.path.exists(repo_git): - ref_dir = repo_git - elif os.path.exists(worktrees_git): - ref_dir = worktrees_git - else: - ref_dir = None + found_ref_dir = None + for path in _expanded_ref_dirs(): + if os.path.exists(path): + found_ref_dir = path + break + ref_dir = found_ref_dir if ref_dir: if not os.path.isabs(ref_dir): diff --git a/subcmds/abandon.py b/subcmds/abandon.py index 85d85f5a..c3d2d5b7 100644 --- a/subcmds/abandon.py +++ b/subcmds/abandon.py @@ -69,7 +69,8 @@ It is equivalent to "git branch -D ". nb = args[0] err = defaultdict(list) success = defaultdict(list) - all_projects = self.GetProjects(args[1:]) + all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only) + _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) def _ProcessResults(_pool, pm, states): for (results, project) in states: @@ -94,7 +95,7 @@ It is equivalent to "git branch -D ". err_msg = "error: cannot abandon %s" % br print(err_msg, file=sys.stderr) for proj in err[br]: - print(' ' * len(err_msg) + " | %s" % proj.relpath, file=sys.stderr) + print(' ' * len(err_msg) + " | %s" % _RelPath(proj), file=sys.stderr) sys.exit(1) elif not success: print('error: no project has local branch(es) : %s' % nb, @@ -110,5 +111,5 @@ It is equivalent to "git branch -D ". result = "all project" else: result = "%s" % ( - ('\n' + ' ' * width + '| ').join(p.relpath for p in success[br])) + ('\n' + ' ' * width + '| ').join(_RelPath(p) for p in success[br])) print("%s%s| %s\n" % (br, ' ' * (width - len(br)), result)) diff --git a/subcmds/branches.py b/subcmds/branches.py index 7b5decc6..b89cc2f8 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py @@ -98,7 +98,7 @@ is shown, then the branch appears in all projects. PARALLEL_JOBS = DEFAULT_LOCAL_JOBS def Execute(self, opt, args): - projects = self.GetProjects(args) + projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) out = BranchColoring(self.manifest.manifestProject.config) all_branches = {} project_cnt = len(projects) @@ -147,6 +147,7 @@ is shown, then the branch appears in all projects. hdr('%c%c %-*s' % (current, published, width, name)) out.write(' |') + _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) if in_cnt < project_cnt: fmt = out.write paths = [] @@ -154,19 +155,20 @@ is shown, then the branch appears in all projects. if i.IsSplitCurrent or (in_cnt <= project_cnt - in_cnt): in_type = 'in' for b in i.projects: + relpath = b.project.relpath if not i.IsSplitCurrent or b.current: - paths.append(b.project.relpath) + paths.append(_RelPath(b.project)) else: - non_cur_paths.append(b.project.relpath) + non_cur_paths.append(_RelPath(b.project)) else: fmt = out.notinproject in_type = 'not in' have = set() for b in i.projects: - have.add(b.project.relpath) + have.add(_RelPath(b.project)) for p in projects: - if p.relpath not in have: - paths.append(p.relpath) + if _RelPath(p) not in have: + paths.append(_RelPath(p)) s = ' %s %s' % (in_type, ', '.join(paths)) if not i.IsSplitCurrent and (width + 7 + len(s) < 80): diff --git a/subcmds/checkout.py b/subcmds/checkout.py index 9b429489..768b6027 100644 --- a/subcmds/checkout.py +++ b/subcmds/checkout.py @@ -47,7 +47,7 @@ The command is equivalent to: nb = args[0] err = [] success = [] - all_projects = self.GetProjects(args[1:]) + all_projects = self.GetProjects(args[1:], all_manifests=not opt.this_manifest_only) def _ProcessResults(_pool, pm, results): for status, project in results: diff --git a/subcmds/diff.py b/subcmds/diff.py index 00a7ec29..a1f4ba88 100644 --- a/subcmds/diff.py +++ b/subcmds/diff.py @@ -50,7 +50,7 @@ to the Unix 'patch' command. return (ret, buf.getvalue()) def Execute(self, opt, args): - all_projects = self.GetProjects(args) + all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) def _ProcessResults(_pool, _output, results): ret = 0 diff --git a/subcmds/diffmanifests.py b/subcmds/diffmanifests.py index f6cc30a2..0e5f4108 100644 --- a/subcmds/diffmanifests.py +++ b/subcmds/diffmanifests.py @@ -179,6 +179,9 @@ synced and their revisions won't be found. def ValidateOptions(self, opt, args): if not args or len(args) > 2: self.OptionParser.error('missing manifests to diff') + if opt.this_manifest_only is False: + raise self.OptionParser.error( + '`diffmanifest` only supports the current tree') def Execute(self, opt, args): self.out = _Coloring(self.client.globalConfig) diff --git a/subcmds/download.py b/subcmds/download.py index 523f25e0..15824843 100644 --- a/subcmds/download.py +++ b/subcmds/download.py @@ -48,7 +48,7 @@ If no project is specified try to use current directory as a project. dest='ffonly', action='store_true', help="force fast-forward merge") - def _ParseChangeIds(self, args): + def _ParseChangeIds(self, opt, args): if not args: self.Usage() @@ -77,7 +77,7 @@ If no project is specified try to use current directory as a project. ps_id = max(int(match.group(1)), ps_id) to_get.append((project, chg_id, ps_id)) else: - projects = self.GetProjects([a]) + projects = self.GetProjects([a], all_manifests=not opt.this_manifest_only) if len(projects) > 1: # If the cwd is one of the projects, assume they want that. try: @@ -88,8 +88,8 @@ If no project is specified try to use current directory as a project. print('error: %s matches too many projects; please re-run inside ' 'the project checkout.' % (a,), file=sys.stderr) for project in projects: - print(' %s/ @ %s' % (project.relpath, project.revisionExpr), - file=sys.stderr) + print(' %s/ @ %s' % (project.RelPath(local=opt.this_manifest_only), + project.revisionExpr), file=sys.stderr) sys.exit(1) else: project = projects[0] @@ -105,7 +105,7 @@ If no project is specified try to use current directory as a project. self.OptionParser.error('-x and --ff are mutually exclusive options') def Execute(self, opt, args): - for project, change_id, ps_id in self._ParseChangeIds(args): + for project, change_id, ps_id in self._ParseChangeIds(opt, args): dl = project.DownloadPatchSet(change_id, ps_id) if not dl: print('[%s] change %d/%d not found' diff --git a/subcmds/forall.py b/subcmds/forall.py index 7c1dea9e..cc578b52 100644 --- a/subcmds/forall.py +++ b/subcmds/forall.py @@ -168,6 +168,7 @@ without iterating through the remaining projects. def Execute(self, opt, args): cmd = [opt.command[0]] + all_trees = not opt.this_manifest_only shell = True if re.compile(r'^[a-z0-9A-Z_/\.-]+$').match(cmd[0]): @@ -213,11 +214,11 @@ without iterating through the remaining projects. self.manifest.Override(smart_sync_manifest_path) if opt.regex: - projects = self.FindProjects(args) + projects = self.FindProjects(args, all_manifests=all_trees) elif opt.inverse_regex: - projects = self.FindProjects(args, inverse=True) + projects = self.FindProjects(args, inverse=True, all_manifests=all_trees) else: - projects = self.GetProjects(args, groups=opt.groups) + projects = self.GetProjects(args, groups=opt.groups, all_manifests=all_trees) os.environ['REPO_COUNT'] = str(len(projects)) @@ -290,6 +291,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config): setenv('REPO_PROJECT', project.name) setenv('REPO_PATH', project.relpath) + setenv('REPO_OUTERPATH', project.RelPath(local=opt.this_manifest_only)) setenv('REPO_REMOTE', project.remote.name) try: # If we aren't in a fully synced state and we don't have the ref the manifest @@ -320,7 +322,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config): output = '' if ((opt.project_header and opt.verbose) or not opt.project_header): - output = 'skipping %s/' % project.relpath + output = 'skipping %s/' % project.RelPath(local=opt.this_manifest_only) return (1, output) if opt.verbose: @@ -344,7 +346,7 @@ def DoWork(project, mirror, opt, cmd, shell, cnt, config): if mirror: project_header_path = project.name else: - project_header_path = project.relpath + project_header_path = project.RelPath(local=opt.this_manifest_only) out.project('project %s/' % project_header_path) out.nl() buf.write(output) diff --git a/subcmds/gitc_init.py b/subcmds/gitc_init.py index e705b613..1d81baf5 100644 --- a/subcmds/gitc_init.py +++ b/subcmds/gitc_init.py @@ -24,6 +24,7 @@ import wrapper class GitcInit(init.Init, GitcAvailableCommand): COMMON = True + MULTI_MANIFEST_SUPPORT = False helpSummary = "Initialize a GITC Client." helpUsage = """ %prog [options] [client name] diff --git a/subcmds/grep.py b/subcmds/grep.py index 8ac4ba14..93c9ae51 100644 --- a/subcmds/grep.py +++ b/subcmds/grep.py @@ -172,15 +172,16 @@ contain a line that matches both expressions: return (project, p.Wait(), p.stdout, p.stderr) @staticmethod - def _ProcessResults(full_name, have_rev, _pool, out, results): + def _ProcessResults(full_name, have_rev, opt, _pool, out, results): git_failed = False bad_rev = False have_match = False + _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) for project, rc, stdout, stderr in results: if rc < 0: git_failed = True - out.project('--- project %s ---' % project.relpath) + out.project('--- project %s ---' % _RelPath(project)) out.nl() out.fail('%s', stderr) out.nl() @@ -192,7 +193,7 @@ contain a line that matches both expressions: if have_rev and 'fatal: ambiguous argument' in stderr: bad_rev = True else: - out.project('--- project %s ---' % project.relpath) + out.project('--- project %s ---' % _RelPath(project)) out.nl() out.fail('%s', stderr.strip()) out.nl() @@ -208,13 +209,13 @@ contain a line that matches both expressions: rev, line = line.split(':', 1) out.write("%s", rev) out.write(':') - out.project(project.relpath) + out.project(_RelPath(project)) out.write('/') out.write("%s", line) out.nl() elif full_name: for line in r: - out.project(project.relpath) + out.project(_RelPath(project)) out.write('/') out.write("%s", line) out.nl() @@ -239,7 +240,7 @@ contain a line that matches both expressions: cmd_argv.append(args[0]) args = args[1:] - projects = self.GetProjects(args) + projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) full_name = False if len(projects) > 1: @@ -259,7 +260,7 @@ contain a line that matches both expressions: opt.jobs, functools.partial(self._ExecuteOne, cmd_argv), projects, - callback=functools.partial(self._ProcessResults, full_name, have_rev), + callback=functools.partial(self._ProcessResults, full_name, have_rev, opt), output=out, ordered=True) diff --git a/subcmds/info.py b/subcmds/info.py index 6c1246ef..4bedf9d5 100644 --- a/subcmds/info.py +++ b/subcmds/info.py @@ -61,6 +61,8 @@ class Info(PagedCommand): self.opt = opt + if not opt.this_manifest_only: + self.manifest = self.manifest.outer_client manifestConfig = self.manifest.manifestProject.config mergeBranch = manifestConfig.GetBranch("default").merge manifestGroups = (manifestConfig.GetString('manifest.groups') @@ -80,17 +82,17 @@ class Info(PagedCommand): self.printSeparator() if not opt.overview: - self.printDiffInfo(args) + self._printDiffInfo(opt, args) else: - self.printCommitOverview(args) + self._printCommitOverview(opt, args) def printSeparator(self): self.text("----------------------------") self.out.nl() - def printDiffInfo(self, args): + def _printDiffInfo(self, opt, args): # We let exceptions bubble up to main as they'll be well structured. - projs = self.GetProjects(args) + projs = self.GetProjects(args, all_manifests=not opt.this_manifest_only) for p in projs: self.heading("Project: ") @@ -179,9 +181,9 @@ class Info(PagedCommand): self.text(" ".join(split[1:])) self.out.nl() - def printCommitOverview(self, args): + def _printCommitOverview(self, opt, args): all_branches = [] - for project in self.GetProjects(args): + for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only): br = [project.GetUploadableBranch(x) for x in project.GetBranches()] br = [x for x in br if x] @@ -200,7 +202,7 @@ class Info(PagedCommand): if project != branch.project: project = branch.project self.out.nl() - self.headtext(project.relpath) + self.headtext(project.RelPath(local=opt.this_manifest_only)) self.out.nl() commits = branch.commits diff --git a/subcmds/init.py b/subcmds/init.py index 32c85f79..b9775a34 100644 --- a/subcmds/init.py +++ b/subcmds/init.py @@ -32,6 +32,7 @@ from wrapper import Wrapper class Init(InteractiveCommand, MirrorSafeCommand): COMMON = True + MULTI_MANIFEST_SUPPORT = False helpSummary = "Initialize a repo client checkout in the current directory" helpUsage = """ %prog [options] [manifest url] @@ -90,6 +91,17 @@ to update the working directory files. def _Options(self, p, gitc_init=False): Wrapper().InitParser(p, gitc_init=gitc_init) + m = p.add_option_group('Multi-manifest') + m.add_option('--outer-manifest', action='store_true', + help='operate starting at the outermost manifest') + m.add_option('--no-outer-manifest', dest='outer_manifest', + action='store_false', default=None, + help='do not operate on outer manifests') + m.add_option('--this-manifest-only', action='store_true', default=None, + help='only operate on this (sub)manifest') + m.add_option('--no-this-manifest-only', '--all-manifests', + dest='this_manifest_only', action='store_false', + help='operate on this manifest and its submanifests') def _RegisteredEnvironmentOptions(self): return {'REPO_MANIFEST_URL': 'manifest_url', diff --git a/subcmds/list.py b/subcmds/list.py index 6adf85b7..ad8036ee 100644 --- a/subcmds/list.py +++ b/subcmds/list.py @@ -77,16 +77,17 @@ This is similar to running: repo forall -c 'echo "$REPO_PATH : $REPO_PROJECT"'. args: Positional args. Can be a list of projects to list, or empty. """ if not opt.regex: - projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all) + projects = self.GetProjects(args, groups=opt.groups, missing_ok=opt.all, + all_manifests=not opt.this_manifest_only) else: - projects = self.FindProjects(args) + projects = self.FindProjects(args, all_manifests=not opt.this_manifest_only) def _getpath(x): if opt.fullpath: return x.worktree if opt.relative_to: return os.path.relpath(x.worktree, opt.relative_to) - return x.relpath + return x.RelPath(local=opt.this_manifest_only) lines = [] for project in projects: diff --git a/subcmds/manifest.py b/subcmds/manifest.py index 0fbdeac0..08905cb4 100644 --- a/subcmds/manifest.py +++ b/subcmds/manifest.py @@ -15,6 +15,7 @@ import json import os import sys +import optparse from command import PagedCommand @@ -75,7 +76,7 @@ to indicate the remote ref to push changes to via 'repo upload'. p.add_option('-o', '--output-file', dest='output_file', default='-', - help='file to save the manifest to', + help='file to save the manifest to. (Filename prefix for multi-tree.)', metavar='-|NAME.xml') def _Output(self, opt): @@ -83,36 +84,45 @@ to indicate the remote ref to push changes to via 'repo upload'. if opt.manifest_name: self.manifest.Override(opt.manifest_name, False) - if opt.output_file == '-': - fd = sys.stdout - else: - fd = open(opt.output_file, 'w') - - self.manifest.SetUseLocalManifests(not opt.ignore_local_manifests) - - if opt.json: - print('warning: --json is experimental!', file=sys.stderr) - doc = self.manifest.ToDict(peg_rev=opt.peg_rev, - peg_rev_upstream=opt.peg_rev_upstream, - peg_rev_dest_branch=opt.peg_rev_dest_branch) - - json_settings = { - # JSON style guide says Uunicode characters are fully allowed. - 'ensure_ascii': False, - # We use 2 space indent to match JSON style guide. - 'indent': 2 if opt.pretty else None, - 'separators': (',', ': ') if opt.pretty else (',', ':'), - 'sort_keys': True, - } - fd.write(json.dumps(doc, **json_settings)) - else: - self.manifest.Save(fd, - peg_rev=opt.peg_rev, - peg_rev_upstream=opt.peg_rev_upstream, - peg_rev_dest_branch=opt.peg_rev_dest_branch) - fd.close() - if opt.output_file != '-': - print('Saved manifest to %s' % opt.output_file, file=sys.stderr) + for manifest in self.ManifestList(opt): + output_file = opt.output_file + if output_file == '-': + fd = sys.stdout + else: + if manifest.path_prefix: + output_file = f'{opt.output_file}:{manifest.path_prefix.replace("/", "%2f")}' + fd = open(output_file, 'w') + + manifest.SetUseLocalManifests(not opt.ignore_local_manifests) + + if opt.json: + print('warning: --json is experimental!', file=sys.stderr) + doc = manifest.ToDict(peg_rev=opt.peg_rev, + peg_rev_upstream=opt.peg_rev_upstream, + peg_rev_dest_branch=opt.peg_rev_dest_branch) + + json_settings = { + # JSON style guide says Uunicode characters are fully allowed. + 'ensure_ascii': False, + # We use 2 space indent to match JSON style guide. + 'indent': 2 if opt.pretty else None, + 'separators': (',', ': ') if opt.pretty else (',', ':'), + 'sort_keys': True, + } + fd.write(json.dumps(doc, **json_settings)) + else: + manifest.Save(fd, + peg_rev=opt.peg_rev, + peg_rev_upstream=opt.peg_rev_upstream, + peg_rev_dest_branch=opt.peg_rev_dest_branch) + if output_file != '-': + fd.close() + if manifest.path_prefix: + print(f'Saved {manifest.path_prefix} submanifest to {output_file}', + file=sys.stderr) + else: + print(f'Saved manifest to {output_file}', file=sys.stderr) + def ValidateOptions(self, opt, args): if args: diff --git a/subcmds/overview.py b/subcmds/overview.py index 63f5a79e..11dba95f 100644 --- a/subcmds/overview.py +++ b/subcmds/overview.py @@ -47,7 +47,7 @@ are displayed. def Execute(self, opt, args): all_branches = [] - for project in self.GetProjects(args): + for project in self.GetProjects(args, all_manifests=not opt.this_manifest_only): br = [project.GetUploadableBranch(x) for x in project.GetBranches()] br = [x for x in br if x] @@ -76,7 +76,7 @@ are displayed. if project != branch.project: project = branch.project out.nl() - out.project('project %s/' % project.relpath) + out.project('project %s/' % project.RelPath(local=opt.this_manifest_only)) out.nl() commits = branch.commits diff --git a/subcmds/prune.py b/subcmds/prune.py index 584ee7ed..251accaa 100644 --- a/subcmds/prune.py +++ b/subcmds/prune.py @@ -31,7 +31,7 @@ class Prune(PagedCommand): return project.PruneHeads() def Execute(self, opt, args): - projects = self.GetProjects(args) + projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) # NB: Should be able to refactor this module to display summary as results # come back from children. @@ -63,7 +63,7 @@ class Prune(PagedCommand): if project != branch.project: project = branch.project out.nl() - out.project('project %s/' % project.relpath) + out.project('project %s/' % project.RelPath(local=opt.this_manifest_only)) out.nl() print('%s %-33s ' % ( diff --git a/subcmds/rebase.py b/subcmds/rebase.py index 7c53eb7a..3d1a63e4 100644 --- a/subcmds/rebase.py +++ b/subcmds/rebase.py @@ -69,7 +69,7 @@ branch but need to incorporate new upstream changes "underneath" them. 'consistent if you previously synced to a manifest)') def Execute(self, opt, args): - all_projects = self.GetProjects(args) + all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) one_project = len(all_projects) == 1 if opt.interactive and not one_project: @@ -98,6 +98,7 @@ branch but need to incorporate new upstream changes "underneath" them. config = self.manifest.manifestProject.config out = RebaseColoring(config) out.redirect(sys.stdout) + _RelPath = lambda p: p.RelPath(local=opt.this_manifest_only) ret = 0 for project in all_projects: @@ -107,7 +108,7 @@ branch but need to incorporate new upstream changes "underneath" them. cb = project.CurrentBranch if not cb: if one_project: - print("error: project %s has a detached HEAD" % project.relpath, + print("error: project %s has a detached HEAD" % _RelPath(project), file=sys.stderr) return 1 # ignore branches with detatched HEADs @@ -117,7 +118,7 @@ branch but need to incorporate new upstream changes "underneath" them. if not upbranch.LocalMerge: if one_project: print("error: project %s does not track any remote branches" - % project.relpath, file=sys.stderr) + % _RelPath(project), file=sys.stderr) return 1 # ignore branches without remotes continue @@ -130,7 +131,7 @@ branch but need to incorporate new upstream changes "underneath" them. args.append(upbranch.LocalMerge) out.project('project %s: rebasing %s -> %s', - project.relpath, cb, upbranch.LocalMerge) + _RelPath(project), cb, upbranch.LocalMerge) out.nl() out.flush() diff --git a/subcmds/stage.py b/subcmds/stage.py index 0389a4ff..5f17cb64 100644 --- a/subcmds/stage.py +++ b/subcmds/stage.py @@ -50,7 +50,9 @@ The '%prog' command stages files to prepare the next commit. self.Usage() def _Interactive(self, opt, args): - all_projects = [p for p in self.GetProjects(args) if p.IsDirty()] + all_projects = [ + p for p in self.GetProjects(args, all_manifests=not opt.this_manifest_only) + if p.IsDirty()] if not all_projects: print('no projects have uncommitted modifications', file=sys.stderr) return @@ -62,7 +64,8 @@ The '%prog' command stages files to prepare the next commit. for i in range(len(all_projects)): project = all_projects[i] - out.write('%3d: %s', i + 1, project.relpath + '/') + out.write('%3d: %s', i + 1, + project.RelPath(local=opt.this_manifest_only) + '/') out.nl() out.nl() @@ -99,7 +102,9 @@ The '%prog' command stages files to prepare the next commit. _AddI(all_projects[a_index - 1]) continue - projects = [p for p in all_projects if a in [p.name, p.relpath]] + projects = [ + p for p in all_projects + if a in [p.name, p.RelPath(local=opt.this_manifest_only)]] if len(projects) == 1: _AddI(projects[0]) continue diff --git a/subcmds/start.py b/subcmds/start.py index 2addaf2e..809df963 100644 --- a/subcmds/start.py +++ b/subcmds/start.py @@ -84,7 +84,8 @@ revision specified in the manifest. projects = ['.'] # start it in the local project by default all_projects = self.GetProjects(projects, - missing_ok=bool(self.gitc_manifest)) + missing_ok=bool(self.gitc_manifest), + all_manifests=not opt.this_manifest_only) # This must happen after we find all_projects, since GetProjects may need # the local directory, which will disappear once we save the GITC manifest. @@ -137,6 +138,6 @@ revision specified in the manifest. if err: for p in err: - print("error: %s/: cannot start %s" % (p.relpath, nb), + print("error: %s/: cannot start %s" % (p.RelPath(local=opt.this_manifest_only), nb), file=sys.stderr) sys.exit(1) diff --git a/subcmds/status.py b/subcmds/status.py index 5b669547..0aa4200f 100644 --- a/subcmds/status.py +++ b/subcmds/status.py @@ -117,7 +117,7 @@ the following meanings: outstring.append(''.join([status_header, item, '/'])) def Execute(self, opt, args): - all_projects = self.GetProjects(args) + all_projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) def _ProcessResults(_pool, _output, results): ret = 0 @@ -141,9 +141,10 @@ the following meanings: if opt.orphans: proj_dirs = set() proj_dirs_parents = set() - for project in self.GetProjects(None, missing_ok=True): - proj_dirs.add(project.relpath) - (head, _tail) = os.path.split(project.relpath) + for project in self.GetProjects(None, missing_ok=True, all_manifests=not opt.this_manifest_only): + relpath = project.RelPath(local=opt.this_manifest_only) + proj_dirs.add(relpath) + (head, _tail) = os.path.split(relpath) while head != "": proj_dirs_parents.add(head) (head, _tail) = os.path.split(head) diff --git a/subcmds/sync.py b/subcmds/sync.py index 707c5bbd..f5584dc8 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py @@ -66,6 +66,7 @@ _ONE_DAY_S = 24 * 60 * 60 class Sync(Command, MirrorSafeCommand): jobs = 1 COMMON = True + MULTI_MANIFEST_SUPPORT = False helpSummary = "Update working tree to the latest revision" helpUsage = """ %prog [...] @@ -704,7 +705,7 @@ later is required to fix a server side protocol bug. if project.relpath: new_project_paths.append(project.relpath) file_name = 'project.list' - file_path = os.path.join(self.repodir, file_name) + file_path = os.path.join(self.manifest.subdir, file_name) old_project_paths = [] if os.path.exists(file_path): @@ -760,7 +761,7 @@ later is required to fix a server side protocol bug. } copylinkfile_name = 'copy-link-files.json' - copylinkfile_path = os.path.join(self.manifest.repodir, copylinkfile_name) + copylinkfile_path = os.path.join(self.manifest.subdir, copylinkfile_name) old_copylinkfile_paths = {} if os.path.exists(copylinkfile_path): @@ -932,6 +933,9 @@ later is required to fix a server side protocol bug. if opt.prune is None: opt.prune = True + if self.manifest.is_multimanifest and not opt.this_manifest_only and args: + self.OptionParser.error('partial syncs must use --this-manifest-only') + def Execute(self, opt, args): if opt.jobs: self.jobs = opt.jobs diff --git a/subcmds/upload.py b/subcmds/upload.py index c48deab6..ef3d8e9d 100644 --- a/subcmds/upload.py +++ b/subcmds/upload.py @@ -226,7 +226,8 @@ Gerrit Code Review: https://www.gerritcodereview.com/ destination = opt.dest_branch or project.dest_branch or project.revisionExpr print('Upload project %s/ to remote branch %s%s:' % - (project.relpath, destination, ' (private)' if opt.private else '')) + (project.RelPath(local=opt.this_manifest_only), destination, + ' (private)' if opt.private else '')) print(' branch %s (%2d commit%s, %s):' % ( name, len(commit_list), @@ -262,7 +263,7 @@ Gerrit Code Review: https://www.gerritcodereview.com/ script.append('# Uncomment the branches to upload:') for project, avail in pending: script.append('#') - script.append('# project %s/:' % project.relpath) + script.append('# project %s/:' % project.RelPath(local=opt.this_manifest_only)) b = {} for branch in avail: @@ -285,7 +286,7 @@ Gerrit Code Review: https://www.gerritcodereview.com/ script.append('# %s' % commit) b[name] = branch - projects[project.relpath] = project + projects[project.RelPath(local=opt.this_manifest_only)] = project branches[project.name] = b script.append('') @@ -313,7 +314,7 @@ Gerrit Code Review: https://www.gerritcodereview.com/ _die('project for branch %s not in script', name) branch = branches[project.name].get(name) if not branch: - _die('branch %s not in %s', name, project.relpath) + _die('branch %s not in %s', name, project.RelPath(local=opt.this_manifest_only)) todo.append(branch) if not todo: _die("nothing uncommented for upload") @@ -481,7 +482,7 @@ Gerrit Code Review: https://www.gerritcodereview.com/ else: fmt = '\n (%s)' print(('[FAILED] %-15s %-15s' + fmt) % ( - branch.project.relpath + '/', + branch.project.RelPath(local=opt.this_manifest_only) + '/', branch.name, str(branch.error)), file=sys.stderr) @@ -490,7 +491,7 @@ Gerrit Code Review: https://www.gerritcodereview.com/ for branch in todo: if branch.uploaded: print('[OK ] %-15s %s' % ( - branch.project.relpath + '/', + branch.project.RelPath(local=opt.this_manifest_only) + '/', branch.name), file=sys.stderr) @@ -524,7 +525,7 @@ Gerrit Code Review: https://www.gerritcodereview.com/ return (project, avail) def Execute(self, opt, args): - projects = self.GetProjects(args) + projects = self.GetProjects(args, all_manifests=not opt.this_manifest_only) def _ProcessResults(_pool, _out, results): pending = [] @@ -534,7 +535,8 @@ Gerrit Code Review: https://www.gerritcodereview.com/ print('repo: error: %s: Unable to upload branch "%s". ' 'You might be able to fix the branch by running:\n' ' git branch --set-upstream-to m/%s' % - (project.relpath, project.CurrentBranch, self.manifest.branch), + (project.RelPath(local=opt.this_manifest_only), project.CurrentBranch, + project.manifest.branch), file=sys.stderr) elif avail: pending.append(result) @@ -554,15 +556,23 @@ Gerrit Code Review: https://www.gerritcodereview.com/ (opt.branch,), file=sys.stderr) return 1 - pending_proj_names = [project.name for (project, available) in pending] - pending_worktrees = [project.worktree for (project, available) in pending] - hook = RepoHook.FromSubcmd( - hook_type='pre-upload', manifest=self.manifest, - opt=opt, abort_if_user_denies=True) - if not hook.Run( - project_list=pending_proj_names, - worktree_list=pending_worktrees): - return 1 + manifests = {project.manifest.topdir: project.manifest + for (project, available) in pending} + ret = 0 + for manifest in manifests.values(): + pending_proj_names = [project.name for (project, available) in pending + if project.manifest.topdir == manifest.topdir] + pending_worktrees = [project.worktree for (project, available) in pending + if project.manifest.topdir == manifest.topdir] + hook = RepoHook.FromSubcmd( + hook_type='pre-upload', manifest=manifest, + opt=opt, abort_if_user_denies=True) + if not hook.Run( + project_list=pending_proj_names, + worktree_list=pending_worktrees): + ret = 1 + if ret: + return ret reviewers = _SplitEmails(opt.reviewers) if opt.reviewers else [] cc = _SplitEmails(opt.cc) if opt.cc else [] -- cgit v1.2.3-54-g00ecf