diff options
| author | LaMont Jones <lamontjones@google.com> | 2021-11-18 22:40:18 +0000 |
|---|---|---|
| committer | LaMont Jones <lamontjones@google.com> | 2022-02-17 21:57:55 +0000 |
| commit | cc879a97c3e2614d19b15b4661c3cab4d33139c9 (patch) | |
| tree | 69d225e9f0e9d79fec8f423d9c40c275f0bf3b8c /manifest_xml.py | |
| parent | 87cce68b28c34fa86895baa8d7f48307382e6c75 (diff) | |
| download | git-repo-cc879a97c3e2614d19b15b4661c3cab4d33139c9.tar.gz | |
Add multi-manifest support with <submanifest> elementv2.22
To be addressed in another change:
- a partial `repo sync` (with a list of projects/paths to sync)
requires `--this-tree-only`.
Change-Id: I6c7400bf001540e9d7694fa70934f8f204cb5f57
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/322657
Tested-by: LaMont Jones <lamontjones@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
Diffstat (limited to 'manifest_xml.py')
| -rw-r--r-- | manifest_xml.py | 454 |
1 files changed, 427 insertions, 27 deletions
diff --git a/manifest_xml.py b/manifest_xml.py index 7c5906da..7a4eb1e8 100644 --- a/manifest_xml.py +++ b/manifest_xml.py | |||
| @@ -33,6 +33,9 @@ from wrapper import Wrapper | |||
| 33 | MANIFEST_FILE_NAME = 'manifest.xml' | 33 | MANIFEST_FILE_NAME = 'manifest.xml' |
| 34 | LOCAL_MANIFEST_NAME = 'local_manifest.xml' | 34 | LOCAL_MANIFEST_NAME = 'local_manifest.xml' |
| 35 | LOCAL_MANIFESTS_DIR_NAME = 'local_manifests' | 35 | LOCAL_MANIFESTS_DIR_NAME = 'local_manifests' |
| 36 | SUBMANIFEST_DIR = 'submanifests' | ||
| 37 | # Limit submanifests to an arbitrary depth for loop detection. | ||
| 38 | MAX_SUBMANIFEST_DEPTH = 8 | ||
| 36 | 39 | ||
| 37 | # Add all projects from local manifest into a group. | 40 | # Add all projects from local manifest into a group. |
| 38 | LOCAL_MANIFEST_GROUP_PREFIX = 'local:' | 41 | LOCAL_MANIFEST_GROUP_PREFIX = 'local:' |
| @@ -197,10 +200,122 @@ class _XmlRemote(object): | |||
| 197 | self.annotations.append(Annotation(name, value, keep)) | 200 | self.annotations.append(Annotation(name, value, keep)) |
| 198 | 201 | ||
| 199 | 202 | ||
| 203 | class _XmlSubmanifest: | ||
| 204 | """Manage the <submanifest> element specified in the manifest. | ||
| 205 | |||
| 206 | Attributes: | ||
| 207 | name: a string, the name for this submanifest. | ||
| 208 | remote: a string, the remote.name for this submanifest. | ||
| 209 | project: a string, the name of the manifest project. | ||
| 210 | revision: a string, the commitish. | ||
| 211 | manifestName: a string, the submanifest file name. | ||
| 212 | groups: a list of strings, the groups to add to all projects in the submanifest. | ||
| 213 | path: a string, the relative path for the submanifest checkout. | ||
| 214 | annotations: (derived) a list of annotations. | ||
| 215 | present: (derived) a boolean, whether the submanifest's manifest file is present. | ||
| 216 | """ | ||
| 217 | def __init__(self, | ||
| 218 | name, | ||
| 219 | remote=None, | ||
| 220 | project=None, | ||
| 221 | revision=None, | ||
| 222 | manifestName=None, | ||
| 223 | groups=None, | ||
| 224 | path=None, | ||
| 225 | parent=None): | ||
| 226 | self.name = name | ||
| 227 | self.remote = remote | ||
| 228 | self.project = project | ||
| 229 | self.revision = revision | ||
| 230 | self.manifestName = manifestName | ||
| 231 | self.groups = groups | ||
| 232 | self.path = path | ||
| 233 | self.annotations = [] | ||
| 234 | outer_client = parent._outer_client or parent | ||
| 235 | if self.remote and not self.project: | ||
| 236 | raise ManifestParseError( | ||
| 237 | f'Submanifest {name}: must specify project when remote is given.') | ||
| 238 | rc = self.repo_client = RepoClient( | ||
| 239 | parent.repodir, manifestName, parent_groups=','.join(groups) or '', | ||
| 240 | submanifest_path=self.relpath, outer_client=outer_client) | ||
| 241 | |||
| 242 | self.present = os.path.exists(os.path.join(self.repo_client.subdir, | ||
| 243 | MANIFEST_FILE_NAME)) | ||
| 244 | |||
| 245 | def __eq__(self, other): | ||
| 246 | if not isinstance(other, _XmlSubmanifest): | ||
| 247 | return False | ||
| 248 | return ( | ||
| 249 | self.name == other.name and | ||
| 250 | self.remote == other.remote and | ||
| 251 | self.project == other.project and | ||
| 252 | self.revision == other.revision and | ||
| 253 | self.manifestName == other.manifestName and | ||
| 254 | self.groups == other.groups and | ||
| 255 | self.path == other.path and | ||
| 256 | sorted(self.annotations) == sorted(other.annotations)) | ||
| 257 | |||
| 258 | def __ne__(self, other): | ||
| 259 | return not self.__eq__(other) | ||
| 260 | |||
| 261 | def ToSubmanifestSpec(self, root): | ||
| 262 | """Return a SubmanifestSpec object, populating attributes""" | ||
| 263 | mp = root.manifestProject | ||
| 264 | remote = root.remotes[self.remote or root.default.remote.name] | ||
| 265 | # If a project was given, generate the url from the remote and project. | ||
| 266 | # If not, use this manifestProject's url. | ||
| 267 | if self.project: | ||
| 268 | manifestUrl = remote.ToRemoteSpec(self.project).url | ||
| 269 | else: | ||
| 270 | manifestUrl = mp.GetRemote(mp.remote.name).url | ||
| 271 | manifestName = self.manifestName or 'default.xml' | ||
| 272 | revision = self.revision or self.name | ||
| 273 | path = self.path or revision.split('/')[-1] | ||
| 274 | groups = self.groups or [] | ||
| 275 | |||
| 276 | return SubmanifestSpec(self.name, manifestUrl, manifestName, revision, path, | ||
| 277 | groups) | ||
| 278 | |||
| 279 | @property | ||
| 280 | def relpath(self): | ||
| 281 | """The path of this submanifest relative to the parent manifest.""" | ||
| 282 | revision = self.revision or self.name | ||
| 283 | return self.path or revision.split('/')[-1] | ||
| 284 | |||
| 285 | def GetGroupsStr(self): | ||
| 286 | """Returns the `groups` given for this submanifest.""" | ||
| 287 | if self.groups: | ||
| 288 | return ','.join(self.groups) | ||
| 289 | return '' | ||
| 290 | |||
| 291 | def AddAnnotation(self, name, value, keep): | ||
| 292 | """Add annotations to the submanifest.""" | ||
| 293 | self.annotations.append(Annotation(name, value, keep)) | ||
| 294 | |||
| 295 | |||
| 296 | class SubmanifestSpec: | ||
| 297 | """The submanifest element, with all fields expanded.""" | ||
| 298 | |||
| 299 | def __init__(self, | ||
| 300 | name, | ||
| 301 | manifestUrl, | ||
| 302 | manifestName, | ||
| 303 | revision, | ||
| 304 | path, | ||
| 305 | groups): | ||
| 306 | self.name = name | ||
| 307 | self.manifestUrl = manifestUrl | ||
| 308 | self.manifestName = manifestName | ||
| 309 | self.revision = revision | ||
| 310 | self.path = path | ||
| 311 | self.groups = groups or [] | ||
| 312 | |||
| 313 | |||
| 200 | class XmlManifest(object): | 314 | class XmlManifest(object): |
| 201 | """manages the repo configuration file""" | 315 | """manages the repo configuration file""" |
| 202 | 316 | ||
| 203 | def __init__(self, repodir, manifest_file, local_manifests=None): | 317 | def __init__(self, repodir, manifest_file, local_manifests=None, |
| 318 | outer_client=None, parent_groups='', submanifest_path=''): | ||
| 204 | """Initialize. | 319 | """Initialize. |
| 205 | 320 | ||
| 206 | Args: | 321 | Args: |
| @@ -210,23 +325,37 @@ class XmlManifest(object): | |||
| 210 | be |repodir|/|MANIFEST_FILE_NAME|. | 325 | be |repodir|/|MANIFEST_FILE_NAME|. |
| 211 | local_manifests: Full path to the directory of local override manifests. | 326 | local_manifests: Full path to the directory of local override manifests. |
| 212 | This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|. | 327 | This will usually be |repodir|/|LOCAL_MANIFESTS_DIR_NAME|. |
| 328 | outer_client: RepoClient of the outertree. | ||
| 329 | parent_groups: a string, the groups to apply to this projects. | ||
| 330 | submanifest_path: The submanifest root relative to the repo root. | ||
| 213 | """ | 331 | """ |
| 214 | # TODO(vapier): Move this out of this class. | 332 | # TODO(vapier): Move this out of this class. |
| 215 | self.globalConfig = GitConfig.ForUser() | 333 | self.globalConfig = GitConfig.ForUser() |
| 216 | 334 | ||
| 217 | self.repodir = os.path.abspath(repodir) | 335 | self.repodir = os.path.abspath(repodir) |
| 218 | self.topdir = os.path.dirname(self.repodir) | 336 | self._CheckLocalPath(submanifest_path) |
| 337 | self.topdir = os.path.join(os.path.dirname(self.repodir), submanifest_path) | ||
| 219 | self.manifestFile = manifest_file | 338 | self.manifestFile = manifest_file |
| 220 | self.local_manifests = local_manifests | 339 | self.local_manifests = local_manifests |
| 221 | self._load_local_manifests = True | 340 | self._load_local_manifests = True |
| 341 | self.parent_groups = parent_groups | ||
| 342 | |||
| 343 | if outer_client and self.isGitcClient: | ||
| 344 | raise ManifestParseError('Multi-manifest is incompatible with `gitc-init`') | ||
| 345 | |||
| 346 | if submanifest_path and not outer_client: | ||
| 347 | # If passing a submanifest_path, there must be an outer_client. | ||
| 348 | raise ManifestParseError(f'Bad call to {self.__class__.__name__}') | ||
| 349 | |||
| 350 | # If self._outer_client is None, this is not a checkout that supports | ||
| 351 | # multi-tree. | ||
| 352 | self._outer_client = outer_client or self | ||
| 222 | 353 | ||
| 223 | self.repoProject = MetaProject(self, 'repo', | 354 | self.repoProject = MetaProject(self, 'repo', |
| 224 | gitdir=os.path.join(repodir, 'repo/.git'), | 355 | gitdir=os.path.join(repodir, 'repo/.git'), |
| 225 | worktree=os.path.join(repodir, 'repo')) | 356 | worktree=os.path.join(repodir, 'repo')) |
| 226 | 357 | ||
| 227 | mp = MetaProject(self, 'manifests', | 358 | mp = self.SubmanifestProject(self.path_prefix) |
| 228 | gitdir=os.path.join(repodir, 'manifests.git'), | ||
| 229 | worktree=os.path.join(repodir, 'manifests')) | ||
| 230 | self.manifestProject = mp | 359 | self.manifestProject = mp |
| 231 | 360 | ||
| 232 | # This is a bit hacky, but we're in a chicken & egg situation: all the | 361 | # This is a bit hacky, but we're in a chicken & egg situation: all the |
| @@ -311,6 +440,31 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 311 | ae.setAttribute('value', a.value) | 440 | ae.setAttribute('value', a.value) |
| 312 | e.appendChild(ae) | 441 | e.appendChild(ae) |
| 313 | 442 | ||
| 443 | def _SubmanifestToXml(self, r, doc, root): | ||
| 444 | """Generate XML <submanifest/> node.""" | ||
| 445 | e = doc.createElement('submanifest') | ||
| 446 | root.appendChild(e) | ||
| 447 | e.setAttribute('name', r.name) | ||
| 448 | if r.remote is not None: | ||
| 449 | e.setAttribute('remote', r.remote) | ||
| 450 | if r.project is not None: | ||
| 451 | e.setAttribute('project', r.project) | ||
| 452 | if r.manifestName is not None: | ||
| 453 | e.setAttribute('manifest-name', r.manifestName) | ||
| 454 | if r.revision is not None: | ||
| 455 | e.setAttribute('revision', r.revision) | ||
| 456 | if r.path is not None: | ||
| 457 | e.setAttribute('path', r.path) | ||
| 458 | if r.groups: | ||
| 459 | e.setAttribute('groups', r.GetGroupsStr()) | ||
| 460 | |||
| 461 | for a in r.annotations: | ||
| 462 | if a.keep == 'true': | ||
| 463 | ae = doc.createElement('annotation') | ||
| 464 | ae.setAttribute('name', a.name) | ||
| 465 | ae.setAttribute('value', a.value) | ||
| 466 | e.appendChild(ae) | ||
| 467 | |||
| 314 | def _ParseList(self, field): | 468 | def _ParseList(self, field): |
| 315 | """Parse fields that contain flattened lists. | 469 | """Parse fields that contain flattened lists. |
| 316 | 470 | ||
| @@ -329,6 +483,8 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 329 | 483 | ||
| 330 | doc = xml.dom.minidom.Document() | 484 | doc = xml.dom.minidom.Document() |
| 331 | root = doc.createElement('manifest') | 485 | root = doc.createElement('manifest') |
| 486 | if self.is_submanifest: | ||
| 487 | root.setAttribute('path', self.path_prefix) | ||
| 332 | doc.appendChild(root) | 488 | doc.appendChild(root) |
| 333 | 489 | ||
| 334 | # Save out the notice. There's a little bit of work here to give it the | 490 | # Save out the notice. There's a little bit of work here to give it the |
| @@ -383,6 +539,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 383 | root.appendChild(e) | 539 | root.appendChild(e) |
| 384 | root.appendChild(doc.createTextNode('')) | 540 | root.appendChild(doc.createTextNode('')) |
| 385 | 541 | ||
| 542 | for r in sorted(self.submanifests): | ||
| 543 | self._SubmanifestToXml(self.submanifests[r], doc, root) | ||
| 544 | if self.submanifests: | ||
| 545 | root.appendChild(doc.createTextNode('')) | ||
| 546 | |||
| 386 | def output_projects(parent, parent_node, projects): | 547 | def output_projects(parent, parent_node, projects): |
| 387 | for project_name in projects: | 548 | for project_name in projects: |
| 388 | for project in self._projects[project_name]: | 549 | for project in self._projects[project_name]: |
| @@ -537,6 +698,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 537 | 'project', | 698 | 'project', |
| 538 | 'extend-project', | 699 | 'extend-project', |
| 539 | 'include', | 700 | 'include', |
| 701 | 'submanifest', | ||
| 540 | # These are children of 'project' nodes. | 702 | # These are children of 'project' nodes. |
| 541 | 'annotation', | 703 | 'annotation', |
| 542 | 'project', | 704 | 'project', |
| @@ -575,12 +737,74 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 575 | """Manifests can modify e if they support extra project attributes.""" | 737 | """Manifests can modify e if they support extra project attributes.""" |
| 576 | 738 | ||
| 577 | @property | 739 | @property |
| 740 | def is_multimanifest(self): | ||
| 741 | """Whether this is a multimanifest checkout""" | ||
| 742 | return bool(self.outer_client.submanifests) | ||
| 743 | |||
| 744 | @property | ||
| 745 | def is_submanifest(self): | ||
| 746 | """Whether this manifest is a submanifest""" | ||
| 747 | return self._outer_client and self._outer_client != self | ||
| 748 | |||
| 749 | @property | ||
| 750 | def outer_client(self): | ||
| 751 | """The instance of the outermost manifest client""" | ||
| 752 | self._Load() | ||
| 753 | return self._outer_client | ||
| 754 | |||
| 755 | @property | ||
| 756 | def all_manifests(self): | ||
| 757 | """Generator yielding all (sub)manifests.""" | ||
| 758 | self._Load() | ||
| 759 | outer = self._outer_client | ||
| 760 | yield outer | ||
| 761 | for tree in outer.all_children: | ||
| 762 | yield tree | ||
| 763 | |||
| 764 | @property | ||
| 765 | def all_children(self): | ||
| 766 | """Generator yielding all child submanifests.""" | ||
| 767 | self._Load() | ||
| 768 | for child in self._submanifests.values(): | ||
| 769 | if child.repo_client: | ||
| 770 | yield child.repo_client | ||
| 771 | for tree in child.repo_client.all_children: | ||
| 772 | yield tree | ||
| 773 | |||
| 774 | @property | ||
| 775 | def path_prefix(self): | ||
| 776 | """The path of this submanifest, relative to the outermost manifest.""" | ||
| 777 | if not self._outer_client or self == self._outer_client: | ||
| 778 | return '' | ||
| 779 | return os.path.relpath(self.topdir, self._outer_client.topdir) | ||
| 780 | |||
| 781 | @property | ||
| 782 | def all_paths(self): | ||
| 783 | """All project paths for all (sub)manifests. See `paths`.""" | ||
| 784 | ret = {} | ||
| 785 | for tree in self.all_manifests: | ||
| 786 | prefix = tree.path_prefix | ||
| 787 | ret.update({os.path.join(prefix, k): v for k, v in tree.paths.items()}) | ||
| 788 | return ret | ||
| 789 | |||
| 790 | @property | ||
| 791 | def all_projects(self): | ||
| 792 | """All projects for all (sub)manifests. See `projects`.""" | ||
| 793 | return list(itertools.chain.from_iterable(x._paths.values() for x in self.all_manifests)) | ||
| 794 | |||
| 795 | @property | ||
| 578 | def paths(self): | 796 | def paths(self): |
| 797 | """Return all paths for this manifest. | ||
| 798 | |||
| 799 | Return: | ||
| 800 | A dictionary of {path: Project()}. `path` is relative to this manifest. | ||
| 801 | """ | ||
| 579 | self._Load() | 802 | self._Load() |
| 580 | return self._paths | 803 | return self._paths |
| 581 | 804 | ||
| 582 | @property | 805 | @property |
| 583 | def projects(self): | 806 | def projects(self): |
| 807 | """Return a list of all Projects in this manifest.""" | ||
| 584 | self._Load() | 808 | self._Load() |
| 585 | return list(self._paths.values()) | 809 | return list(self._paths.values()) |
| 586 | 810 | ||
| @@ -595,6 +819,12 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 595 | return self._default | 819 | return self._default |
| 596 | 820 | ||
| 597 | @property | 821 | @property |
| 822 | def submanifests(self): | ||
| 823 | """All submanifests in this manifest.""" | ||
| 824 | self._Load() | ||
| 825 | return self._submanifests | ||
| 826 | |||
| 827 | @property | ||
| 598 | def repo_hooks_project(self): | 828 | def repo_hooks_project(self): |
| 599 | self._Load() | 829 | self._Load() |
| 600 | return self._repo_hooks_project | 830 | return self._repo_hooks_project |
| @@ -651,8 +881,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 651 | return self._load_local_manifests and self.local_manifests | 881 | return self._load_local_manifests and self.local_manifests |
| 652 | 882 | ||
| 653 | def IsFromLocalManifest(self, project): | 883 | def IsFromLocalManifest(self, project): |
| 654 | """Is the project from a local manifest? | 884 | """Is the project from a local manifest?""" |
| 655 | """ | ||
| 656 | return any(x.startswith(LOCAL_MANIFEST_GROUP_PREFIX) | 885 | return any(x.startswith(LOCAL_MANIFEST_GROUP_PREFIX) |
| 657 | for x in project.groups) | 886 | for x in project.groups) |
| 658 | 887 | ||
| @@ -676,6 +905,50 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 676 | def EnableGitLfs(self): | 905 | def EnableGitLfs(self): |
| 677 | return self.manifestProject.config.GetBoolean('repo.git-lfs') | 906 | return self.manifestProject.config.GetBoolean('repo.git-lfs') |
| 678 | 907 | ||
| 908 | def FindManifestByPath(self, path): | ||
| 909 | """Returns the manifest containing path.""" | ||
| 910 | path = os.path.abspath(path) | ||
| 911 | manifest = self._outer_client or self | ||
| 912 | old = None | ||
| 913 | while manifest._submanifests and manifest != old: | ||
| 914 | old = manifest | ||
| 915 | for name in manifest._submanifests: | ||
| 916 | tree = manifest._submanifests[name] | ||
| 917 | if path.startswith(tree.repo_client.manifest.topdir): | ||
| 918 | manifest = tree.repo_client | ||
| 919 | break | ||
| 920 | return manifest | ||
| 921 | |||
| 922 | @property | ||
| 923 | def subdir(self): | ||
| 924 | """Returns the path for per-submanifest objects for this manifest.""" | ||
| 925 | return self.SubmanifestInfoDir(self.path_prefix) | ||
| 926 | |||
| 927 | def SubmanifestInfoDir(self, submanifest_path, object_path=''): | ||
| 928 | """Return the path to submanifest-specific info for a submanifest. | ||
| 929 | |||
| 930 | Return the full path of the directory in which to put per-manifest objects. | ||
| 931 | |||
| 932 | Args: | ||
| 933 | submanifest_path: a string, the path of the submanifest, relative to the | ||
| 934 | outermost topdir. If empty, then repodir is returned. | ||
| 935 | object_path: a string, relative path to append to the submanifest info | ||
| 936 | directory path. | ||
| 937 | """ | ||
| 938 | if submanifest_path: | ||
| 939 | return os.path.join(self.repodir, SUBMANIFEST_DIR, submanifest_path, | ||
| 940 | object_path) | ||
| 941 | else: | ||
| 942 | return os.path.join(self.repodir, object_path) | ||
| 943 | |||
| 944 | def SubmanifestProject(self, submanifest_path): | ||
| 945 | """Return a manifestProject for a submanifest.""" | ||
| 946 | subdir = self.SubmanifestInfoDir(submanifest_path) | ||
| 947 | mp = MetaProject(self, 'manifests', | ||
| 948 | gitdir=os.path.join(subdir, 'manifests.git'), | ||
| 949 | worktree=os.path.join(subdir, 'manifests')) | ||
| 950 | return mp | ||
| 951 | |||
| 679 | def GetDefaultGroupsStr(self): | 952 | def GetDefaultGroupsStr(self): |
| 680 | """Returns the default group string for the platform.""" | 953 | """Returns the default group string for the platform.""" |
| 681 | return 'default,platform-' + platform.system().lower() | 954 | return 'default,platform-' + platform.system().lower() |
| @@ -693,6 +966,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 693 | self._paths = {} | 966 | self._paths = {} |
| 694 | self._remotes = {} | 967 | self._remotes = {} |
| 695 | self._default = None | 968 | self._default = None |
| 969 | self._submanifests = {} | ||
| 696 | self._repo_hooks_project = None | 970 | self._repo_hooks_project = None |
| 697 | self._superproject = {} | 971 | self._superproject = {} |
| 698 | self._contactinfo = ContactInfo(Wrapper().BUG_URL) | 972 | self._contactinfo = ContactInfo(Wrapper().BUG_URL) |
| @@ -700,20 +974,29 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 700 | self.branch = None | 974 | self.branch = None |
| 701 | self._manifest_server = None | 975 | self._manifest_server = None |
| 702 | 976 | ||
| 703 | def _Load(self): | 977 | def _Load(self, initial_client=None, submanifest_depth=0): |
| 978 | if submanifest_depth > MAX_SUBMANIFEST_DEPTH: | ||
| 979 | raise ManifestParseError('maximum submanifest depth %d exceeded.' % | ||
| 980 | MAX_SUBMANIFEST_DEPTH) | ||
| 704 | if not self._loaded: | 981 | if not self._loaded: |
| 982 | if self._outer_client and self._outer_client != self: | ||
| 983 | # This will load all clients. | ||
| 984 | self._outer_client._Load(initial_client=self) | ||
| 985 | |||
| 705 | m = self.manifestProject | 986 | m = self.manifestProject |
| 706 | b = m.GetBranch(m.CurrentBranch).merge | 987 | b = m.GetBranch(m.CurrentBranch).merge |
| 707 | if b is not None and b.startswith(R_HEADS): | 988 | if b is not None and b.startswith(R_HEADS): |
| 708 | b = b[len(R_HEADS):] | 989 | b = b[len(R_HEADS):] |
| 709 | self.branch = b | 990 | self.branch = b |
| 710 | 991 | ||
| 992 | parent_groups = self.parent_groups | ||
| 993 | |||
| 711 | # The manifestFile was specified by the user which is why we allow include | 994 | # The manifestFile was specified by the user which is why we allow include |
| 712 | # paths to point anywhere. | 995 | # paths to point anywhere. |
| 713 | nodes = [] | 996 | nodes = [] |
| 714 | nodes.append(self._ParseManifestXml( | 997 | nodes.append(self._ParseManifestXml( |
| 715 | self.manifestFile, self.manifestProject.worktree, | 998 | self.manifestFile, self.manifestProject.worktree, |
| 716 | restrict_includes=False)) | 999 | parent_groups=parent_groups, restrict_includes=False)) |
| 717 | 1000 | ||
| 718 | if self._load_local_manifests and self.local_manifests: | 1001 | if self._load_local_manifests and self.local_manifests: |
| 719 | try: | 1002 | try: |
| @@ -722,9 +1005,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 722 | local = os.path.join(self.local_manifests, local_file) | 1005 | local = os.path.join(self.local_manifests, local_file) |
| 723 | # Since local manifests are entirely managed by the user, allow | 1006 | # Since local manifests are entirely managed by the user, allow |
| 724 | # them to point anywhere the user wants. | 1007 | # them to point anywhere the user wants. |
| 1008 | local_group = f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}' | ||
| 725 | nodes.append(self._ParseManifestXml( | 1009 | nodes.append(self._ParseManifestXml( |
| 726 | local, self.repodir, | 1010 | local, self.subdir, |
| 727 | parent_groups=f'{LOCAL_MANIFEST_GROUP_PREFIX}:{local_file[:-4]}', | 1011 | parent_groups=f'{local_group},{parent_groups}', |
| 728 | restrict_includes=False)) | 1012 | restrict_includes=False)) |
| 729 | except OSError: | 1013 | except OSError: |
| 730 | pass | 1014 | pass |
| @@ -743,6 +1027,23 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 743 | 1027 | ||
| 744 | self._loaded = True | 1028 | self._loaded = True |
| 745 | 1029 | ||
| 1030 | # Now that we have loaded this manifest, load any submanifest manifests | ||
| 1031 | # as well. We need to do this after self._loaded is set to avoid looping. | ||
| 1032 | if self._outer_client: | ||
| 1033 | for name in self._submanifests: | ||
| 1034 | tree = self._submanifests[name] | ||
| 1035 | spec = tree.ToSubmanifestSpec(self) | ||
| 1036 | present = os.path.exists(os.path.join(self.subdir, MANIFEST_FILE_NAME)) | ||
| 1037 | if present and tree.present and not tree.repo_client: | ||
| 1038 | if initial_client and initial_client.topdir == self.topdir: | ||
| 1039 | tree.repo_client = self | ||
| 1040 | tree.present = present | ||
| 1041 | elif not os.path.exists(self.subdir): | ||
| 1042 | tree.present = False | ||
| 1043 | if tree.present: | ||
| 1044 | tree.repo_client._Load(initial_client=initial_client, | ||
| 1045 | submanifest_depth=submanifest_depth + 1) | ||
| 1046 | |||
| 746 | def _ParseManifestXml(self, path, include_root, parent_groups='', | 1047 | def _ParseManifestXml(self, path, include_root, parent_groups='', |
| 747 | restrict_includes=True): | 1048 | restrict_includes=True): |
| 748 | """Parse a manifest XML and return the computed nodes. | 1049 | """Parse a manifest XML and return the computed nodes. |
| @@ -832,6 +1133,20 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 832 | if self._default is None: | 1133 | if self._default is None: |
| 833 | self._default = _Default() | 1134 | self._default = _Default() |
| 834 | 1135 | ||
| 1136 | submanifest_paths = set() | ||
| 1137 | for node in itertools.chain(*node_list): | ||
| 1138 | if node.nodeName == 'submanifest': | ||
| 1139 | submanifest = self._ParseSubmanifest(node) | ||
| 1140 | if submanifest: | ||
| 1141 | if submanifest.name in self._submanifests: | ||
| 1142 | if submanifest != self._submanifests[submanifest.name]: | ||
| 1143 | raise ManifestParseError( | ||
| 1144 | 'submanifest %s already exists with different attributes' % | ||
| 1145 | (submanifest.name)) | ||
| 1146 | else: | ||
| 1147 | self._submanifests[submanifest.name] = submanifest | ||
| 1148 | submanifest_paths.add(submanifest.relpath) | ||
| 1149 | |||
| 835 | for node in itertools.chain(*node_list): | 1150 | for node in itertools.chain(*node_list): |
| 836 | if node.nodeName == 'notice': | 1151 | if node.nodeName == 'notice': |
| 837 | if self._notice is not None: | 1152 | if self._notice is not None: |
| @@ -859,6 +1174,11 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 859 | raise ManifestParseError( | 1174 | raise ManifestParseError( |
| 860 | 'duplicate path %s in %s' % | 1175 | 'duplicate path %s in %s' % |
| 861 | (project.relpath, self.manifestFile)) | 1176 | (project.relpath, self.manifestFile)) |
| 1177 | for tree in submanifest_paths: | ||
| 1178 | if project.relpath.startswith(tree): | ||
| 1179 | raise ManifestParseError( | ||
| 1180 | 'project %s conflicts with submanifest path %s' % | ||
| 1181 | (project.relpath, tree)) | ||
| 862 | self._paths[project.relpath] = project | 1182 | self._paths[project.relpath] = project |
| 863 | projects.append(project) | 1183 | projects.append(project) |
| 864 | for subproject in project.subprojects: | 1184 | for subproject in project.subprojects: |
| @@ -883,8 +1203,10 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 883 | if groups: | 1203 | if groups: |
| 884 | groups = self._ParseList(groups) | 1204 | groups = self._ParseList(groups) |
| 885 | revision = node.getAttribute('revision') | 1205 | revision = node.getAttribute('revision') |
| 886 | remote = node.getAttribute('remote') | 1206 | remote_name = node.getAttribute('remote') |
| 887 | if remote: | 1207 | if not remote_name: |
| 1208 | remote = self._default.remote | ||
| 1209 | else: | ||
| 888 | remote = self._get_remote(node) | 1210 | remote = self._get_remote(node) |
| 889 | 1211 | ||
| 890 | named_projects = self._projects[name] | 1212 | named_projects = self._projects[name] |
| @@ -899,12 +1221,13 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 899 | if revision: | 1221 | if revision: |
| 900 | p.SetRevision(revision) | 1222 | p.SetRevision(revision) |
| 901 | 1223 | ||
| 902 | if remote: | 1224 | if remote_name: |
| 903 | p.remote = remote.ToRemoteSpec(name) | 1225 | p.remote = remote.ToRemoteSpec(name) |
| 904 | 1226 | ||
| 905 | if dest_path: | 1227 | if dest_path: |
| 906 | del self._paths[p.relpath] | 1228 | del self._paths[p.relpath] |
| 907 | relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths(name, dest_path) | 1229 | relpath, worktree, gitdir, objdir, _ = self.GetProjectPaths( |
| 1230 | name, dest_path, remote.name) | ||
| 908 | p.UpdatePaths(relpath, worktree, gitdir, objdir) | 1231 | p.UpdatePaths(relpath, worktree, gitdir, objdir) |
| 909 | self._paths[p.relpath] = p | 1232 | self._paths[p.relpath] = p |
| 910 | 1233 | ||
| @@ -1109,6 +1432,53 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 1109 | 1432 | ||
| 1110 | return '\n'.join(cleanLines) | 1433 | return '\n'.join(cleanLines) |
| 1111 | 1434 | ||
| 1435 | def _ParseSubmanifest(self, node): | ||
| 1436 | """Reads a <submanifest> element from the manifest file.""" | ||
| 1437 | name = self._reqatt(node, 'name') | ||
| 1438 | remote = node.getAttribute('remote') | ||
| 1439 | if remote == '': | ||
| 1440 | remote = None | ||
| 1441 | project = node.getAttribute('project') | ||
| 1442 | if project == '': | ||
| 1443 | project = None | ||
| 1444 | revision = node.getAttribute('revision') | ||
| 1445 | if revision == '': | ||
| 1446 | revision = None | ||
| 1447 | manifestName = node.getAttribute('manifest-name') | ||
| 1448 | if manifestName == '': | ||
| 1449 | manifestName = None | ||
| 1450 | groups = '' | ||
| 1451 | if node.hasAttribute('groups'): | ||
| 1452 | groups = node.getAttribute('groups') | ||
| 1453 | groups = self._ParseList(groups) | ||
| 1454 | path = node.getAttribute('path') | ||
| 1455 | if path == '': | ||
| 1456 | path = None | ||
| 1457 | if revision: | ||
| 1458 | msg = self._CheckLocalPath(revision.split('/')[-1]) | ||
| 1459 | if msg: | ||
| 1460 | raise ManifestInvalidPathError( | ||
| 1461 | '<submanifest> invalid "revision": %s: %s' % (revision, msg)) | ||
| 1462 | else: | ||
| 1463 | msg = self._CheckLocalPath(name) | ||
| 1464 | if msg: | ||
| 1465 | raise ManifestInvalidPathError( | ||
| 1466 | '<submanifest> invalid "name": %s: %s' % (name, msg)) | ||
| 1467 | else: | ||
| 1468 | msg = self._CheckLocalPath(path) | ||
| 1469 | if msg: | ||
| 1470 | raise ManifestInvalidPathError( | ||
| 1471 | '<submanifest> invalid "path": %s: %s' % (path, msg)) | ||
| 1472 | |||
| 1473 | submanifest = _XmlSubmanifest(name, remote, project, revision, manifestName, | ||
| 1474 | groups, path, self) | ||
| 1475 | |||
| 1476 | for n in node.childNodes: | ||
| 1477 | if n.nodeName == 'annotation': | ||
| 1478 | self._ParseAnnotation(submanifest, n) | ||
| 1479 | |||
| 1480 | return submanifest | ||
| 1481 | |||
| 1112 | def _JoinName(self, parent_name, name): | 1482 | def _JoinName(self, parent_name, name): |
| 1113 | return os.path.join(parent_name, name) | 1483 | return os.path.join(parent_name, name) |
| 1114 | 1484 | ||
| @@ -1172,7 +1542,7 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 1172 | 1542 | ||
| 1173 | if parent is None: | 1543 | if parent is None: |
| 1174 | relpath, worktree, gitdir, objdir, use_git_worktrees = \ | 1544 | relpath, worktree, gitdir, objdir, use_git_worktrees = \ |
| 1175 | self.GetProjectPaths(name, path) | 1545 | self.GetProjectPaths(name, path, remote.name) |
| 1176 | else: | 1546 | else: |
| 1177 | use_git_worktrees = False | 1547 | use_git_worktrees = False |
| 1178 | relpath, worktree, gitdir, objdir = \ | 1548 | relpath, worktree, gitdir, objdir = \ |
| @@ -1218,31 +1588,54 @@ https://gerrit.googlesource.com/git-repo/+/HEAD/docs/manifest-format.md | |||
| 1218 | 1588 | ||
| 1219 | return project | 1589 | return project |
| 1220 | 1590 | ||
| 1221 | def GetProjectPaths(self, name, path): | 1591 | def GetProjectPaths(self, name, path, remote): |
| 1592 | """Return the paths for a project. | ||
| 1593 | |||
| 1594 | Args: | ||
| 1595 | name: a string, the name of the project. | ||
| 1596 | path: a string, the path of the project. | ||
| 1597 | remote: a string, the remote.name of the project. | ||
| 1598 | """ | ||
| 1222 | # The manifest entries might have trailing slashes. Normalize them to avoid | 1599 | # The manifest entries might have trailing slashes. Normalize them to avoid |
| 1223 | # unexpected filesystem behavior since we do string concatenation below. | 1600 | # unexpected filesystem behavior since we do string concatenation below. |
| 1224 | path = path.rstrip('/') | 1601 | path = path.rstrip('/') |
| 1225 | name = name.rstrip('/') | 1602 | name = name.rstrip('/') |
| 1603 | remote = remote.rstrip('/') | ||
| 1226 | use_git_worktrees = False | 1604 | use_git_worktrees = False |
| 1605 | use_remote_name = bool(self._outer_client._submanifests) | ||
| 1227 | relpath = path | 1606 | relpath = path |
| 1228 | if self.IsMirror: | 1607 | if self.IsMirror: |
| 1229 | worktree = None | 1608 | worktree = None |
| 1230 | gitdir = os.path.join(self.topdir, '%s.git' % name) | 1609 | gitdir = os.path.join(self.topdir, '%s.git' % name) |
| 1231 | objdir = gitdir | 1610 | objdir = gitdir |
| 1232 | else: | 1611 | else: |
| 1612 | if use_remote_name: | ||
| 1613 | namepath = os.path.join(remote, f'{name}.git') | ||
| 1614 | else: | ||
| 1615 | namepath = f'{name}.git' | ||
| 1233 | worktree = os.path.join(self.topdir, path).replace('\\', '/') | 1616 | worktree = os.path.join(self.topdir, path).replace('\\', '/') |
| 1234 | gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path) | 1617 | gitdir = os.path.join(self.subdir, 'projects', '%s.git' % path) |
| 1235 | # We allow people to mix git worktrees & non-git worktrees for now. | 1618 | # We allow people to mix git worktrees & non-git worktrees for now. |
| 1236 | # This allows for in situ migration of repo clients. | 1619 | # This allows for in situ migration of repo clients. |
| 1237 | if os.path.exists(gitdir) or not self.UseGitWorktrees: | 1620 | if os.path.exists(gitdir) or not self.UseGitWorktrees: |
| 1238 | objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name) | 1621 | objdir = os.path.join(self.subdir, 'project-objects', namepath) |
| 1239 | else: | 1622 | else: |
| 1240 | use_git_worktrees = True | 1623 | use_git_worktrees = True |
| 1241 | gitdir = os.path.join(self.repodir, 'worktrees', '%s.git' % name) | 1624 | gitdir = os.path.join(self.repodir, 'worktrees', namepath) |
| 1242 | objdir = gitdir | 1625 | objdir = gitdir |
| 1243 | return relpath, worktree, gitdir, objdir, use_git_worktrees | 1626 | return relpath, worktree, gitdir, objdir, use_git_worktrees |
| 1244 | 1627 | ||
| 1245 | def GetProjectsWithName(self, name): | 1628 | def GetProjectsWithName(self, name, all_manifests=False): |
| 1629 | """All projects with |name|. | ||
| 1630 | |||
| 1631 | Args: | ||
| 1632 | name: a string, the name of the project. | ||
| 1633 | all_manifests: a boolean, if True, then all manifests are searched. If | ||
| 1634 | False, then only this manifest is searched. | ||
| 1635 | """ | ||
| 1636 | if all_manifests: | ||
| 1637 | return list(itertools.chain.from_iterable( | ||
| 1638 | x._projects.get(name, []) for x in self.all_manifests)) | ||
| 1246 | return self._projects.get(name, []) | 1639 | return self._projects.get(name, []) |
| 1247 | 1640 | ||
| 1248 | def GetSubprojectName(self, parent, submodule_path): | 1641 | def GetSubprojectName(self, parent, submodule_path): |
| @@ -1498,19 +1891,26 @@ class GitcManifest(XmlManifest): | |||
| 1498 | class RepoClient(XmlManifest): | 1891 | class RepoClient(XmlManifest): |
| 1499 | """Manages a repo client checkout.""" | 1892 | """Manages a repo client checkout.""" |
| 1500 | 1893 | ||
| 1501 | def __init__(self, repodir, manifest_file=None): | 1894 | def __init__(self, repodir, manifest_file=None, submanifest_path='', **kwargs): |
| 1502 | self.isGitcClient = False | 1895 | self.isGitcClient = False |
| 1896 | submanifest_path = submanifest_path or '' | ||
| 1897 | if submanifest_path: | ||
| 1898 | self._CheckLocalPath(submanifest_path) | ||
| 1899 | prefix = os.path.join(repodir, SUBMANIFEST_DIR, submanifest_path) | ||
| 1900 | else: | ||
| 1901 | prefix = repodir | ||
| 1503 | 1902 | ||
| 1504 | if os.path.exists(os.path.join(repodir, LOCAL_MANIFEST_NAME)): | 1903 | if os.path.exists(os.path.join(prefix, LOCAL_MANIFEST_NAME)): |
| 1505 | print('error: %s is not supported; put local manifests in `%s` instead' % | 1904 | print('error: %s is not supported; put local manifests in `%s` instead' % |
| 1506 | (LOCAL_MANIFEST_NAME, os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)), | 1905 | (LOCAL_MANIFEST_NAME, os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)), |
| 1507 | file=sys.stderr) | 1906 | file=sys.stderr) |
| 1508 | sys.exit(1) | 1907 | sys.exit(1) |
| 1509 | 1908 | ||
| 1510 | if manifest_file is None: | 1909 | if manifest_file is None: |
| 1511 | manifest_file = os.path.join(repodir, MANIFEST_FILE_NAME) | 1910 | manifest_file = os.path.join(prefix, MANIFEST_FILE_NAME) |
| 1512 | local_manifests = os.path.abspath(os.path.join(repodir, LOCAL_MANIFESTS_DIR_NAME)) | 1911 | local_manifests = os.path.abspath(os.path.join(prefix, LOCAL_MANIFESTS_DIR_NAME)) |
| 1513 | super().__init__(repodir, manifest_file, local_manifests) | 1912 | super().__init__(repodir, manifest_file, local_manifests, |
| 1913 | submanifest_path=submanifest_path, **kwargs) | ||
| 1514 | 1914 | ||
| 1515 | # TODO: Completely separate manifest logic out of the client. | 1915 | # TODO: Completely separate manifest logic out of the client. |
| 1516 | self.manifest = self | 1916 | self.manifest = self |
