diff options
| author | Raman Tenneti <rtenneti@google.com> | 2021-06-11 17:29:45 -0700 |
|---|---|---|
| committer | Raman Tenneti <rtenneti@google.com> | 2021-06-16 04:48:35 +0000 |
| commit | 784e16f3aa941ca3564d823cc686017a161621a1 (patch) | |
| tree | 611fe2add0b6638330ea7b34fa1f70d05c0d895e | |
| parent | b8c84483a55b575277ab791c6834a8682d47f1af (diff) | |
| download | git-repo-784e16f3aa941ca3564d823cc686017a161621a1.tar.gz | |
superproject: Don't exit if superproject tag doesn't exist in manifest.v2.16
Don't exit if there are missing commit ids in superproject.
This change implements the following suggestion from delphij@:
"we should note the event (so we know that --use-superproject but there
were some errors, e.g. manifest didn't specify commit id for some
reason, or if there is no superproject but --use-superproject is
used), print out a message telling the use that this is not support,
but continue as if --no-use-superproject was specified?"
Changes:
superproject:
+ Added git_trace2_event_log as an argument to the constructor.
+ Sync method returns SyncResult a NamedTuple of
++ success - True if sync of superproject is successful, or False.
++ fatal - True if caller should exit, Or False.
+ UpdateProjectsRevisionId returns UpdateProjectsResult a NamedTuple of
++ manifest_path - path name of the overriding manifest file instead
of None
++ fatal - True if caller should exit, Or False
+ _GetAllProjectsCommitIds returns CommitIdsResult a NamedTuple of
++ commit_ids - a dictionary with the projects/commit ids on success,
otherwise None
++ fatal - True if caller should exit, Or False
+ Added _SkipUpdatingProjectRevisionId a helper function to see if a
project's revision id needs to be updated or not. This function is
used to exclude projects from local manifest file.
+ Added the following error events into git_trace2_event_log
++ If superproject is missing in a manifest
++ If there are missing commit ids for projects.
command.py:
+ Deleted unused import - platform
+ Added git_trace2_event_log as a member so all subcmds can log error
events.
main.py:
+ Initialized git_trace2_event_log as a member of command object.
init.py:
+ Deleted unused import - optparse
init.py:
+ Called sys.exit only if Sync returns exit=True
sync.py:
+ Called sys.exit only if Superproject's UpdateProjectsRevisionId returns
exit=True
+ Reloaded the manifest only if manifest path is returned by
UpdateProjectsRevisionId. If not, fall back to the old way of doing
repo sync.
test_git_superproject:
+ Added code to verify error events are being logged.
+ Added a test for no superproject tag
+ Added test for UpdateProjectsRevisionId not updating the revision id
with the commit ids.
Tested the code with the following commands.
+ Positive test case with aosp-master.
$ repo_dev init -u persistent-https://android.git.corp.google.com/platform/manifest -b master --use-superproject
NOTICE: --use-superproject is in beta; report any issues to the address described in `repo version`
.../android/aosp/.repo/exp-superproject/925043f706ba64db713e9bf3b55987e2-superproject.git: Initial setup for superproject completed.
Your identity is: Raman Tenneti <rtenneti@google.com>
If you want to change this, please re-run 'repo init' with --config-name
repo has been initialized in .../android/aosp
$ repo_dev sync -j40 --use-superproject
remote: Total 12 (delta 4), reused 12 (delta 4)
NOTICE: --use-superproject is in beta; report any issues to the address described in `repo version`
.../android/aosp/.repo/exp-superproject/925043f706ba64db713e9bf3b55987e2-superproject.git: Initial setup for superproject completed.
...
repo sync has finished successfully.
+ Negative test case without superproject tag.
$ repo_dev sync -j40 --use-superproject
NOTICE: --use-superproject is in beta; report any issues to the address described in `repo version`
repo error: superproject tag is not defined in manifest: .../android/aosp/.repo/manifest.xml
error: Cannot get project commit ids from manifest
error: Update of revsionId from superproject has failed. Please resync with --no-use-superproject option
...
Checking out: 100% (1022/1022), done in 3.589s
repo sync has finished successfully.
+ Test for missing commit_id for a project.
$ repo_dev sync -j40 --use-superproject
NOTICE: --use-superproject is in beta; report any issues to the address described in `repo version`
.../android/aosp/.repo/exp-superproject/925043f706ba64db713e9bf3b55987e2-superproject.git: Initial setup for superproject completed.
error: please file a bug using go/repo-bug to report missing commit_ids for: ['build/blueprint']
error: Update of revsionId from superproject has failed. Please resync with --no-use-superproject option
...
Checking out: 100% (1022/1022), done in 3.364s
repo sync has finished successfully.
$ ./run_tests -v
...
...== 164 passed in 2.87s ==...
Bug: [google internal] b/189371541
Change-Id: I5ea49f87e8fa41be590fc0c914573e16c8cdfcfa
Reviewed-on: https://gerrit-review.googlesource.com/c/git-repo/+/309162
Tested-by: Raman Tenneti <rtenneti@google.com>
Reviewed-by: Mike Frysinger <vapier@google.com>
| -rw-r--r-- | command.py | 5 | ||||
| -rw-r--r-- | git_superproject.py | 118 | ||||
| -rwxr-xr-x | main.py | 3 | ||||
| -rw-r--r-- | subcmds/init.py | 8 | ||||
| -rw-r--r-- | subcmds/sync.py | 18 | ||||
| -rw-r--r-- | tests/test_git_superproject.py | 134 |
6 files changed, 225 insertions, 61 deletions
| @@ -15,7 +15,6 @@ | |||
| 15 | import multiprocessing | 15 | import multiprocessing |
| 16 | import os | 16 | import os |
| 17 | import optparse | 17 | import optparse |
| 18 | import platform | ||
| 19 | import re | 18 | import re |
| 20 | import sys | 19 | import sys |
| 21 | 20 | ||
| @@ -58,11 +57,13 @@ class Command(object): | |||
| 58 | # it is the number of parallel jobs to default to. | 57 | # it is the number of parallel jobs to default to. |
| 59 | PARALLEL_JOBS = None | 58 | PARALLEL_JOBS = None |
| 60 | 59 | ||
| 61 | def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None): | 60 | def __init__(self, repodir=None, client=None, manifest=None, gitc_manifest=None, |
| 61 | git_event_log=None): | ||
| 62 | self.repodir = repodir | 62 | self.repodir = repodir |
| 63 | self.client = client | 63 | self.client = client |
| 64 | self.manifest = manifest | 64 | self.manifest = manifest |
| 65 | self.gitc_manifest = gitc_manifest | 65 | self.gitc_manifest = gitc_manifest |
| 66 | self.git_event_log = git_event_log | ||
| 66 | 67 | ||
| 67 | # Cache for the OptionParser property. | 68 | # Cache for the OptionParser property. |
| 68 | self._optparse = None | 69 | self._optparse = None |
diff --git a/git_superproject.py b/git_superproject.py index 3c4144dd..8f1e04d6 100644 --- a/git_superproject.py +++ b/git_superproject.py | |||
| @@ -19,12 +19,13 @@ https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects | |||
| 19 | 19 | ||
| 20 | Examples: | 20 | Examples: |
| 21 | superproject = Superproject() | 21 | superproject = Superproject() |
| 22 | project_commit_ids = superproject.UpdateProjectsRevisionId(projects) | 22 | UpdateProjectsResult = superproject.UpdateProjectsRevisionId(projects) |
| 23 | """ | 23 | """ |
| 24 | 24 | ||
| 25 | import hashlib | 25 | import hashlib |
| 26 | import os | 26 | import os |
| 27 | import sys | 27 | import sys |
| 28 | from typing import NamedTuple | ||
| 28 | 29 | ||
| 29 | from git_command import git_require, GitCommand | 30 | from git_command import git_require, GitCommand |
| 30 | from git_refs import R_HEADS | 31 | from git_refs import R_HEADS |
| @@ -34,6 +35,33 @@ _SUPERPROJECT_GIT_NAME = 'superproject.git' | |||
| 34 | _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml' | 35 | _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml' |
| 35 | 36 | ||
| 36 | 37 | ||
| 38 | class SyncResult(NamedTuple): | ||
| 39 | """Return the status of sync and whether caller should exit.""" | ||
| 40 | |||
| 41 | # Whether the superproject sync was successful. | ||
| 42 | success: bool | ||
| 43 | # Whether the caller should exit. | ||
| 44 | fatal: bool | ||
| 45 | |||
| 46 | |||
| 47 | class CommitIdsResult(NamedTuple): | ||
| 48 | """Return the commit ids and whether caller should exit.""" | ||
| 49 | |||
| 50 | # A dictionary with the projects/commit ids on success, otherwise None. | ||
| 51 | commit_ids: dict | ||
| 52 | # Whether the caller should exit. | ||
| 53 | fatal: bool | ||
| 54 | |||
| 55 | |||
| 56 | class UpdateProjectsResult(NamedTuple): | ||
| 57 | """Return the overriding manifest file and whether caller should exit.""" | ||
| 58 | |||
| 59 | # Path name of the overriding manfiest file if successful, otherwise None. | ||
| 60 | manifest_path: str | ||
| 61 | # Whether the caller should exit. | ||
| 62 | fatal: bool | ||
| 63 | |||
| 64 | |||
| 37 | class Superproject(object): | 65 | class Superproject(object): |
| 38 | """Get commit ids from superproject. | 66 | """Get commit ids from superproject. |
| 39 | 67 | ||
| @@ -41,19 +69,21 @@ class Superproject(object): | |||
| 41 | lookup of commit ids for all projects. It contains _project_commit_ids which | 69 | lookup of commit ids for all projects. It contains _project_commit_ids which |
| 42 | is a dictionary with project/commit id entries. | 70 | is a dictionary with project/commit id entries. |
| 43 | """ | 71 | """ |
| 44 | def __init__(self, manifest, repodir, superproject_dir='exp-superproject', | 72 | def __init__(self, manifest, repodir, git_event_log, |
| 45 | quiet=False): | 73 | superproject_dir='exp-superproject', quiet=False): |
| 46 | """Initializes superproject. | 74 | """Initializes superproject. |
| 47 | 75 | ||
| 48 | Args: | 76 | Args: |
| 49 | manifest: A Manifest object that is to be written to a file. | 77 | manifest: A Manifest object that is to be written to a file. |
| 50 | repodir: Path to the .repo/ dir for holding all internal checkout state. | 78 | repodir: Path to the .repo/ dir for holding all internal checkout state. |
| 51 | It must be in the top directory of the repo client checkout. | 79 | It must be in the top directory of the repo client checkout. |
| 80 | git_event_log: A git trace2 event log to log events. | ||
| 52 | superproject_dir: Relative path under |repodir| to checkout superproject. | 81 | superproject_dir: Relative path under |repodir| to checkout superproject. |
| 53 | quiet: If True then only print the progress messages. | 82 | quiet: If True then only print the progress messages. |
| 54 | """ | 83 | """ |
| 55 | self._project_commit_ids = None | 84 | self._project_commit_ids = None |
| 56 | self._manifest = manifest | 85 | self._manifest = manifest |
| 86 | self._git_event_log = git_event_log | ||
| 57 | self._quiet = quiet | 87 | self._quiet = quiet |
| 58 | self._branch = self._GetBranch() | 88 | self._branch = self._GetBranch() |
| 59 | self._repodir = os.path.abspath(repodir) | 89 | self._repodir = os.path.abspath(repodir) |
| @@ -172,44 +202,48 @@ class Superproject(object): | |||
| 172 | """Gets a local copy of a superproject for the manifest. | 202 | """Gets a local copy of a superproject for the manifest. |
| 173 | 203 | ||
| 174 | Returns: | 204 | Returns: |
| 175 | True if sync of superproject is successful, or False. | 205 | SyncResult |
| 176 | """ | 206 | """ |
| 177 | print('NOTICE: --use-superproject is in beta; report any issues to the ' | 207 | print('NOTICE: --use-superproject is in beta; report any issues to the ' |
| 178 | 'address described in `repo version`', file=sys.stderr) | 208 | 'address described in `repo version`', file=sys.stderr) |
| 179 | 209 | ||
| 180 | if not self._manifest.superproject: | 210 | if not self._manifest.superproject: |
| 181 | print('error: superproject tag is not defined in manifest', | 211 | msg = (f'repo error: superproject tag is not defined in manifest: ' |
| 182 | file=sys.stderr) | 212 | f'{self._manifest.manifestFile}') |
| 183 | return False | 213 | print(msg, file=sys.stderr) |
| 214 | self._git_event_log.ErrorEvent(msg, '') | ||
| 215 | return SyncResult(False, False) | ||
| 184 | 216 | ||
| 217 | should_exit = True | ||
| 185 | url = self._manifest.superproject['remote'].url | 218 | url = self._manifest.superproject['remote'].url |
| 186 | if not url: | 219 | if not url: |
| 187 | print('error: superproject URL is not defined in manifest', | 220 | print('error: superproject URL is not defined in manifest', |
| 188 | file=sys.stderr) | 221 | file=sys.stderr) |
| 189 | return False | 222 | return SyncResult(False, should_exit) |
| 190 | 223 | ||
| 191 | if not self._Init(): | 224 | if not self._Init(): |
| 192 | return False | 225 | return SyncResult(False, should_exit) |
| 193 | if not self._Fetch(url): | 226 | if not self._Fetch(url): |
| 194 | return False | 227 | return SyncResult(False, should_exit) |
| 195 | if not self._quiet: | 228 | if not self._quiet: |
| 196 | print('%s: Initial setup for superproject completed.' % self._work_git) | 229 | print('%s: Initial setup for superproject completed.' % self._work_git) |
| 197 | return True | 230 | return SyncResult(True, False) |
| 198 | 231 | ||
| 199 | def _GetAllProjectsCommitIds(self): | 232 | def _GetAllProjectsCommitIds(self): |
| 200 | """Get commit ids for all projects from superproject and save them in _project_commit_ids. | 233 | """Get commit ids for all projects from superproject and save them in _project_commit_ids. |
| 201 | 234 | ||
| 202 | Returns: | 235 | Returns: |
| 203 | A dictionary with the projects/commit ids on success, otherwise None. | 236 | CommitIdsResult |
| 204 | """ | 237 | """ |
| 205 | if not self.Sync(): | 238 | sync_result = self.Sync() |
| 206 | return None | 239 | if not sync_result.success: |
| 240 | return CommitIdsResult(None, sync_result.fatal) | ||
| 207 | 241 | ||
| 208 | data = self._LsTree() | 242 | data = self._LsTree() |
| 209 | if not data: | 243 | if not data: |
| 210 | print('error: git ls-tree failed to return data for superproject', | 244 | print('error: git ls-tree failed to return data for superproject', |
| 211 | file=sys.stderr) | 245 | file=sys.stderr) |
| 212 | return None | 246 | return CommitIdsResult(None, True) |
| 213 | 247 | ||
| 214 | # Parse lines like the following to select lines starting with '160000' and | 248 | # Parse lines like the following to select lines starting with '160000' and |
| 215 | # build a dictionary with project path (last element) and its commit id (3rd element). | 249 | # build a dictionary with project path (last element) and its commit id (3rd element). |
| @@ -225,7 +259,7 @@ class Superproject(object): | |||
| 225 | commit_ids[ls_data[3]] = ls_data[2] | 259 | commit_ids[ls_data[3]] = ls_data[2] |
| 226 | 260 | ||
| 227 | self._project_commit_ids = commit_ids | 261 | self._project_commit_ids = commit_ids |
| 228 | return commit_ids | 262 | return CommitIdsResult(commit_ids, False) |
| 229 | 263 | ||
| 230 | def _WriteManfiestFile(self): | 264 | def _WriteManfiestFile(self): |
| 231 | """Writes manifest to a file. | 265 | """Writes manifest to a file. |
| @@ -250,6 +284,23 @@ class Superproject(object): | |||
| 250 | return None | 284 | return None |
| 251 | return manifest_path | 285 | return manifest_path |
| 252 | 286 | ||
| 287 | def _SkipUpdatingProjectRevisionId(self, project): | ||
| 288 | """Checks if a project's revision id needs to be updated or not. | ||
| 289 | |||
| 290 | Revision id for projects from local manifest will not be updated. | ||
| 291 | |||
| 292 | Args: | ||
| 293 | project: project whose revision id is being updated. | ||
| 294 | |||
| 295 | Returns: | ||
| 296 | True if a project's revision id should not be updated, or False, | ||
| 297 | """ | ||
| 298 | path = project.relpath | ||
| 299 | if not path: | ||
| 300 | return True | ||
| 301 | # Skip the project if it comes from the local manifest. | ||
| 302 | return any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups) | ||
| 303 | |||
| 253 | def UpdateProjectsRevisionId(self, projects): | 304 | def UpdateProjectsRevisionId(self, projects): |
| 254 | """Update revisionId of every project in projects with the commit id. | 305 | """Update revisionId of every project in projects with the commit id. |
| 255 | 306 | ||
| @@ -257,30 +308,35 @@ class Superproject(object): | |||
| 257 | projects: List of projects whose revisionId needs to be updated. | 308 | projects: List of projects whose revisionId needs to be updated. |
| 258 | 309 | ||
| 259 | Returns: | 310 | Returns: |
| 260 | manifest_path: Path name of the overriding manfiest file instead of None. | 311 | UpdateProjectsResult |
| 261 | """ | 312 | """ |
| 262 | commit_ids = self._GetAllProjectsCommitIds() | 313 | commit_ids_result = self._GetAllProjectsCommitIds() |
| 314 | commit_ids = commit_ids_result.commit_ids | ||
| 263 | if not commit_ids: | 315 | if not commit_ids: |
| 264 | print('error: Cannot get project commit ids from manifest', file=sys.stderr) | 316 | print('error: Cannot get project commit ids from manifest', file=sys.stderr) |
| 265 | return None | 317 | return UpdateProjectsResult(None, commit_ids_result.fatal) |
| 266 | 318 | ||
| 267 | projects_missing_commit_ids = [] | 319 | projects_missing_commit_ids = [] |
| 268 | for project in projects: | 320 | for project in projects: |
| 269 | path = project.relpath | 321 | if self._SkipUpdatingProjectRevisionId(project): |
| 270 | if not path: | ||
| 271 | continue | ||
| 272 | # Skip the project if it comes from local manifest. | ||
| 273 | if any(s.startswith(LOCAL_MANIFEST_GROUP_PREFIX) for s in project.groups): | ||
| 274 | continue | 322 | continue |
| 323 | path = project.relpath | ||
| 275 | commit_id = commit_ids.get(path) | 324 | commit_id = commit_ids.get(path) |
| 276 | if commit_id: | 325 | if not commit_id: |
| 277 | project.SetRevisionId(commit_id) | ||
| 278 | else: | ||
| 279 | projects_missing_commit_ids.append(path) | 326 | projects_missing_commit_ids.append(path) |
| 327 | |||
| 328 | # If superproject doesn't have a commit id for a project, then report an | ||
| 329 | # error event and continue as if do not use superproject is specified. | ||
| 280 | if projects_missing_commit_ids: | 330 | if projects_missing_commit_ids: |
| 281 | print('error: please file a bug using %s to report missing commit_ids for: %s' % | 331 | msg = (f'error: please file a bug using {self._manifest.contactinfo.bugurl} ' |
| 282 | (self._manifest.contactinfo.bugurl, projects_missing_commit_ids), file=sys.stderr) | 332 | f'to report missing commit_ids for: {projects_missing_commit_ids}') |
| 283 | return None | 333 | print(msg, file=sys.stderr) |
| 334 | self._git_event_log.ErrorEvent(msg, '') | ||
| 335 | return UpdateProjectsResult(None, False) | ||
| 336 | |||
| 337 | for project in projects: | ||
| 338 | if not self._SkipUpdatingProjectRevisionId(project): | ||
| 339 | project.SetRevisionId(commit_ids.get(project.relpath)) | ||
| 284 | 340 | ||
| 285 | manifest_path = self._WriteManfiestFile() | 341 | manifest_path = self._WriteManfiestFile() |
| 286 | return manifest_path | 342 | return UpdateProjectsResult(manifest_path, False) |
| @@ -208,7 +208,8 @@ class _Repo(object): | |||
| 208 | repodir=self.repodir, | 208 | repodir=self.repodir, |
| 209 | client=repo_client, | 209 | client=repo_client, |
| 210 | manifest=repo_client.manifest, | 210 | manifest=repo_client.manifest, |
| 211 | gitc_manifest=gitc_manifest) | 211 | gitc_manifest=gitc_manifest, |
| 212 | git_event_log=git_trace2_event_log) | ||
| 212 | except KeyError: | 213 | except KeyError: |
| 213 | print("repo: '%s' is not a repo command. See 'repo help'." % name, | 214 | print("repo: '%s' is not a repo command. See 'repo help'." % name, |
| 214 | file=sys.stderr) | 215 | file=sys.stderr) |
diff --git a/subcmds/init.py b/subcmds/init.py index 750facba..536e367c 100644 --- a/subcmds/init.py +++ b/subcmds/init.py | |||
| @@ -12,7 +12,6 @@ | |||
| 12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. | 13 | # limitations under the License. |
| 14 | 14 | ||
| 15 | import optparse | ||
| 16 | import os | 15 | import os |
| 17 | import platform | 16 | import platform |
| 18 | import re | 17 | import re |
| @@ -97,10 +96,13 @@ to update the working directory files. | |||
| 97 | """ | 96 | """ |
| 98 | superproject = git_superproject.Superproject(self.manifest, | 97 | superproject = git_superproject.Superproject(self.manifest, |
| 99 | self.repodir, | 98 | self.repodir, |
| 99 | self.git_event_log, | ||
| 100 | quiet=opt.quiet) | 100 | quiet=opt.quiet) |
| 101 | if not superproject.Sync(): | 101 | sync_result = superproject.Sync() |
| 102 | if not sync_result.success: | ||
| 102 | print('error: git update of superproject failed', file=sys.stderr) | 103 | print('error: git update of superproject failed', file=sys.stderr) |
| 103 | sys.exit(1) | 104 | if sync_result.fatal: |
| 105 | sys.exit(1) | ||
| 104 | 106 | ||
| 105 | def _SyncManifest(self, opt): | 107 | def _SyncManifest(self, opt): |
| 106 | m = self.manifest.manifestProject | 108 | m = self.manifest.manifestProject |
diff --git a/subcmds/sync.py b/subcmds/sync.py index b15d9477..8d89cf72 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
| @@ -302,21 +302,25 @@ later is required to fix a server side protocol bug. | |||
| 302 | load_local_manifests: Whether to load local manifests. | 302 | load_local_manifests: Whether to load local manifests. |
| 303 | 303 | ||
| 304 | Returns: | 304 | Returns: |
| 305 | Returns path to the overriding manifest file. | 305 | Returns path to the overriding manifest file instead of None. |
| 306 | """ | 306 | """ |
| 307 | superproject = git_superproject.Superproject(self.manifest, | 307 | superproject = git_superproject.Superproject(self.manifest, |
| 308 | self.repodir, | 308 | self.repodir, |
| 309 | self.git_event_log, | ||
| 309 | quiet=opt.quiet) | 310 | quiet=opt.quiet) |
| 310 | all_projects = self.GetProjects(args, | 311 | all_projects = self.GetProjects(args, |
| 311 | missing_ok=True, | 312 | missing_ok=True, |
| 312 | submodules_ok=opt.fetch_submodules) | 313 | submodules_ok=opt.fetch_submodules) |
| 313 | manifest_path = superproject.UpdateProjectsRevisionId(all_projects) | 314 | update_result = superproject.UpdateProjectsRevisionId(all_projects) |
| 314 | if not manifest_path: | 315 | manifest_path = update_result.manifest_path |
| 316 | if manifest_path: | ||
| 317 | self._ReloadManifest(manifest_path, load_local_manifests) | ||
| 318 | else: | ||
| 315 | print('error: Update of revsionId from superproject has failed. ' | 319 | print('error: Update of revsionId from superproject has failed. ' |
| 316 | 'Please resync with --no-use-superproject option', | 320 | 'Please resync with --no-use-superproject option', |
| 317 | file=sys.stderr) | 321 | file=sys.stderr) |
| 318 | sys.exit(1) | 322 | if update_result.fatal: |
| 319 | self._ReloadManifest(manifest_path, load_local_manifests) | 323 | sys.exit(1) |
| 320 | return manifest_path | 324 | return manifest_path |
| 321 | 325 | ||
| 322 | def _FetchProjectList(self, opt, projects): | 326 | def _FetchProjectList(self, opt, projects): |
| @@ -961,7 +965,9 @@ later is required to fix a server side protocol bug. | |||
| 961 | 965 | ||
| 962 | load_local_manifests = not self.manifest.HasLocalManifests | 966 | load_local_manifests = not self.manifest.HasLocalManifests |
| 963 | if self._UseSuperproject(opt): | 967 | if self._UseSuperproject(opt): |
| 964 | manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests) | 968 | new_manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests) |
| 969 | if not new_manifest_name: | ||
| 970 | manifest_name = new_manifest_name | ||
| 965 | 971 | ||
| 966 | if self.gitc_manifest: | 972 | if self.gitc_manifest: |
| 967 | gitc_manifest_projects = self.GetProjects(args, | 973 | gitc_manifest_projects = self.GetProjects(args, |
diff --git a/tests/test_git_superproject.py b/tests/test_git_superproject.py index ba61a3d1..d612f4e7 100644 --- a/tests/test_git_superproject.py +++ b/tests/test_git_superproject.py | |||
| @@ -14,6 +14,7 @@ | |||
| 14 | 14 | ||
| 15 | """Unittests for the git_superproject.py module.""" | 15 | """Unittests for the git_superproject.py module.""" |
| 16 | 16 | ||
| 17 | import json | ||
| 17 | import os | 18 | import os |
| 18 | import platform | 19 | import platform |
| 19 | import tempfile | 20 | import tempfile |
| @@ -21,6 +22,7 @@ import unittest | |||
| 21 | from unittest import mock | 22 | from unittest import mock |
| 22 | 23 | ||
| 23 | import git_superproject | 24 | import git_superproject |
| 25 | import git_trace2_event_log | ||
| 24 | import manifest_xml | 26 | import manifest_xml |
| 25 | import platform_utils | 27 | import platform_utils |
| 26 | from test_manifest_xml import sort_attributes | 28 | from test_manifest_xml import sort_attributes |
| @@ -29,6 +31,11 @@ from test_manifest_xml import sort_attributes | |||
| 29 | class SuperprojectTestCase(unittest.TestCase): | 31 | class SuperprojectTestCase(unittest.TestCase): |
| 30 | """TestCase for the Superproject module.""" | 32 | """TestCase for the Superproject module.""" |
| 31 | 33 | ||
| 34 | PARENT_SID_KEY = 'GIT_TRACE2_PARENT_SID' | ||
| 35 | PARENT_SID_VALUE = 'parent_sid' | ||
| 36 | SELF_SID_REGEX = r'repo-\d+T\d+Z-.*' | ||
| 37 | FULL_SID_REGEX = r'^%s/%s' % (PARENT_SID_VALUE, SELF_SID_REGEX) | ||
| 38 | |||
| 32 | def setUp(self): | 39 | def setUp(self): |
| 33 | """Set up superproject every time.""" | 40 | """Set up superproject every time.""" |
| 34 | self.tempdir = tempfile.mkdtemp(prefix='repo_tests') | 41 | self.tempdir = tempfile.mkdtemp(prefix='repo_tests') |
| @@ -38,6 +45,13 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 38 | os.mkdir(self.repodir) | 45 | os.mkdir(self.repodir) |
| 39 | self.platform = platform.system().lower() | 46 | self.platform = platform.system().lower() |
| 40 | 47 | ||
| 48 | # By default we initialize with the expected case where | ||
| 49 | # repo launches us (so GIT_TRACE2_PARENT_SID is set). | ||
| 50 | env = { | ||
| 51 | self.PARENT_SID_KEY: self.PARENT_SID_VALUE, | ||
| 52 | } | ||
| 53 | self.git_event_log = git_trace2_event_log.EventLog(env=env) | ||
| 54 | |||
| 41 | # The manifest parsing really wants a git repo currently. | 55 | # The manifest parsing really wants a git repo currently. |
| 42 | gitdir = os.path.join(self.repodir, 'manifests.git') | 56 | gitdir = os.path.join(self.repodir, 'manifests.git') |
| 43 | os.mkdir(gitdir) | 57 | os.mkdir(gitdir) |
| @@ -54,7 +68,8 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 54 | <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """ | 68 | <project path="art" name="platform/art" groups="notdefault,platform-""" + self.platform + """ |
| 55 | " /></manifest> | 69 | " /></manifest> |
| 56 | """) | 70 | """) |
| 57 | self._superproject = git_superproject.Superproject(manifest, self.repodir) | 71 | self._superproject = git_superproject.Superproject(manifest, self.repodir, |
| 72 | self.git_event_log) | ||
| 58 | 73 | ||
| 59 | def tearDown(self): | 74 | def tearDown(self): |
| 60 | """Tear down superproject every time.""" | 75 | """Tear down superproject every time.""" |
| @@ -66,14 +81,56 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 66 | fp.write(data) | 81 | fp.write(data) |
| 67 | return manifest_xml.XmlManifest(self.repodir, self.manifest_file) | 82 | return manifest_xml.XmlManifest(self.repodir, self.manifest_file) |
| 68 | 83 | ||
| 84 | def verifyCommonKeys(self, log_entry, expected_event_name, full_sid=True): | ||
| 85 | """Helper function to verify common event log keys.""" | ||
| 86 | self.assertIn('event', log_entry) | ||
| 87 | self.assertIn('sid', log_entry) | ||
| 88 | self.assertIn('thread', log_entry) | ||
| 89 | self.assertIn('time', log_entry) | ||
| 90 | |||
| 91 | # Do basic data format validation. | ||
| 92 | self.assertEqual(expected_event_name, log_entry['event']) | ||
| 93 | if full_sid: | ||
| 94 | self.assertRegex(log_entry['sid'], self.FULL_SID_REGEX) | ||
| 95 | else: | ||
| 96 | self.assertRegex(log_entry['sid'], self.SELF_SID_REGEX) | ||
| 97 | self.assertRegex(log_entry['time'], r'^\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z$') | ||
| 98 | |||
| 99 | def readLog(self, log_path): | ||
| 100 | """Helper function to read log data into a list.""" | ||
| 101 | log_data = [] | ||
| 102 | with open(log_path, mode='rb') as f: | ||
| 103 | for line in f: | ||
| 104 | log_data.append(json.loads(line)) | ||
| 105 | return log_data | ||
| 106 | |||
| 107 | def verifyErrorEvent(self): | ||
| 108 | """Helper to verify that error event is written.""" | ||
| 109 | |||
| 110 | with tempfile.TemporaryDirectory(prefix='event_log_tests') as tempdir: | ||
| 111 | log_path = self.git_event_log.Write(path=tempdir) | ||
| 112 | self.log_data = self.readLog(log_path) | ||
| 113 | |||
| 114 | self.assertEqual(len(self.log_data), 2) | ||
| 115 | error_event = self.log_data[1] | ||
| 116 | self.verifyCommonKeys(self.log_data[0], expected_event_name='version') | ||
| 117 | self.verifyCommonKeys(error_event, expected_event_name='error') | ||
| 118 | # Check for 'error' event specific fields. | ||
| 119 | self.assertIn('msg', error_event) | ||
| 120 | self.assertIn('fmt', error_event) | ||
| 121 | |||
| 69 | def test_superproject_get_superproject_no_superproject(self): | 122 | def test_superproject_get_superproject_no_superproject(self): |
| 70 | """Test with no url.""" | 123 | """Test with no url.""" |
| 71 | manifest = self.getXmlManifest(""" | 124 | manifest = self.getXmlManifest(""" |
| 72 | <manifest> | 125 | <manifest> |
| 73 | </manifest> | 126 | </manifest> |
| 74 | """) | 127 | """) |
| 75 | superproject = git_superproject.Superproject(manifest, self.repodir) | 128 | superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log) |
| 76 | self.assertFalse(superproject.Sync()) | 129 | # Test that exit condition is false when there is no superproject tag. |
| 130 | sync_result = superproject.Sync() | ||
| 131 | self.assertFalse(sync_result.success) | ||
| 132 | self.assertFalse(sync_result.fatal) | ||
| 133 | self.verifyErrorEvent() | ||
| 77 | 134 | ||
| 78 | def test_superproject_get_superproject_invalid_url(self): | 135 | def test_superproject_get_superproject_invalid_url(self): |
| 79 | """Test with an invalid url.""" | 136 | """Test with an invalid url.""" |
| @@ -84,8 +141,10 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 84 | <superproject name="superproject"/> | 141 | <superproject name="superproject"/> |
| 85 | </manifest> | 142 | </manifest> |
| 86 | """) | 143 | """) |
| 87 | superproject = git_superproject.Superproject(manifest, self.repodir) | 144 | superproject = git_superproject.Superproject(manifest, self.repodir, self.git_event_log) |
| 88 | self.assertFalse(superproject.Sync()) | 145 | sync_result = superproject.Sync() |
| 146 | self.assertFalse(sync_result.success) | ||
| 147 | self.assertTrue(sync_result.fatal) | ||
| 89 | 148 | ||
| 90 | def test_superproject_get_superproject_invalid_branch(self): | 149 | def test_superproject_get_superproject_invalid_branch(self): |
| 91 | """Test with an invalid branch.""" | 150 | """Test with an invalid branch.""" |
| @@ -96,21 +155,28 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 96 | <superproject name="superproject"/> | 155 | <superproject name="superproject"/> |
| 97 | </manifest> | 156 | </manifest> |
| 98 | """) | 157 | """) |
| 99 | superproject = git_superproject.Superproject(manifest, self.repodir) | 158 | self._superproject = git_superproject.Superproject(manifest, self.repodir, |
| 159 | self.git_event_log) | ||
| 100 | with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'): | 160 | with mock.patch.object(self._superproject, '_GetBranch', return_value='junk'): |
| 101 | self.assertFalse(superproject.Sync()) | 161 | sync_result = self._superproject.Sync() |
| 162 | self.assertFalse(sync_result.success) | ||
| 163 | self.assertTrue(sync_result.fatal) | ||
| 102 | 164 | ||
| 103 | def test_superproject_get_superproject_mock_init(self): | 165 | def test_superproject_get_superproject_mock_init(self): |
| 104 | """Test with _Init failing.""" | 166 | """Test with _Init failing.""" |
| 105 | with mock.patch.object(self._superproject, '_Init', return_value=False): | 167 | with mock.patch.object(self._superproject, '_Init', return_value=False): |
| 106 | self.assertFalse(self._superproject.Sync()) | 168 | sync_result = self._superproject.Sync() |
| 169 | self.assertFalse(sync_result.success) | ||
| 170 | self.assertTrue(sync_result.fatal) | ||
| 107 | 171 | ||
| 108 | def test_superproject_get_superproject_mock_fetch(self): | 172 | def test_superproject_get_superproject_mock_fetch(self): |
| 109 | """Test with _Fetch failing.""" | 173 | """Test with _Fetch failing.""" |
| 110 | with mock.patch.object(self._superproject, '_Init', return_value=True): | 174 | with mock.patch.object(self._superproject, '_Init', return_value=True): |
| 111 | os.mkdir(self._superproject._superproject_path) | 175 | os.mkdir(self._superproject._superproject_path) |
| 112 | with mock.patch.object(self._superproject, '_Fetch', return_value=False): | 176 | with mock.patch.object(self._superproject, '_Fetch', return_value=False): |
| 113 | self.assertFalse(self._superproject.Sync()) | 177 | sync_result = self._superproject.Sync() |
| 178 | self.assertFalse(sync_result.success) | ||
| 179 | self.assertTrue(sync_result.fatal) | ||
| 114 | 180 | ||
| 115 | def test_superproject_get_all_project_commit_ids_mock_ls_tree(self): | 181 | def test_superproject_get_all_project_commit_ids_mock_ls_tree(self): |
| 116 | """Test with LsTree being a mock.""" | 182 | """Test with LsTree being a mock.""" |
| @@ -122,12 +188,13 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 122 | with mock.patch.object(self._superproject, '_Init', return_value=True): | 188 | with mock.patch.object(self._superproject, '_Init', return_value=True): |
| 123 | with mock.patch.object(self._superproject, '_Fetch', return_value=True): | 189 | with mock.patch.object(self._superproject, '_Fetch', return_value=True): |
| 124 | with mock.patch.object(self._superproject, '_LsTree', return_value=data): | 190 | with mock.patch.object(self._superproject, '_LsTree', return_value=data): |
| 125 | commit_ids = self._superproject._GetAllProjectsCommitIds() | 191 | commit_ids_result = self._superproject._GetAllProjectsCommitIds() |
| 126 | self.assertEqual(commit_ids, { | 192 | self.assertEqual(commit_ids_result.commit_ids, { |
| 127 | 'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea', | 193 | 'art': '2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea', |
| 128 | 'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06', | 194 | 'bootable/recovery': 'e9d25da64d8d365dbba7c8ee00fe8c4473fe9a06', |
| 129 | 'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928' | 195 | 'build/bazel': 'ade9b7a0d874e25fff4bf2552488825c6f111928' |
| 130 | }) | 196 | }) |
| 197 | self.assertFalse(commit_ids_result.fatal) | ||
| 131 | 198 | ||
| 132 | def test_superproject_write_manifest_file(self): | 199 | def test_superproject_write_manifest_file(self): |
| 133 | """Test with writing manifest to a file after setting revisionId.""" | 200 | """Test with writing manifest to a file after setting revisionId.""" |
| @@ -163,9 +230,10 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 163 | return_value=data): | 230 | return_value=data): |
| 164 | # Create temporary directory so that it can write the file. | 231 | # Create temporary directory so that it can write the file. |
| 165 | os.mkdir(self._superproject._superproject_path) | 232 | os.mkdir(self._superproject._superproject_path) |
| 166 | manifest_path = self._superproject.UpdateProjectsRevisionId(projects) | 233 | update_result = self._superproject.UpdateProjectsRevisionId(projects) |
| 167 | self.assertIsNotNone(manifest_path) | 234 | self.assertIsNotNone(update_result.manifest_path) |
| 168 | with open(manifest_path, 'r') as fp: | 235 | self.assertFalse(update_result.fatal) |
| 236 | with open(update_result.manifest_path, 'r') as fp: | ||
| 169 | manifest_xml_data = fp.read() | 237 | manifest_xml_data = fp.read() |
| 170 | self.assertEqual( | 238 | self.assertEqual( |
| 171 | sort_attributes(manifest_xml_data), | 239 | sort_attributes(manifest_xml_data), |
| @@ -178,6 +246,34 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 178 | '<superproject name="superproject"/>' | 246 | '<superproject name="superproject"/>' |
| 179 | '</manifest>') | 247 | '</manifest>') |
| 180 | 248 | ||
| 249 | def test_superproject_update_project_revision_id_no_superproject_tag(self): | ||
| 250 | """Test update of commit ids of a manifest without superproject tag.""" | ||
| 251 | manifest = self.getXmlManifest(""" | ||
| 252 | <manifest> | ||
| 253 | <remote name="default-remote" fetch="http://localhost" /> | ||
| 254 | <default remote="default-remote" revision="refs/heads/main" /> | ||
| 255 | <project name="test-name"/> | ||
| 256 | </manifest> | ||
| 257 | """) | ||
| 258 | self.maxDiff = None | ||
| 259 | self._superproject = git_superproject.Superproject(manifest, self.repodir, | ||
| 260 | self.git_event_log) | ||
| 261 | self.assertEqual(len(self._superproject._manifest.projects), 1) | ||
| 262 | projects = self._superproject._manifest.projects | ||
| 263 | project = projects[0] | ||
| 264 | project.SetRevisionId('ABCDEF') | ||
| 265 | update_result = self._superproject.UpdateProjectsRevisionId(projects) | ||
| 266 | self.assertIsNone(update_result.manifest_path) | ||
| 267 | self.assertFalse(update_result.fatal) | ||
| 268 | self.verifyErrorEvent() | ||
| 269 | self.assertEqual( | ||
| 270 | sort_attributes(manifest.ToXml().toxml()), | ||
| 271 | '<?xml version="1.0" ?><manifest>' | ||
| 272 | '<remote fetch="http://localhost" name="default-remote"/>' | ||
| 273 | '<default remote="default-remote" revision="refs/heads/main"/>' | ||
| 274 | '<project name="test-name" revision="ABCDEF"/>' | ||
| 275 | '</manifest>') | ||
| 276 | |||
| 181 | def test_superproject_update_project_revision_id_from_local_manifest_group(self): | 277 | def test_superproject_update_project_revision_id_from_local_manifest_group(self): |
| 182 | """Test update of commit ids of a manifest that have local manifest no superproject group.""" | 278 | """Test update of commit ids of a manifest that have local manifest no superproject group.""" |
| 183 | local_group = manifest_xml.LOCAL_MANIFEST_GROUP_PREFIX + ':local' | 279 | local_group = manifest_xml.LOCAL_MANIFEST_GROUP_PREFIX + ':local' |
| @@ -194,7 +290,8 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 194 | " /></manifest> | 290 | " /></manifest> |
| 195 | """) | 291 | """) |
| 196 | self.maxDiff = None | 292 | self.maxDiff = None |
| 197 | self._superproject = git_superproject.Superproject(manifest, self.repodir) | 293 | self._superproject = git_superproject.Superproject(manifest, self.repodir, |
| 294 | self.git_event_log) | ||
| 198 | self.assertEqual(len(self._superproject._manifest.projects), 2) | 295 | self.assertEqual(len(self._superproject._manifest.projects), 2) |
| 199 | projects = self._superproject._manifest.projects | 296 | projects = self._superproject._manifest.projects |
| 200 | data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00' | 297 | data = ('160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00' |
| @@ -206,9 +303,10 @@ class SuperprojectTestCase(unittest.TestCase): | |||
| 206 | return_value=data): | 303 | return_value=data): |
| 207 | # Create temporary directory so that it can write the file. | 304 | # Create temporary directory so that it can write the file. |
| 208 | os.mkdir(self._superproject._superproject_path) | 305 | os.mkdir(self._superproject._superproject_path) |
| 209 | manifest_path = self._superproject.UpdateProjectsRevisionId(projects) | 306 | update_result = self._superproject.UpdateProjectsRevisionId(projects) |
| 210 | self.assertIsNotNone(manifest_path) | 307 | self.assertIsNotNone(update_result.manifest_path) |
| 211 | with open(manifest_path, 'r') as fp: | 308 | self.assertFalse(update_result.fatal) |
| 309 | with open(update_result.manifest_path, 'r') as fp: | ||
| 212 | manifest_xml_data = fp.read() | 310 | manifest_xml_data = fp.read() |
| 213 | # Verify platform/vendor/x's project revision hasn't changed. | 311 | # Verify platform/vendor/x's project revision hasn't changed. |
| 214 | self.assertEqual( | 312 | self.assertEqual( |
