diff options
Diffstat (limited to 'git_superproject.py')
| -rw-r--r-- | git_superproject.py | 880 |
1 files changed, 485 insertions, 395 deletions
diff --git a/git_superproject.py b/git_superproject.py index 69a4d1fe..f1b4f231 100644 --- a/git_superproject.py +++ b/git_superproject.py | |||
| @@ -12,7 +12,7 @@ | |||
| 12 | # See the License for the specific language governing permissions and | 12 | # See the License for the specific language governing permissions and |
| 13 | # limitations under the License. | 13 | # limitations under the License. |
| 14 | 14 | ||
| 15 | """Provide functionality to get all projects and their commit ids from Superproject. | 15 | """Provide functionality to get projects and their commit ids from Superproject. |
| 16 | 16 | ||
| 17 | For more information on superproject, check out: | 17 | For more information on superproject, check out: |
| 18 | https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects | 18 | https://en.wikibooks.org/wiki/Git/Submodules_and_Superprojects |
| @@ -33,434 +33,524 @@ from git_command import git_require, GitCommand | |||
| 33 | from git_config import RepoConfig | 33 | from git_config import RepoConfig |
| 34 | from git_refs import GitRefs | 34 | from git_refs import GitRefs |
| 35 | 35 | ||
| 36 | _SUPERPROJECT_GIT_NAME = 'superproject.git' | 36 | _SUPERPROJECT_GIT_NAME = "superproject.git" |
| 37 | _SUPERPROJECT_MANIFEST_NAME = 'superproject_override.xml' | 37 | _SUPERPROJECT_MANIFEST_NAME = "superproject_override.xml" |
| 38 | 38 | ||
| 39 | 39 | ||
| 40 | class SyncResult(NamedTuple): | 40 | class SyncResult(NamedTuple): |
| 41 | """Return the status of sync and whether caller should exit.""" | 41 | """Return the status of sync and whether caller should exit.""" |
| 42 | 42 | ||
| 43 | # Whether the superproject sync was successful. | 43 | # Whether the superproject sync was successful. |
| 44 | success: bool | 44 | success: bool |
| 45 | # Whether the caller should exit. | 45 | # Whether the caller should exit. |
| 46 | fatal: bool | 46 | fatal: bool |
| 47 | 47 | ||
| 48 | 48 | ||
| 49 | class CommitIdsResult(NamedTuple): | 49 | class CommitIdsResult(NamedTuple): |
| 50 | """Return the commit ids and whether caller should exit.""" | 50 | """Return the commit ids and whether caller should exit.""" |
| 51 | 51 | ||
| 52 | # A dictionary with the projects/commit ids on success, otherwise None. | 52 | # A dictionary with the projects/commit ids on success, otherwise None. |
| 53 | commit_ids: dict | 53 | commit_ids: dict |
| 54 | # Whether the caller should exit. | 54 | # Whether the caller should exit. |
| 55 | fatal: bool | 55 | fatal: bool |
| 56 | 56 | ||
| 57 | 57 | ||
| 58 | class UpdateProjectsResult(NamedTuple): | 58 | class UpdateProjectsResult(NamedTuple): |
| 59 | """Return the overriding manifest file and whether caller should exit.""" | 59 | """Return the overriding manifest file and whether caller should exit.""" |
| 60 | 60 | ||
| 61 | # Path name of the overriding manifest file if successful, otherwise None. | 61 | # Path name of the overriding manifest file if successful, otherwise None. |
| 62 | manifest_path: str | 62 | manifest_path: str |
| 63 | # Whether the caller should exit. | 63 | # Whether the caller should exit. |
| 64 | fatal: bool | 64 | fatal: bool |
| 65 | 65 | ||
| 66 | 66 | ||
| 67 | class Superproject(object): | 67 | class Superproject(object): |
| 68 | """Get commit ids from superproject. | 68 | """Get commit ids from superproject. |
| 69 | 69 | ||
| 70 | Initializes a local copy of a superproject for the manifest. This allows | 70 | Initializes a local copy of a superproject for the manifest. This allows |
| 71 | lookup of commit ids for all projects. It contains _project_commit_ids which | 71 | lookup of commit ids for all projects. It contains _project_commit_ids which |
| 72 | is a dictionary with project/commit id entries. | 72 | is a dictionary with project/commit id entries. |
| 73 | """ | ||
| 74 | def __init__(self, manifest, name, remote, revision, | ||
| 75 | superproject_dir='exp-superproject'): | ||
| 76 | """Initializes superproject. | ||
| 77 | |||
| 78 | Args: | ||
| 79 | manifest: A Manifest object that is to be written to a file. | ||
| 80 | name: The unique name of the superproject | ||
| 81 | remote: The RemoteSpec for the remote. | ||
| 82 | revision: The name of the git branch to track. | ||
| 83 | superproject_dir: Relative path under |manifest.subdir| to checkout | ||
| 84 | superproject. | ||
| 85 | """ | ||
| 86 | self._project_commit_ids = None | ||
| 87 | self._manifest = manifest | ||
| 88 | self.name = name | ||
| 89 | self.remote = remote | ||
| 90 | self.revision = self._branch = revision | ||
| 91 | self._repodir = manifest.repodir | ||
| 92 | self._superproject_dir = superproject_dir | ||
| 93 | self._superproject_path = manifest.SubmanifestInfoDir(manifest.path_prefix, | ||
| 94 | superproject_dir) | ||
| 95 | self._manifest_path = os.path.join(self._superproject_path, | ||
| 96 | _SUPERPROJECT_MANIFEST_NAME) | ||
| 97 | git_name = hashlib.md5(remote.name.encode('utf8')).hexdigest() + '-' | ||
| 98 | self._remote_url = remote.url | ||
| 99 | self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME | ||
| 100 | self._work_git = os.path.join(self._superproject_path, self._work_git_name) | ||
| 101 | |||
| 102 | # The following are command arguemnts, rather than superproject attributes, | ||
| 103 | # and were included here originally. They should eventually become | ||
| 104 | # arguments that are passed down from the public methods, instead of being | ||
| 105 | # treated as attributes. | ||
| 106 | self._git_event_log = None | ||
| 107 | self._quiet = False | ||
| 108 | self._print_messages = False | ||
| 109 | |||
| 110 | def SetQuiet(self, value): | ||
| 111 | """Set the _quiet attribute.""" | ||
| 112 | self._quiet = value | ||
| 113 | |||
| 114 | def SetPrintMessages(self, value): | ||
| 115 | """Set the _print_messages attribute.""" | ||
| 116 | self._print_messages = value | ||
| 117 | |||
| 118 | @property | ||
| 119 | def project_commit_ids(self): | ||
| 120 | """Returns a dictionary of projects and their commit ids.""" | ||
| 121 | return self._project_commit_ids | ||
| 122 | |||
| 123 | @property | ||
| 124 | def manifest_path(self): | ||
| 125 | """Returns the manifest path if the path exists or None.""" | ||
| 126 | return self._manifest_path if os.path.exists(self._manifest_path) else None | ||
| 127 | |||
| 128 | def _LogMessage(self, fmt, *inputs): | ||
| 129 | """Logs message to stderr and _git_event_log.""" | ||
| 130 | message = f'{self._LogMessagePrefix()} {fmt.format(*inputs)}' | ||
| 131 | if self._print_messages: | ||
| 132 | print(message, file=sys.stderr) | ||
| 133 | self._git_event_log.ErrorEvent(message, fmt) | ||
| 134 | |||
| 135 | def _LogMessagePrefix(self): | ||
| 136 | """Returns the prefix string to be logged in each log message""" | ||
| 137 | return f'repo superproject branch: {self._branch} url: {self._remote_url}' | ||
| 138 | |||
| 139 | def _LogError(self, fmt, *inputs): | ||
| 140 | """Logs error message to stderr and _git_event_log.""" | ||
| 141 | self._LogMessage(f'error: {fmt}', *inputs) | ||
| 142 | |||
| 143 | def _LogWarning(self, fmt, *inputs): | ||
| 144 | """Logs warning message to stderr and _git_event_log.""" | ||
| 145 | self._LogMessage(f'warning: {fmt}', *inputs) | ||
| 146 | |||
| 147 | def _Init(self): | ||
| 148 | """Sets up a local Git repository to get a copy of a superproject. | ||
| 149 | |||
| 150 | Returns: | ||
| 151 | True if initialization is successful, or False. | ||
| 152 | """ | ||
| 153 | if not os.path.exists(self._superproject_path): | ||
| 154 | os.mkdir(self._superproject_path) | ||
| 155 | if not self._quiet and not os.path.exists(self._work_git): | ||
| 156 | print('%s: Performing initial setup for superproject; this might take ' | ||
| 157 | 'several minutes.' % self._work_git) | ||
| 158 | cmd = ['init', '--bare', self._work_git_name] | ||
| 159 | p = GitCommand(None, | ||
| 160 | cmd, | ||
| 161 | cwd=self._superproject_path, | ||
| 162 | capture_stdout=True, | ||
| 163 | capture_stderr=True) | ||
| 164 | retval = p.Wait() | ||
| 165 | if retval: | ||
| 166 | self._LogWarning('git init call failed, command: git {}, ' | ||
| 167 | 'return code: {}, stderr: {}', cmd, retval, p.stderr) | ||
| 168 | return False | ||
| 169 | return True | ||
| 170 | |||
| 171 | def _Fetch(self): | ||
| 172 | """Fetches a local copy of a superproject for the manifest based on |_remote_url|. | ||
| 173 | |||
| 174 | Returns: | ||
| 175 | True if fetch is successful, or False. | ||
| 176 | """ | ||
| 177 | if not os.path.exists(self._work_git): | ||
| 178 | self._LogWarning('git fetch missing directory: {}', self._work_git) | ||
| 179 | return False | ||
| 180 | if not git_require((2, 28, 0)): | ||
| 181 | self._LogWarning('superproject requires a git version 2.28 or later') | ||
| 182 | return False | ||
| 183 | cmd = ['fetch', self._remote_url, '--depth', '1', '--force', '--no-tags', | ||
| 184 | '--filter', 'blob:none'] | ||
| 185 | |||
| 186 | # Check if there is a local ref that we can pass to --negotiation-tip. | ||
| 187 | # If this is the first fetch, it does not exist yet. | ||
| 188 | # We use --negotiation-tip to speed up the fetch. Superproject branches do | ||
| 189 | # not share commits. So this lets git know it only needs to send commits | ||
| 190 | # reachable from the specified local refs. | ||
| 191 | rev_commit = GitRefs(self._work_git).get(f'refs/heads/{self.revision}') | ||
| 192 | if rev_commit: | ||
| 193 | cmd.extend(['--negotiation-tip', rev_commit]) | ||
| 194 | |||
| 195 | if self._branch: | ||
| 196 | cmd += [self._branch + ':' + self._branch] | ||
| 197 | p = GitCommand(None, | ||
| 198 | cmd, | ||
| 199 | cwd=self._work_git, | ||
| 200 | capture_stdout=True, | ||
| 201 | capture_stderr=True) | ||
| 202 | retval = p.Wait() | ||
| 203 | if retval: | ||
| 204 | self._LogWarning('git fetch call failed, command: git {}, ' | ||
| 205 | 'return code: {}, stderr: {}', cmd, retval, p.stderr) | ||
| 206 | return False | ||
| 207 | return True | ||
| 208 | |||
| 209 | def _LsTree(self): | ||
| 210 | """Gets the commit ids for all projects. | ||
| 211 | |||
| 212 | Works only in git repositories. | ||
| 213 | |||
| 214 | Returns: | ||
| 215 | data: data returned from 'git ls-tree ...' instead of None. | ||
| 216 | """ | ||
| 217 | if not os.path.exists(self._work_git): | ||
| 218 | self._LogWarning('git ls-tree missing directory: {}', self._work_git) | ||
| 219 | return None | ||
| 220 | data = None | ||
| 221 | branch = 'HEAD' if not self._branch else self._branch | ||
| 222 | cmd = ['ls-tree', '-z', '-r', branch] | ||
| 223 | |||
| 224 | p = GitCommand(None, | ||
| 225 | cmd, | ||
| 226 | cwd=self._work_git, | ||
| 227 | capture_stdout=True, | ||
| 228 | capture_stderr=True) | ||
| 229 | retval = p.Wait() | ||
| 230 | if retval == 0: | ||
| 231 | data = p.stdout | ||
| 232 | else: | ||
| 233 | self._LogWarning('git ls-tree call failed, command: git {}, ' | ||
| 234 | 'return code: {}, stderr: {}', cmd, retval, p.stderr) | ||
| 235 | return data | ||
| 236 | |||
| 237 | def Sync(self, git_event_log): | ||
| 238 | """Gets a local copy of a superproject for the manifest. | ||
| 239 | |||
| 240 | Args: | ||
| 241 | git_event_log: an EventLog, for git tracing. | ||
| 242 | |||
| 243 | Returns: | ||
| 244 | SyncResult | ||
| 245 | """ | ||
| 246 | self._git_event_log = git_event_log | ||
| 247 | if not self._manifest.superproject: | ||
| 248 | self._LogWarning('superproject tag is not defined in manifest: {}', | ||
| 249 | self._manifest.manifestFile) | ||
| 250 | return SyncResult(False, False) | ||
| 251 | |||
| 252 | _PrintBetaNotice() | ||
| 253 | |||
| 254 | should_exit = True | ||
| 255 | if not self._remote_url: | ||
| 256 | self._LogWarning('superproject URL is not defined in manifest: {}', | ||
| 257 | self._manifest.manifestFile) | ||
| 258 | return SyncResult(False, should_exit) | ||
| 259 | |||
| 260 | if not self._Init(): | ||
| 261 | return SyncResult(False, should_exit) | ||
| 262 | if not self._Fetch(): | ||
| 263 | return SyncResult(False, should_exit) | ||
| 264 | if not self._quiet: | ||
| 265 | print('%s: Initial setup for superproject completed.' % self._work_git) | ||
| 266 | return SyncResult(True, False) | ||
| 267 | |||
| 268 | def _GetAllProjectsCommitIds(self): | ||
| 269 | """Get commit ids for all projects from superproject and save them in _project_commit_ids. | ||
| 270 | |||
| 271 | Returns: | ||
| 272 | CommitIdsResult | ||
| 273 | """ | ||
| 274 | sync_result = self.Sync(self._git_event_log) | ||
| 275 | if not sync_result.success: | ||
| 276 | return CommitIdsResult(None, sync_result.fatal) | ||
| 277 | |||
| 278 | data = self._LsTree() | ||
| 279 | if not data: | ||
| 280 | self._LogWarning('git ls-tree failed to return data for manifest: {}', | ||
| 281 | self._manifest.manifestFile) | ||
| 282 | return CommitIdsResult(None, True) | ||
| 283 | |||
| 284 | # Parse lines like the following to select lines starting with '160000' and | ||
| 285 | # build a dictionary with project path (last element) and its commit id (3rd element). | ||
| 286 | # | ||
| 287 | # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00 | ||
| 288 | # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00 | ||
| 289 | commit_ids = {} | ||
| 290 | for line in data.split('\x00'): | ||
| 291 | ls_data = line.split(None, 3) | ||
| 292 | if not ls_data: | ||
| 293 | break | ||
| 294 | if ls_data[0] == '160000': | ||
| 295 | commit_ids[ls_data[3]] = ls_data[2] | ||
| 296 | |||
| 297 | self._project_commit_ids = commit_ids | ||
| 298 | return CommitIdsResult(commit_ids, False) | ||
| 299 | |||
| 300 | def _WriteManifestFile(self): | ||
| 301 | """Writes manifest to a file. | ||
| 302 | |||
| 303 | Returns: | ||
| 304 | manifest_path: Path name of the file into which manifest is written instead of None. | ||
| 305 | """ | ||
| 306 | if not os.path.exists(self._superproject_path): | ||
| 307 | self._LogWarning('missing superproject directory: {}', self._superproject_path) | ||
| 308 | return None | ||
| 309 | manifest_str = self._manifest.ToXml(groups=self._manifest.GetGroupsStr(), | ||
| 310 | omit_local=True).toxml() | ||
| 311 | manifest_path = self._manifest_path | ||
| 312 | try: | ||
| 313 | with open(manifest_path, 'w', encoding='utf-8') as fp: | ||
| 314 | fp.write(manifest_str) | ||
| 315 | except IOError as e: | ||
| 316 | self._LogError('cannot write manifest to : {} {}', | ||
| 317 | manifest_path, e) | ||
| 318 | return None | ||
| 319 | return manifest_path | ||
| 320 | |||
| 321 | def _SkipUpdatingProjectRevisionId(self, project): | ||
| 322 | """Checks if a project's revision id needs to be updated or not. | ||
| 323 | |||
| 324 | Revision id for projects from local manifest will not be updated. | ||
| 325 | |||
| 326 | Args: | ||
| 327 | project: project whose revision id is being updated. | ||
| 328 | |||
| 329 | Returns: | ||
| 330 | True if a project's revision id should not be updated, or False, | ||
| 331 | """ | 73 | """ |
| 332 | path = project.relpath | ||
| 333 | if not path: | ||
| 334 | return True | ||
| 335 | # Skip the project with revisionId. | ||
| 336 | if project.revisionId: | ||
| 337 | return True | ||
| 338 | # Skip the project if it comes from the local manifest. | ||
| 339 | return project.manifest.IsFromLocalManifest(project) | ||
| 340 | |||
| 341 | def UpdateProjectsRevisionId(self, projects, git_event_log): | ||
| 342 | """Update revisionId of every project in projects with the commit id. | ||
| 343 | |||
| 344 | Args: | ||
| 345 | projects: a list of projects whose revisionId needs to be updated. | ||
| 346 | git_event_log: an EventLog, for git tracing. | ||
| 347 | 74 | ||
| 348 | Returns: | 75 | def __init__( |
| 349 | UpdateProjectsResult | 76 | self, |
| 350 | """ | 77 | manifest, |
| 351 | self._git_event_log = git_event_log | 78 | name, |
| 352 | commit_ids_result = self._GetAllProjectsCommitIds() | 79 | remote, |
| 353 | commit_ids = commit_ids_result.commit_ids | 80 | revision, |
| 354 | if not commit_ids: | 81 | superproject_dir="exp-superproject", |
| 355 | return UpdateProjectsResult(None, commit_ids_result.fatal) | 82 | ): |
| 356 | 83 | """Initializes superproject. | |
| 357 | projects_missing_commit_ids = [] | 84 | |
| 358 | for project in projects: | 85 | Args: |
| 359 | if self._SkipUpdatingProjectRevisionId(project): | 86 | manifest: A Manifest object that is to be written to a file. |
| 360 | continue | 87 | name: The unique name of the superproject |
| 361 | path = project.relpath | 88 | remote: The RemoteSpec for the remote. |
| 362 | commit_id = commit_ids.get(path) | 89 | revision: The name of the git branch to track. |
| 363 | if not commit_id: | 90 | superproject_dir: Relative path under |manifest.subdir| to checkout |
| 364 | projects_missing_commit_ids.append(path) | 91 | superproject. |
| 365 | 92 | """ | |
| 366 | # If superproject doesn't have a commit id for a project, then report an | 93 | self._project_commit_ids = None |
| 367 | # error event and continue as if do not use superproject is specified. | 94 | self._manifest = manifest |
| 368 | if projects_missing_commit_ids: | 95 | self.name = name |
| 369 | self._LogWarning('please file a bug using {} to report missing ' | 96 | self.remote = remote |
| 370 | 'commit_ids for: {}', self._manifest.contactinfo.bugurl, | 97 | self.revision = self._branch = revision |
| 371 | projects_missing_commit_ids) | 98 | self._repodir = manifest.repodir |
| 372 | return UpdateProjectsResult(None, False) | 99 | self._superproject_dir = superproject_dir |
| 373 | 100 | self._superproject_path = manifest.SubmanifestInfoDir( | |
| 374 | for project in projects: | 101 | manifest.path_prefix, superproject_dir |
| 375 | if not self._SkipUpdatingProjectRevisionId(project): | 102 | ) |
| 376 | project.SetRevisionId(commit_ids.get(project.relpath)) | 103 | self._manifest_path = os.path.join( |
| 377 | 104 | self._superproject_path, _SUPERPROJECT_MANIFEST_NAME | |
| 378 | manifest_path = self._WriteManifestFile() | 105 | ) |
| 379 | return UpdateProjectsResult(manifest_path, False) | 106 | git_name = hashlib.md5(remote.name.encode("utf8")).hexdigest() + "-" |
| 107 | self._remote_url = remote.url | ||
| 108 | self._work_git_name = git_name + _SUPERPROJECT_GIT_NAME | ||
| 109 | self._work_git = os.path.join( | ||
| 110 | self._superproject_path, self._work_git_name | ||
| 111 | ) | ||
| 112 | |||
| 113 | # The following are command arguemnts, rather than superproject | ||
| 114 | # attributes, and were included here originally. They should eventually | ||
| 115 | # become arguments that are passed down from the public methods, instead | ||
| 116 | # of being treated as attributes. | ||
| 117 | self._git_event_log = None | ||
| 118 | self._quiet = False | ||
| 119 | self._print_messages = False | ||
| 120 | |||
| 121 | def SetQuiet(self, value): | ||
| 122 | """Set the _quiet attribute.""" | ||
| 123 | self._quiet = value | ||
| 124 | |||
| 125 | def SetPrintMessages(self, value): | ||
| 126 | """Set the _print_messages attribute.""" | ||
| 127 | self._print_messages = value | ||
| 128 | |||
| 129 | @property | ||
| 130 | def project_commit_ids(self): | ||
| 131 | """Returns a dictionary of projects and their commit ids.""" | ||
| 132 | return self._project_commit_ids | ||
| 133 | |||
| 134 | @property | ||
| 135 | def manifest_path(self): | ||
| 136 | """Returns the manifest path if the path exists or None.""" | ||
| 137 | return ( | ||
| 138 | self._manifest_path if os.path.exists(self._manifest_path) else None | ||
| 139 | ) | ||
| 140 | |||
| 141 | def _LogMessage(self, fmt, *inputs): | ||
| 142 | """Logs message to stderr and _git_event_log.""" | ||
| 143 | message = f"{self._LogMessagePrefix()} {fmt.format(*inputs)}" | ||
| 144 | if self._print_messages: | ||
| 145 | print(message, file=sys.stderr) | ||
| 146 | self._git_event_log.ErrorEvent(message, fmt) | ||
| 147 | |||
| 148 | def _LogMessagePrefix(self): | ||
| 149 | """Returns the prefix string to be logged in each log message""" | ||
| 150 | return ( | ||
| 151 | f"repo superproject branch: {self._branch} url: {self._remote_url}" | ||
| 152 | ) | ||
| 153 | |||
| 154 | def _LogError(self, fmt, *inputs): | ||
| 155 | """Logs error message to stderr and _git_event_log.""" | ||
| 156 | self._LogMessage(f"error: {fmt}", *inputs) | ||
| 157 | |||
| 158 | def _LogWarning(self, fmt, *inputs): | ||
| 159 | """Logs warning message to stderr and _git_event_log.""" | ||
| 160 | self._LogMessage(f"warning: {fmt}", *inputs) | ||
| 161 | |||
| 162 | def _Init(self): | ||
| 163 | """Sets up a local Git repository to get a copy of a superproject. | ||
| 164 | |||
| 165 | Returns: | ||
| 166 | True if initialization is successful, or False. | ||
| 167 | """ | ||
| 168 | if not os.path.exists(self._superproject_path): | ||
| 169 | os.mkdir(self._superproject_path) | ||
| 170 | if not self._quiet and not os.path.exists(self._work_git): | ||
| 171 | print( | ||
| 172 | "%s: Performing initial setup for superproject; this might " | ||
| 173 | "take several minutes." % self._work_git | ||
| 174 | ) | ||
| 175 | cmd = ["init", "--bare", self._work_git_name] | ||
| 176 | p = GitCommand( | ||
| 177 | None, | ||
| 178 | cmd, | ||
| 179 | cwd=self._superproject_path, | ||
| 180 | capture_stdout=True, | ||
| 181 | capture_stderr=True, | ||
| 182 | ) | ||
| 183 | retval = p.Wait() | ||
| 184 | if retval: | ||
| 185 | self._LogWarning( | ||
| 186 | "git init call failed, command: git {}, " | ||
| 187 | "return code: {}, stderr: {}", | ||
| 188 | cmd, | ||
| 189 | retval, | ||
| 190 | p.stderr, | ||
| 191 | ) | ||
| 192 | return False | ||
| 193 | return True | ||
| 194 | |||
| 195 | def _Fetch(self): | ||
| 196 | """Fetches a superproject for the manifest based on |_remote_url|. | ||
| 197 | |||
| 198 | This runs git fetch which stores a local copy the superproject. | ||
| 199 | |||
| 200 | Returns: | ||
| 201 | True if fetch is successful, or False. | ||
| 202 | """ | ||
| 203 | if not os.path.exists(self._work_git): | ||
| 204 | self._LogWarning("git fetch missing directory: {}", self._work_git) | ||
| 205 | return False | ||
| 206 | if not git_require((2, 28, 0)): | ||
| 207 | self._LogWarning( | ||
| 208 | "superproject requires a git version 2.28 or later" | ||
| 209 | ) | ||
| 210 | return False | ||
| 211 | cmd = [ | ||
| 212 | "fetch", | ||
| 213 | self._remote_url, | ||
| 214 | "--depth", | ||
| 215 | "1", | ||
| 216 | "--force", | ||
| 217 | "--no-tags", | ||
| 218 | "--filter", | ||
| 219 | "blob:none", | ||
| 220 | ] | ||
| 221 | |||
| 222 | # Check if there is a local ref that we can pass to --negotiation-tip. | ||
| 223 | # If this is the first fetch, it does not exist yet. | ||
| 224 | # We use --negotiation-tip to speed up the fetch. Superproject branches | ||
| 225 | # do not share commits. So this lets git know it only needs to send | ||
| 226 | # commits reachable from the specified local refs. | ||
| 227 | rev_commit = GitRefs(self._work_git).get(f"refs/heads/{self.revision}") | ||
| 228 | if rev_commit: | ||
| 229 | cmd.extend(["--negotiation-tip", rev_commit]) | ||
| 230 | |||
| 231 | if self._branch: | ||
| 232 | cmd += [self._branch + ":" + self._branch] | ||
| 233 | p = GitCommand( | ||
| 234 | None, | ||
| 235 | cmd, | ||
| 236 | cwd=self._work_git, | ||
| 237 | capture_stdout=True, | ||
| 238 | capture_stderr=True, | ||
| 239 | ) | ||
| 240 | retval = p.Wait() | ||
| 241 | if retval: | ||
| 242 | self._LogWarning( | ||
| 243 | "git fetch call failed, command: git {}, " | ||
| 244 | "return code: {}, stderr: {}", | ||
| 245 | cmd, | ||
| 246 | retval, | ||
| 247 | p.stderr, | ||
| 248 | ) | ||
| 249 | return False | ||
| 250 | return True | ||
| 251 | |||
| 252 | def _LsTree(self): | ||
| 253 | """Gets the commit ids for all projects. | ||
| 254 | |||
| 255 | Works only in git repositories. | ||
| 256 | |||
| 257 | Returns: | ||
| 258 | data: data returned from 'git ls-tree ...' instead of None. | ||
| 259 | """ | ||
| 260 | if not os.path.exists(self._work_git): | ||
| 261 | self._LogWarning( | ||
| 262 | "git ls-tree missing directory: {}", self._work_git | ||
| 263 | ) | ||
| 264 | return None | ||
| 265 | data = None | ||
| 266 | branch = "HEAD" if not self._branch else self._branch | ||
| 267 | cmd = ["ls-tree", "-z", "-r", branch] | ||
| 268 | |||
| 269 | p = GitCommand( | ||
| 270 | None, | ||
| 271 | cmd, | ||
| 272 | cwd=self._work_git, | ||
| 273 | capture_stdout=True, | ||
| 274 | capture_stderr=True, | ||
| 275 | ) | ||
| 276 | retval = p.Wait() | ||
| 277 | if retval == 0: | ||
| 278 | data = p.stdout | ||
| 279 | else: | ||
| 280 | self._LogWarning( | ||
| 281 | "git ls-tree call failed, command: git {}, " | ||
| 282 | "return code: {}, stderr: {}", | ||
| 283 | cmd, | ||
| 284 | retval, | ||
| 285 | p.stderr, | ||
| 286 | ) | ||
| 287 | return data | ||
| 288 | |||
| 289 | def Sync(self, git_event_log): | ||
| 290 | """Gets a local copy of a superproject for the manifest. | ||
| 291 | |||
| 292 | Args: | ||
| 293 | git_event_log: an EventLog, for git tracing. | ||
| 294 | |||
| 295 | Returns: | ||
| 296 | SyncResult | ||
| 297 | """ | ||
| 298 | self._git_event_log = git_event_log | ||
| 299 | if not self._manifest.superproject: | ||
| 300 | self._LogWarning( | ||
| 301 | "superproject tag is not defined in manifest: {}", | ||
| 302 | self._manifest.manifestFile, | ||
| 303 | ) | ||
| 304 | return SyncResult(False, False) | ||
| 305 | |||
| 306 | _PrintBetaNotice() | ||
| 307 | |||
| 308 | should_exit = True | ||
| 309 | if not self._remote_url: | ||
| 310 | self._LogWarning( | ||
| 311 | "superproject URL is not defined in manifest: {}", | ||
| 312 | self._manifest.manifestFile, | ||
| 313 | ) | ||
| 314 | return SyncResult(False, should_exit) | ||
| 315 | |||
| 316 | if not self._Init(): | ||
| 317 | return SyncResult(False, should_exit) | ||
| 318 | if not self._Fetch(): | ||
| 319 | return SyncResult(False, should_exit) | ||
| 320 | if not self._quiet: | ||
| 321 | print( | ||
| 322 | "%s: Initial setup for superproject completed." % self._work_git | ||
| 323 | ) | ||
| 324 | return SyncResult(True, False) | ||
| 325 | |||
| 326 | def _GetAllProjectsCommitIds(self): | ||
| 327 | """Get commit ids for all projects from superproject and save them. | ||
| 328 | |||
| 329 | Commit ids are saved in _project_commit_ids. | ||
| 330 | |||
| 331 | Returns: | ||
| 332 | CommitIdsResult | ||
| 333 | """ | ||
| 334 | sync_result = self.Sync(self._git_event_log) | ||
| 335 | if not sync_result.success: | ||
| 336 | return CommitIdsResult(None, sync_result.fatal) | ||
| 337 | |||
| 338 | data = self._LsTree() | ||
| 339 | if not data: | ||
| 340 | self._LogWarning( | ||
| 341 | "git ls-tree failed to return data for manifest: {}", | ||
| 342 | self._manifest.manifestFile, | ||
| 343 | ) | ||
| 344 | return CommitIdsResult(None, True) | ||
| 345 | |||
| 346 | # Parse lines like the following to select lines starting with '160000' | ||
| 347 | # and build a dictionary with project path (last element) and its commit | ||
| 348 | # id (3rd element). | ||
| 349 | # | ||
| 350 | # 160000 commit 2c2724cb36cd5a9cec6c852c681efc3b7c6b86ea\tart\x00 | ||
| 351 | # 120000 blob acc2cbdf438f9d2141f0ae424cec1d8fc4b5d97f\tbootstrap.bash\x00 # noqa: E501 | ||
| 352 | commit_ids = {} | ||
| 353 | for line in data.split("\x00"): | ||
| 354 | ls_data = line.split(None, 3) | ||
| 355 | if not ls_data: | ||
| 356 | break | ||
| 357 | if ls_data[0] == "160000": | ||
| 358 | commit_ids[ls_data[3]] = ls_data[2] | ||
| 359 | |||
| 360 | self._project_commit_ids = commit_ids | ||
| 361 | return CommitIdsResult(commit_ids, False) | ||
| 362 | |||
| 363 | def _WriteManifestFile(self): | ||
| 364 | """Writes manifest to a file. | ||
| 365 | |||
| 366 | Returns: | ||
| 367 | manifest_path: Path name of the file into which manifest is written | ||
| 368 | instead of None. | ||
| 369 | """ | ||
| 370 | if not os.path.exists(self._superproject_path): | ||
| 371 | self._LogWarning( | ||
| 372 | "missing superproject directory: {}", self._superproject_path | ||
| 373 | ) | ||
| 374 | return None | ||
| 375 | manifest_str = self._manifest.ToXml( | ||
| 376 | groups=self._manifest.GetGroupsStr(), omit_local=True | ||
| 377 | ).toxml() | ||
| 378 | manifest_path = self._manifest_path | ||
| 379 | try: | ||
| 380 | with open(manifest_path, "w", encoding="utf-8") as fp: | ||
| 381 | fp.write(manifest_str) | ||
| 382 | except IOError as e: | ||
| 383 | self._LogError("cannot write manifest to : {} {}", manifest_path, e) | ||
| 384 | return None | ||
| 385 | return manifest_path | ||
| 386 | |||
| 387 | def _SkipUpdatingProjectRevisionId(self, project): | ||
| 388 | """Checks if a project's revision id needs to be updated or not. | ||
| 389 | |||
| 390 | Revision id for projects from local manifest will not be updated. | ||
| 391 | |||
| 392 | Args: | ||
| 393 | project: project whose revision id is being updated. | ||
| 394 | |||
| 395 | Returns: | ||
| 396 | True if a project's revision id should not be updated, or False, | ||
| 397 | """ | ||
| 398 | path = project.relpath | ||
| 399 | if not path: | ||
| 400 | return True | ||
| 401 | # Skip the project with revisionId. | ||
| 402 | if project.revisionId: | ||
| 403 | return True | ||
| 404 | # Skip the project if it comes from the local manifest. | ||
| 405 | return project.manifest.IsFromLocalManifest(project) | ||
| 406 | |||
| 407 | def UpdateProjectsRevisionId(self, projects, git_event_log): | ||
| 408 | """Update revisionId of every project in projects with the commit id. | ||
| 409 | |||
| 410 | Args: | ||
| 411 | projects: a list of projects whose revisionId needs to be updated. | ||
| 412 | git_event_log: an EventLog, for git tracing. | ||
| 413 | |||
| 414 | Returns: | ||
| 415 | UpdateProjectsResult | ||
| 416 | """ | ||
| 417 | self._git_event_log = git_event_log | ||
| 418 | commit_ids_result = self._GetAllProjectsCommitIds() | ||
| 419 | commit_ids = commit_ids_result.commit_ids | ||
| 420 | if not commit_ids: | ||
| 421 | return UpdateProjectsResult(None, commit_ids_result.fatal) | ||
| 422 | |||
| 423 | projects_missing_commit_ids = [] | ||
| 424 | for project in projects: | ||
| 425 | if self._SkipUpdatingProjectRevisionId(project): | ||
| 426 | continue | ||
| 427 | path = project.relpath | ||
| 428 | commit_id = commit_ids.get(path) | ||
| 429 | if not commit_id: | ||
| 430 | projects_missing_commit_ids.append(path) | ||
| 431 | |||
| 432 | # If superproject doesn't have a commit id for a project, then report an | ||
| 433 | # error event and continue as if do not use superproject is specified. | ||
| 434 | if projects_missing_commit_ids: | ||
| 435 | self._LogWarning( | ||
| 436 | "please file a bug using {} to report missing " | ||
| 437 | "commit_ids for: {}", | ||
| 438 | self._manifest.contactinfo.bugurl, | ||
| 439 | projects_missing_commit_ids, | ||
| 440 | ) | ||
| 441 | return UpdateProjectsResult(None, False) | ||
| 442 | |||
| 443 | for project in projects: | ||
| 444 | if not self._SkipUpdatingProjectRevisionId(project): | ||
| 445 | project.SetRevisionId(commit_ids.get(project.relpath)) | ||
| 446 | |||
| 447 | manifest_path = self._WriteManifestFile() | ||
| 448 | return UpdateProjectsResult(manifest_path, False) | ||
| 380 | 449 | ||
| 381 | 450 | ||
| 382 | @functools.lru_cache(maxsize=10) | 451 | @functools.lru_cache(maxsize=10) |
| 383 | def _PrintBetaNotice(): | 452 | def _PrintBetaNotice(): |
| 384 | """Print the notice of beta status.""" | 453 | """Print the notice of beta status.""" |
| 385 | print('NOTICE: --use-superproject is in beta; report any issues to the ' | 454 | print( |
| 386 | 'address described in `repo version`', file=sys.stderr) | 455 | "NOTICE: --use-superproject is in beta; report any issues to the " |
| 456 | "address described in `repo version`", | ||
| 457 | file=sys.stderr, | ||
| 458 | ) | ||
| 387 | 459 | ||
| 388 | 460 | ||
| 389 | @functools.lru_cache(maxsize=None) | 461 | @functools.lru_cache(maxsize=None) |
| 390 | def _UseSuperprojectFromConfiguration(): | 462 | def _UseSuperprojectFromConfiguration(): |
| 391 | """Returns the user choice of whether to use superproject.""" | 463 | """Returns the user choice of whether to use superproject.""" |
| 392 | user_cfg = RepoConfig.ForUser() | 464 | user_cfg = RepoConfig.ForUser() |
| 393 | time_now = int(time.time()) | 465 | time_now = int(time.time()) |
| 394 | 466 | ||
| 395 | user_value = user_cfg.GetBoolean('repo.superprojectChoice') | 467 | user_value = user_cfg.GetBoolean("repo.superprojectChoice") |
| 396 | if user_value is not None: | 468 | if user_value is not None: |
| 397 | user_expiration = user_cfg.GetInt('repo.superprojectChoiceExpire') | 469 | user_expiration = user_cfg.GetInt("repo.superprojectChoiceExpire") |
| 398 | if user_expiration is None or user_expiration <= 0 or user_expiration >= time_now: | 470 | if ( |
| 399 | # TODO(b/190688390) - Remove prompt when we are comfortable with the new | 471 | user_expiration is None |
| 400 | # default value. | 472 | or user_expiration <= 0 |
| 401 | if user_value: | 473 | or user_expiration >= time_now |
| 402 | print(('You are currently enrolled in Git submodules experiment ' | 474 | ): |
| 403 | '(go/android-submodules-quickstart). Use --no-use-superproject ' | 475 | # TODO(b/190688390) - Remove prompt when we are comfortable with the |
| 404 | 'to override.\n'), file=sys.stderr) | 476 | # new default value. |
| 405 | else: | 477 | if user_value: |
| 406 | print(('You are not currently enrolled in Git submodules experiment ' | 478 | print( |
| 407 | '(go/android-submodules-quickstart). Use --use-superproject ' | 479 | ( |
| 408 | 'to override.\n'), file=sys.stderr) | 480 | "You are currently enrolled in Git submodules " |
| 409 | return user_value | 481 | "experiment (go/android-submodules-quickstart). Use " |
| 410 | 482 | "--no-use-superproject to override.\n" | |
| 411 | # We don't have an unexpired choice, ask for one. | 483 | ), |
| 412 | system_cfg = RepoConfig.ForSystem() | 484 | file=sys.stderr, |
| 413 | system_value = system_cfg.GetBoolean('repo.superprojectChoice') | 485 | ) |
| 414 | if system_value: | 486 | else: |
| 415 | # The system configuration is proposing that we should enable the | 487 | print( |
| 416 | # use of superproject. Treat the user as enrolled for two weeks. | 488 | ( |
| 417 | # | 489 | "You are not currently enrolled in Git submodules " |
| 418 | # TODO(b/190688390) - Remove prompt when we are comfortable with the new | 490 | "experiment (go/android-submodules-quickstart). Use " |
| 419 | # default value. | 491 | "--use-superproject to override.\n" |
| 420 | userchoice = True | 492 | ), |
| 421 | time_choiceexpire = time_now + (86400 * 14) | 493 | file=sys.stderr, |
| 422 | user_cfg.SetString('repo.superprojectChoiceExpire', str(time_choiceexpire)) | 494 | ) |
| 423 | user_cfg.SetBoolean('repo.superprojectChoice', userchoice) | 495 | return user_value |
| 424 | print('You are automatically enrolled in Git submodules experiment ' | 496 | |
| 425 | '(go/android-submodules-quickstart) for another two weeks.\n', | 497 | # We don't have an unexpired choice, ask for one. |
| 426 | file=sys.stderr) | 498 | system_cfg = RepoConfig.ForSystem() |
| 427 | return True | 499 | system_value = system_cfg.GetBoolean("repo.superprojectChoice") |
| 428 | 500 | if system_value: | |
| 429 | # For all other cases, we would not use superproject by default. | 501 | # The system configuration is proposing that we should enable the |
| 430 | return False | 502 | # use of superproject. Treat the user as enrolled for two weeks. |
| 503 | # | ||
| 504 | # TODO(b/190688390) - Remove prompt when we are comfortable with the new | ||
| 505 | # default value. | ||
| 506 | userchoice = True | ||
| 507 | time_choiceexpire = time_now + (86400 * 14) | ||
| 508 | user_cfg.SetString( | ||
| 509 | "repo.superprojectChoiceExpire", str(time_choiceexpire) | ||
| 510 | ) | ||
| 511 | user_cfg.SetBoolean("repo.superprojectChoice", userchoice) | ||
| 512 | print( | ||
| 513 | "You are automatically enrolled in Git submodules experiment " | ||
| 514 | "(go/android-submodules-quickstart) for another two weeks.\n", | ||
| 515 | file=sys.stderr, | ||
| 516 | ) | ||
| 517 | return True | ||
| 518 | |||
| 519 | # For all other cases, we would not use superproject by default. | ||
| 520 | return False | ||
| 431 | 521 | ||
| 432 | 522 | ||
| 433 | def PrintMessages(use_superproject, manifest): | 523 | def PrintMessages(use_superproject, manifest): |
| 434 | """Returns a boolean if error/warning messages are to be printed. | 524 | """Returns a boolean if error/warning messages are to be printed. |
| 435 | 525 | ||
| 436 | Args: | 526 | Args: |
| 437 | use_superproject: option value from optparse. | 527 | use_superproject: option value from optparse. |
| 438 | manifest: manifest to use. | 528 | manifest: manifest to use. |
| 439 | """ | 529 | """ |
| 440 | return use_superproject is not None or bool(manifest.superproject) | 530 | return use_superproject is not None or bool(manifest.superproject) |
| 441 | 531 | ||
| 442 | 532 | ||
| 443 | def UseSuperproject(use_superproject, manifest): | 533 | def UseSuperproject(use_superproject, manifest): |
| 444 | """Returns a boolean if use-superproject option is enabled. | 534 | """Returns a boolean if use-superproject option is enabled. |
| 445 | 535 | ||
| 446 | Args: | 536 | Args: |
| 447 | use_superproject: option value from optparse. | 537 | use_superproject: option value from optparse. |
| 448 | manifest: manifest to use. | 538 | manifest: manifest to use. |
| 449 | 539 | ||
| 450 | Returns: | 540 | Returns: |
| 451 | Whether the superproject should be used. | 541 | Whether the superproject should be used. |
| 452 | """ | 542 | """ |
| 453 | 543 | ||
| 454 | if not manifest.superproject: | 544 | if not manifest.superproject: |
| 455 | # This (sub) manifest does not have a superproject definition. | 545 | # This (sub) manifest does not have a superproject definition. |
| 456 | return False | 546 | return False |
| 457 | elif use_superproject is not None: | 547 | elif use_superproject is not None: |
| 458 | return use_superproject | 548 | return use_superproject |
| 459 | else: | ||
| 460 | client_value = manifest.manifestProject.use_superproject | ||
| 461 | if client_value is not None: | ||
| 462 | return client_value | ||
| 463 | elif manifest.superproject: | ||
| 464 | return _UseSuperprojectFromConfiguration() | ||
| 465 | else: | 549 | else: |
| 466 | return False | 550 | client_value = manifest.manifestProject.use_superproject |
| 551 | if client_value is not None: | ||
| 552 | return client_value | ||
| 553 | elif manifest.superproject: | ||
| 554 | return _UseSuperprojectFromConfiguration() | ||
| 555 | else: | ||
| 556 | return False | ||
