diff options
Diffstat (limited to 'subcmds/sync.py')
| -rw-r--r-- | subcmds/sync.py | 49 |
1 files changed, 19 insertions, 30 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py index 21166af5..4763fadc 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
| @@ -51,7 +51,7 @@ import git_superproject | |||
| 51 | import gitc_utils | 51 | import gitc_utils |
| 52 | from project import Project | 52 | from project import Project |
| 53 | from project import RemoteSpec | 53 | from project import RemoteSpec |
| 54 | from command import Command, MirrorSafeCommand, WORKER_BATCH_SIZE | 54 | from command import Command, MirrorSafeCommand |
| 55 | from error import RepoChangedException, GitError, ManifestParseError | 55 | from error import RepoChangedException, GitError, ManifestParseError |
| 56 | import platform_utils | 56 | import platform_utils |
| 57 | from project import SyncBuffer | 57 | from project import SyncBuffer |
| @@ -428,11 +428,12 @@ later is required to fix a server side protocol bug. | |||
| 428 | 428 | ||
| 429 | return (ret, fetched) | 429 | return (ret, fetched) |
| 430 | 430 | ||
| 431 | def _CheckoutOne(self, opt, project): | 431 | def _CheckoutOne(self, detach_head, force_sync, project): |
| 432 | """Checkout work tree for one project | 432 | """Checkout work tree for one project |
| 433 | 433 | ||
| 434 | Args: | 434 | Args: |
| 435 | opt: Program options returned from optparse. See _Options(). | 435 | detach_head: Whether to leave a detached HEAD. |
| 436 | force_sync: Force checking out of the repo. | ||
| 436 | project: Project object for the project to checkout. | 437 | project: Project object for the project to checkout. |
| 437 | 438 | ||
| 438 | Returns: | 439 | Returns: |
| @@ -440,10 +441,10 @@ later is required to fix a server side protocol bug. | |||
| 440 | """ | 441 | """ |
| 441 | start = time.time() | 442 | start = time.time() |
| 442 | syncbuf = SyncBuffer(self.manifest.manifestProject.config, | 443 | syncbuf = SyncBuffer(self.manifest.manifestProject.config, |
| 443 | detach_head=opt.detach_head) | 444 | detach_head=detach_head) |
| 444 | success = False | 445 | success = False |
| 445 | try: | 446 | try: |
| 446 | project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync) | 447 | project.Sync_LocalHalf(syncbuf, force_sync=force_sync) |
| 447 | success = syncbuf.Finish() | 448 | success = syncbuf.Finish() |
| 448 | except Exception as e: | 449 | except Exception as e: |
| 449 | print('error: Cannot checkout %s: %s: %s' % | 450 | print('error: Cannot checkout %s: %s: %s' % |
| @@ -464,44 +465,32 @@ later is required to fix a server side protocol bug. | |||
| 464 | opt: Program options returned from optparse. See _Options(). | 465 | opt: Program options returned from optparse. See _Options(). |
| 465 | err_results: A list of strings, paths to git repos where checkout failed. | 466 | err_results: A list of strings, paths to git repos where checkout failed. |
| 466 | """ | 467 | """ |
| 467 | ret = True | ||
| 468 | jobs = opt.jobs_checkout if opt.jobs_checkout else self.jobs | ||
| 469 | |||
| 470 | # Only checkout projects with worktrees. | 468 | # Only checkout projects with worktrees. |
| 471 | all_projects = [x for x in all_projects if x.worktree] | 469 | all_projects = [x for x in all_projects if x.worktree] |
| 472 | 470 | ||
| 473 | pm = Progress('Checking out', len(all_projects), quiet=opt.quiet) | 471 | def _ProcessResults(pool, pm, results): |
| 474 | 472 | ret = True | |
| 475 | def _ProcessResults(results): | ||
| 476 | for (success, project, start, finish) in results: | 473 | for (success, project, start, finish) in results: |
| 477 | self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, | 474 | self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, |
| 478 | start, finish, success) | 475 | start, finish, success) |
| 479 | # Check for any errors before running any more tasks. | 476 | # Check for any errors before running any more tasks. |
| 480 | # ...we'll let existing jobs finish, though. | 477 | # ...we'll let existing jobs finish, though. |
| 481 | if not success: | 478 | if not success: |
| 479 | ret = False | ||
| 482 | err_results.append(project.relpath) | 480 | err_results.append(project.relpath) |
| 483 | if opt.fail_fast: | 481 | if opt.fail_fast: |
| 484 | return False | 482 | if pool: |
| 483 | pool.close() | ||
| 484 | return ret | ||
| 485 | pm.update(msg=project.name) | 485 | pm.update(msg=project.name) |
| 486 | return True | 486 | return ret |
| 487 | |||
| 488 | # NB: Multiprocessing is heavy, so don't spin it up for one job. | ||
| 489 | if len(all_projects) == 1 or jobs == 1: | ||
| 490 | if not _ProcessResults(self._CheckoutOne(opt, x) for x in all_projects): | ||
| 491 | ret = False | ||
| 492 | else: | ||
| 493 | with multiprocessing.Pool(jobs) as pool: | ||
| 494 | results = pool.imap_unordered( | ||
| 495 | functools.partial(self._CheckoutOne, opt), | ||
| 496 | all_projects, | ||
| 497 | chunksize=WORKER_BATCH_SIZE) | ||
| 498 | if not _ProcessResults(results): | ||
| 499 | ret = False | ||
| 500 | pool.close() | ||
| 501 | |||
| 502 | pm.end() | ||
| 503 | 487 | ||
| 504 | return ret and not err_results | 488 | return self.ExecuteInParallel( |
| 489 | opt.jobs_checkout if opt.jobs_checkout else self.jobs, | ||
| 490 | functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync), | ||
| 491 | all_projects, | ||
| 492 | callback=_ProcessResults, | ||
| 493 | output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results | ||
| 505 | 494 | ||
| 506 | def _GCProjects(self, projects, opt, err_event): | 495 | def _GCProjects(self, projects, opt, err_event): |
| 507 | pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet) | 496 | pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet) |
