diff options
Diffstat (limited to 'subcmds')
| -rw-r--r-- | subcmds/selfupdate.py | 2 | ||||
| -rw-r--r-- | subcmds/sync.py | 95 |
2 files changed, 84 insertions, 13 deletions
diff --git a/subcmds/selfupdate.py b/subcmds/selfupdate.py index 282f518e..898bc3f2 100644 --- a/subcmds/selfupdate.py +++ b/subcmds/selfupdate.py | |||
| @@ -51,7 +51,7 @@ need to be performed by an end-user. | |||
| 51 | _PostRepoUpgrade(self.manifest) | 51 | _PostRepoUpgrade(self.manifest) |
| 52 | 52 | ||
| 53 | else: | 53 | else: |
| 54 | if not rp.Sync_NetworkHalf(): | 54 | if not rp.Sync_NetworkHalf().success: |
| 55 | print("error: can't update repo", file=sys.stderr) | 55 | print("error: can't update repo", file=sys.stderr) |
| 56 | sys.exit(1) | 56 | sys.exit(1) |
| 57 | 57 | ||
diff --git a/subcmds/sync.py b/subcmds/sync.py index 9e9c8f02..1c49b46e 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
| @@ -26,6 +26,7 @@ import socket | |||
| 26 | import sys | 26 | import sys |
| 27 | import tempfile | 27 | import tempfile |
| 28 | import time | 28 | import time |
| 29 | from typing import NamedTuple | ||
| 29 | import urllib.error | 30 | import urllib.error |
| 30 | import urllib.parse | 31 | import urllib.parse |
| 31 | import urllib.request | 32 | import urllib.request |
| @@ -71,6 +72,58 @@ REPO_BACKUP_OBJECTS = 'REPO_BACKUP_OBJECTS' | |||
| 71 | _BACKUP_OBJECTS = os.environ.get(REPO_BACKUP_OBJECTS) != '0' | 72 | _BACKUP_OBJECTS = os.environ.get(REPO_BACKUP_OBJECTS) != '0' |
| 72 | 73 | ||
| 73 | 74 | ||
| 75 | class _FetchOneResult(NamedTuple): | ||
| 76 | """_FetchOne return value. | ||
| 77 | |||
| 78 | Attributes: | ||
| 79 | success (bool): True if successful. | ||
| 80 | project (Project): The fetched project. | ||
| 81 | start (float): The starting time.time(). | ||
| 82 | finish (float): The ending time.time(). | ||
| 83 | remote_fetched (bool): True if the remote was actually queried. | ||
| 84 | """ | ||
| 85 | success: bool | ||
| 86 | project: Project | ||
| 87 | start: float | ||
| 88 | finish: float | ||
| 89 | remote_fetched: bool | ||
| 90 | |||
| 91 | |||
| 92 | class _FetchResult(NamedTuple): | ||
| 93 | """_Fetch return value. | ||
| 94 | |||
| 95 | Attributes: | ||
| 96 | success (bool): True if successful. | ||
| 97 | projects (set[str]): The names of the git directories of fetched projects. | ||
| 98 | """ | ||
| 99 | success: bool | ||
| 100 | projects: set[str] | ||
| 101 | |||
| 102 | |||
| 103 | class _FetchMainResult(NamedTuple): | ||
| 104 | """_FetchMain return value. | ||
| 105 | |||
| 106 | Attributes: | ||
| 107 | all_projects (list[Project]): The fetched projects. | ||
| 108 | """ | ||
| 109 | all_projects: list[Project] | ||
| 110 | |||
| 111 | |||
| 112 | class _CheckoutOneResult(NamedTuple): | ||
| 113 | """_CheckoutOne return value. | ||
| 114 | |||
| 115 | Attributes: | ||
| 116 | success (bool): True if successful. | ||
| 117 | project (Project): The project. | ||
| 118 | start (float): The starting time.time(). | ||
| 119 | finish (float): The ending time.time(). | ||
| 120 | """ | ||
| 121 | success: bool | ||
| 122 | project: Project | ||
| 123 | start: float | ||
| 124 | finish: float | ||
| 125 | |||
| 126 | |||
| 74 | class Sync(Command, MirrorSafeCommand): | 127 | class Sync(Command, MirrorSafeCommand): |
| 75 | COMMON = True | 128 | COMMON = True |
| 76 | MULTI_MANIFEST_SUPPORT = True | 129 | MULTI_MANIFEST_SUPPORT = True |
| @@ -412,7 +465,7 @@ later is required to fix a server side protocol bug. | |||
| 412 | success = False | 465 | success = False |
| 413 | buf = io.StringIO() | 466 | buf = io.StringIO() |
| 414 | try: | 467 | try: |
| 415 | success = project.Sync_NetworkHalf( | 468 | sync_result = project.Sync_NetworkHalf( |
| 416 | quiet=opt.quiet, | 469 | quiet=opt.quiet, |
| 417 | verbose=opt.verbose, | 470 | verbose=opt.verbose, |
| 418 | output_redir=buf, | 471 | output_redir=buf, |
| @@ -426,6 +479,7 @@ later is required to fix a server side protocol bug. | |||
| 426 | ssh_proxy=self.ssh_proxy, | 479 | ssh_proxy=self.ssh_proxy, |
| 427 | clone_filter=project.manifest.CloneFilter, | 480 | clone_filter=project.manifest.CloneFilter, |
| 428 | partial_clone_exclude=project.manifest.PartialCloneExclude) | 481 | partial_clone_exclude=project.manifest.PartialCloneExclude) |
| 482 | success = sync_result.success | ||
| 429 | 483 | ||
| 430 | output = buf.getvalue() | 484 | output = buf.getvalue() |
| 431 | if (opt.verbose or not success) and output: | 485 | if (opt.verbose or not success) and output: |
| @@ -443,7 +497,8 @@ later is required to fix a server side protocol bug. | |||
| 443 | raise | 497 | raise |
| 444 | 498 | ||
| 445 | finish = time.time() | 499 | finish = time.time() |
| 446 | return (success, project, start, finish) | 500 | return _FetchOneResult(success, project, start, finish, |
| 501 | sync_result.remote_fetched) | ||
| 447 | 502 | ||
| 448 | @classmethod | 503 | @classmethod |
| 449 | def _FetchInitChild(cls, ssh_proxy): | 504 | def _FetchInitChild(cls, ssh_proxy): |
| @@ -454,6 +509,7 @@ later is required to fix a server side protocol bug. | |||
| 454 | 509 | ||
| 455 | jobs = opt.jobs_network | 510 | jobs = opt.jobs_network |
| 456 | fetched = set() | 511 | fetched = set() |
| 512 | remote_fetched = set() | ||
| 457 | pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet) | 513 | pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet) |
| 458 | 514 | ||
| 459 | objdir_project_map = dict() | 515 | objdir_project_map = dict() |
| @@ -464,10 +520,16 @@ later is required to fix a server side protocol bug. | |||
| 464 | def _ProcessResults(results_sets): | 520 | def _ProcessResults(results_sets): |
| 465 | ret = True | 521 | ret = True |
| 466 | for results in results_sets: | 522 | for results in results_sets: |
| 467 | for (success, project, start, finish) in results: | 523 | for result in results: |
| 524 | success = result.success | ||
| 525 | project = result.project | ||
| 526 | start = result.start | ||
| 527 | finish = result.finish | ||
| 468 | self._fetch_times.Set(project, finish - start) | 528 | self._fetch_times.Set(project, finish - start) |
| 469 | self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK, | 529 | self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK, |
| 470 | start, finish, success) | 530 | start, finish, success) |
| 531 | if result.remote_fetched: | ||
| 532 | remote_fetched.add(project) | ||
| 471 | # Check for any errors before running any more tasks. | 533 | # Check for any errors before running any more tasks. |
| 472 | # ...we'll let existing jobs finish, though. | 534 | # ...we'll let existing jobs finish, though. |
| 473 | if not success: | 535 | if not success: |
| @@ -525,7 +587,7 @@ later is required to fix a server side protocol bug. | |||
| 525 | if not self.outer_client.manifest.IsArchive: | 587 | if not self.outer_client.manifest.IsArchive: |
| 526 | self._GCProjects(projects, opt, err_event) | 588 | self._GCProjects(projects, opt, err_event) |
| 527 | 589 | ||
| 528 | return (ret, fetched) | 590 | return _FetchResult(ret, fetched) |
| 529 | 591 | ||
| 530 | def _FetchMain(self, opt, args, all_projects, err_event, | 592 | def _FetchMain(self, opt, args, all_projects, err_event, |
| 531 | ssh_proxy, manifest): | 593 | ssh_proxy, manifest): |
| @@ -551,7 +613,9 @@ later is required to fix a server side protocol bug. | |||
| 551 | to_fetch.extend(all_projects) | 613 | to_fetch.extend(all_projects) |
| 552 | to_fetch.sort(key=self._fetch_times.Get, reverse=True) | 614 | to_fetch.sort(key=self._fetch_times.Get, reverse=True) |
| 553 | 615 | ||
| 554 | success, fetched = self._Fetch(to_fetch, opt, err_event, ssh_proxy) | 616 | result = self._Fetch(to_fetch, opt, err_event, ssh_proxy) |
| 617 | success = result.success | ||
| 618 | fetched = result.projects | ||
| 555 | if not success: | 619 | if not success: |
| 556 | err_event.set() | 620 | err_event.set() |
| 557 | 621 | ||
| @@ -561,7 +625,7 @@ later is required to fix a server side protocol bug. | |||
| 561 | if err_event.is_set(): | 625 | if err_event.is_set(): |
| 562 | print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr) | 626 | print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr) |
| 563 | sys.exit(1) | 627 | sys.exit(1) |
| 564 | return | 628 | return _FetchMainResult([]) |
| 565 | 629 | ||
| 566 | # Iteratively fetch missing and/or nested unregistered submodules | 630 | # Iteratively fetch missing and/or nested unregistered submodules |
| 567 | previously_missing_set = set() | 631 | previously_missing_set = set() |
| @@ -584,12 +648,14 @@ later is required to fix a server side protocol bug. | |||
| 584 | if previously_missing_set == missing_set: | 648 | if previously_missing_set == missing_set: |
| 585 | break | 649 | break |
| 586 | previously_missing_set = missing_set | 650 | previously_missing_set = missing_set |
| 587 | success, new_fetched = self._Fetch(missing, opt, err_event, ssh_proxy) | 651 | result = self._Fetch(missing, opt, err_event, ssh_proxy) |
| 652 | success = result.success | ||
| 653 | new_fetched = result.projects | ||
| 588 | if not success: | 654 | if not success: |
| 589 | err_event.set() | 655 | err_event.set() |
| 590 | fetched.update(new_fetched) | 656 | fetched.update(new_fetched) |
| 591 | 657 | ||
| 592 | return all_projects | 658 | return _FetchMainResult(all_projects) |
| 593 | 659 | ||
| 594 | def _CheckoutOne(self, detach_head, force_sync, project): | 660 | def _CheckoutOne(self, detach_head, force_sync, project): |
| 595 | """Checkout work tree for one project | 661 | """Checkout work tree for one project |
| @@ -621,7 +687,7 @@ later is required to fix a server side protocol bug. | |||
| 621 | if not success: | 687 | if not success: |
| 622 | print('error: Cannot checkout %s' % (project.name), file=sys.stderr) | 688 | print('error: Cannot checkout %s' % (project.name), file=sys.stderr) |
| 623 | finish = time.time() | 689 | finish = time.time() |
| 624 | return (success, project, start, finish) | 690 | return _CheckoutOneResult(success, project, start, finish) |
| 625 | 691 | ||
| 626 | def _Checkout(self, all_projects, opt, err_results): | 692 | def _Checkout(self, all_projects, opt, err_results): |
| 627 | """Checkout projects listed in all_projects | 693 | """Checkout projects listed in all_projects |
| @@ -636,7 +702,11 @@ later is required to fix a server side protocol bug. | |||
| 636 | 702 | ||
| 637 | def _ProcessResults(pool, pm, results): | 703 | def _ProcessResults(pool, pm, results): |
| 638 | ret = True | 704 | ret = True |
| 639 | for (success, project, start, finish) in results: | 705 | for result in results: |
| 706 | success = result.success | ||
| 707 | project = result.project | ||
| 708 | start = result.start | ||
| 709 | finish = result.finish | ||
| 640 | self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, | 710 | self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL, |
| 641 | start, finish, success) | 711 | start, finish, success) |
| 642 | # Check for any errors before running any more tasks. | 712 | # Check for any errors before running any more tasks. |
| @@ -1208,8 +1278,9 @@ later is required to fix a server side protocol bug. | |||
| 1208 | with ssh.ProxyManager(manager) as ssh_proxy: | 1278 | with ssh.ProxyManager(manager) as ssh_proxy: |
| 1209 | # Initialize the socket dir once in the parent. | 1279 | # Initialize the socket dir once in the parent. |
| 1210 | ssh_proxy.sock() | 1280 | ssh_proxy.sock() |
| 1211 | all_projects = self._FetchMain(opt, args, all_projects, err_event, | 1281 | result = self._FetchMain(opt, args, all_projects, err_event, |
| 1212 | ssh_proxy, manifest) | 1282 | ssh_proxy, manifest) |
| 1283 | all_projects = result.all_projects | ||
| 1213 | 1284 | ||
| 1214 | if opt.network_only: | 1285 | if opt.network_only: |
| 1215 | return | 1286 | return |
