diff options
| -rw-r--r-- | progress.py | 49 | ||||
| -rw-r--r-- | subcmds/abandon.py | 2 | ||||
| -rw-r--r-- | subcmds/checkout.py | 2 | ||||
| -rw-r--r-- | subcmds/start.py | 4 | ||||
| -rw-r--r-- | subcmds/sync.py | 39 |
5 files changed, 70 insertions, 26 deletions
diff --git a/progress.py b/progress.py index 4844eb88..6686ad4a 100644 --- a/progress.py +++ b/progress.py | |||
| @@ -82,10 +82,10 @@ class Progress(object): | |||
| 82 | title, | 82 | title, |
| 83 | total=0, | 83 | total=0, |
| 84 | units="", | 84 | units="", |
| 85 | print_newline=False, | ||
| 86 | delay=True, | 85 | delay=True, |
| 87 | quiet=False, | 86 | quiet=False, |
| 88 | show_elapsed=False, | 87 | show_elapsed=False, |
| 88 | elide=False, | ||
| 89 | ): | 89 | ): |
| 90 | self._title = title | 90 | self._title = title |
| 91 | self._total = total | 91 | self._total = total |
| @@ -93,7 +93,7 @@ class Progress(object): | |||
| 93 | self._start = time.time() | 93 | self._start = time.time() |
| 94 | self._show = not delay | 94 | self._show = not delay |
| 95 | self._units = units | 95 | self._units = units |
| 96 | self._print_newline = print_newline | 96 | self._elide = elide |
| 97 | # Only show the active jobs section if we run more than one in parallel. | 97 | # Only show the active jobs section if we run more than one in parallel. |
| 98 | self._show_jobs = False | 98 | self._show_jobs = False |
| 99 | self._active = 0 | 99 | self._active = 0 |
| @@ -118,10 +118,18 @@ class Progress(object): | |||
| 118 | 118 | ||
| 119 | def _update_loop(self): | 119 | def _update_loop(self): |
| 120 | while True: | 120 | while True: |
| 121 | if self._update_event.is_set(): | 121 | self.update(inc=0) |
| 122 | if self._update_event.wait(timeout=1): | ||
| 122 | return | 123 | return |
| 123 | self.update(inc=0, msg=self._last_msg) | 124 | |
| 124 | time.sleep(1) | 125 | def _write(self, s): |
| 126 | s = "\r" + s | ||
| 127 | if self._elide: | ||
| 128 | col = os.get_terminal_size().columns | ||
| 129 | if len(s) > col: | ||
| 130 | s = s[: col - 1] + ".." | ||
| 131 | sys.stderr.write(s) | ||
| 132 | sys.stderr.flush() | ||
| 125 | 133 | ||
| 126 | def start(self, name): | 134 | def start(self, name): |
| 127 | self._active += 1 | 135 | self._active += 1 |
| @@ -133,8 +141,16 @@ class Progress(object): | |||
| 133 | self.update(msg="finished " + name) | 141 | self.update(msg="finished " + name) |
| 134 | self._active -= 1 | 142 | self._active -= 1 |
| 135 | 143 | ||
| 136 | def update(self, inc=1, msg=""): | 144 | def update(self, inc=1, msg=None): |
| 145 | """Updates the progress indicator. | ||
| 146 | |||
| 147 | Args: | ||
| 148 | inc: The number of items completed. | ||
| 149 | msg: The message to display. If None, use the last message. | ||
| 150 | """ | ||
| 137 | self._done += inc | 151 | self._done += inc |
| 152 | if msg is None: | ||
| 153 | msg = self._last_msg | ||
| 138 | self._last_msg = msg | 154 | self._last_msg = msg |
| 139 | 155 | ||
| 140 | if _NOT_TTY or IsTraceToStderr(): | 156 | if _NOT_TTY or IsTraceToStderr(): |
| @@ -148,10 +164,9 @@ class Progress(object): | |||
| 148 | return | 164 | return |
| 149 | 165 | ||
| 150 | if self._total <= 0: | 166 | if self._total <= 0: |
| 151 | sys.stderr.write( | 167 | self._write( |
| 152 | "\r%s: %d,%s" % (self._title, self._done, CSI_ERASE_LINE_AFTER) | 168 | "%s: %d,%s" % (self._title, self._done, CSI_ERASE_LINE_AFTER) |
| 153 | ) | 169 | ) |
| 154 | sys.stderr.flush() | ||
| 155 | else: | 170 | else: |
| 156 | p = (100 * self._done) / self._total | 171 | p = (100 * self._done) / self._total |
| 157 | if self._show_jobs: | 172 | if self._show_jobs: |
| @@ -165,8 +180,8 @@ class Progress(object): | |||
| 165 | elapsed = f" {elapsed_str(elapsed_sec)} |" | 180 | elapsed = f" {elapsed_str(elapsed_sec)} |" |
| 166 | else: | 181 | else: |
| 167 | elapsed = "" | 182 | elapsed = "" |
| 168 | sys.stderr.write( | 183 | self._write( |
| 169 | "\r%s: %2d%% %s(%d%s/%d%s)%s %s%s%s" | 184 | "%s: %2d%% %s(%d%s/%d%s)%s %s%s" |
| 170 | % ( | 185 | % ( |
| 171 | self._title, | 186 | self._title, |
| 172 | p, | 187 | p, |
| @@ -178,10 +193,8 @@ class Progress(object): | |||
| 178 | elapsed, | 193 | elapsed, |
| 179 | msg, | 194 | msg, |
| 180 | CSI_ERASE_LINE_AFTER, | 195 | CSI_ERASE_LINE_AFTER, |
| 181 | "\n" if self._print_newline else "", | ||
| 182 | ) | 196 | ) |
| 183 | ) | 197 | ) |
| 184 | sys.stderr.flush() | ||
| 185 | 198 | ||
| 186 | def end(self): | 199 | def end(self): |
| 187 | self._update_event.set() | 200 | self._update_event.set() |
| @@ -190,15 +203,14 @@ class Progress(object): | |||
| 190 | 203 | ||
| 191 | duration = duration_str(time.time() - self._start) | 204 | duration = duration_str(time.time() - self._start) |
| 192 | if self._total <= 0: | 205 | if self._total <= 0: |
| 193 | sys.stderr.write( | 206 | self._write( |
| 194 | "\r%s: %d, done in %s%s\n" | 207 | "%s: %d, done in %s%s\n" |
| 195 | % (self._title, self._done, duration, CSI_ERASE_LINE_AFTER) | 208 | % (self._title, self._done, duration, CSI_ERASE_LINE_AFTER) |
| 196 | ) | 209 | ) |
| 197 | sys.stderr.flush() | ||
| 198 | else: | 210 | else: |
| 199 | p = (100 * self._done) / self._total | 211 | p = (100 * self._done) / self._total |
| 200 | sys.stderr.write( | 212 | self._write( |
| 201 | "\r%s: %3d%% (%d%s/%d%s), done in %s%s\n" | 213 | "%s: %3d%% (%d%s/%d%s), done in %s%s\n" |
| 202 | % ( | 214 | % ( |
| 203 | self._title, | 215 | self._title, |
| 204 | p, | 216 | p, |
| @@ -210,4 +222,3 @@ class Progress(object): | |||
| 210 | CSI_ERASE_LINE_AFTER, | 222 | CSI_ERASE_LINE_AFTER, |
| 211 | ) | 223 | ) |
| 212 | ) | 224 | ) |
| 213 | sys.stderr.flush() | ||
diff --git a/subcmds/abandon.py b/subcmds/abandon.py index 4036f306..ded287f6 100644 --- a/subcmds/abandon.py +++ b/subcmds/abandon.py | |||
| @@ -90,7 +90,7 @@ It is equivalent to "git branch -D <branchname>". | |||
| 90 | success[branch].append(project) | 90 | success[branch].append(project) |
| 91 | else: | 91 | else: |
| 92 | err[branch].append(project) | 92 | err[branch].append(project) |
| 93 | pm.update() | 93 | pm.update(msg="") |
| 94 | 94 | ||
| 95 | self.ExecuteInParallel( | 95 | self.ExecuteInParallel( |
| 96 | opt.jobs, | 96 | opt.jobs, |
diff --git a/subcmds/checkout.py b/subcmds/checkout.py index 08012a82..6448518f 100644 --- a/subcmds/checkout.py +++ b/subcmds/checkout.py | |||
| @@ -58,7 +58,7 @@ The command is equivalent to: | |||
| 58 | success.append(project) | 58 | success.append(project) |
| 59 | else: | 59 | else: |
| 60 | err.append(project) | 60 | err.append(project) |
| 61 | pm.update() | 61 | pm.update(msg="") |
| 62 | 62 | ||
| 63 | self.ExecuteInParallel( | 63 | self.ExecuteInParallel( |
| 64 | opt.jobs, | 64 | opt.jobs, |
diff --git a/subcmds/start.py b/subcmds/start.py index 9baf4256..f6355126 100644 --- a/subcmds/start.py +++ b/subcmds/start.py | |||
| @@ -142,14 +142,14 @@ revision specified in the manifest. | |||
| 142 | sync_buf = SyncBuffer(self.manifest.manifestProject.config) | 142 | sync_buf = SyncBuffer(self.manifest.manifestProject.config) |
| 143 | project.Sync_LocalHalf(sync_buf) | 143 | project.Sync_LocalHalf(sync_buf) |
| 144 | project.revisionId = gitc_project.old_revision | 144 | project.revisionId = gitc_project.old_revision |
| 145 | pm.update() | 145 | pm.update(msg="") |
| 146 | pm.end() | 146 | pm.end() |
| 147 | 147 | ||
| 148 | def _ProcessResults(_pool, pm, results): | 148 | def _ProcessResults(_pool, pm, results): |
| 149 | for result, project in results: | 149 | for result, project in results: |
| 150 | if not result: | 150 | if not result: |
| 151 | err.append(project) | 151 | err.append(project) |
| 152 | pm.update() | 152 | pm.update(msg="") |
| 153 | 153 | ||
| 154 | self.ExecuteInParallel( | 154 | self.ExecuteInParallel( |
| 155 | opt.jobs, | 155 | opt.jobs, |
diff --git a/subcmds/sync.py b/subcmds/sync.py index 8f73d27f..da9918b9 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
| @@ -66,7 +66,7 @@ from command import ( | |||
| 66 | from error import RepoChangedException, GitError | 66 | from error import RepoChangedException, GitError |
| 67 | import platform_utils | 67 | import platform_utils |
| 68 | from project import SyncBuffer | 68 | from project import SyncBuffer |
| 69 | from progress import Progress | 69 | from progress import Progress, elapsed_str |
| 70 | from repo_trace import Trace | 70 | from repo_trace import Trace |
| 71 | import ssh | 71 | import ssh |
| 72 | from wrapper import Wrapper | 72 | from wrapper import Wrapper |
| @@ -596,7 +596,7 @@ later is required to fix a server side protocol bug. | |||
| 596 | The projects we're given share the same underlying git object store, so | 596 | The projects we're given share the same underlying git object store, so |
| 597 | we have to fetch them in serial. | 597 | we have to fetch them in serial. |
| 598 | 598 | ||
| 599 | Delegates most of the work to _FetchHelper. | 599 | Delegates most of the work to _FetchOne. |
| 600 | 600 | ||
| 601 | Args: | 601 | Args: |
| 602 | opt: Program options returned from optparse. See _Options(). | 602 | opt: Program options returned from optparse. See _Options(). |
| @@ -615,6 +615,8 @@ later is required to fix a server side protocol bug. | |||
| 615 | Whether the fetch was successful. | 615 | Whether the fetch was successful. |
| 616 | """ | 616 | """ |
| 617 | start = time.time() | 617 | start = time.time() |
| 618 | k = f"{project.name} @ {project.relpath}" | ||
| 619 | self._sync_dict[k] = start | ||
| 618 | success = False | 620 | success = False |
| 619 | remote_fetched = False | 621 | remote_fetched = False |
| 620 | buf = io.StringIO() | 622 | buf = io.StringIO() |
| @@ -660,15 +662,31 @@ later is required to fix a server side protocol bug. | |||
| 660 | % (project.name, type(e).__name__, str(e)), | 662 | % (project.name, type(e).__name__, str(e)), |
| 661 | file=sys.stderr, | 663 | file=sys.stderr, |
| 662 | ) | 664 | ) |
| 665 | del self._sync_dict[k] | ||
| 663 | raise | 666 | raise |
| 664 | 667 | ||
| 665 | finish = time.time() | 668 | finish = time.time() |
| 669 | del self._sync_dict[k] | ||
| 666 | return _FetchOneResult(success, project, start, finish, remote_fetched) | 670 | return _FetchOneResult(success, project, start, finish, remote_fetched) |
| 667 | 671 | ||
| 668 | @classmethod | 672 | @classmethod |
| 669 | def _FetchInitChild(cls, ssh_proxy): | 673 | def _FetchInitChild(cls, ssh_proxy): |
| 670 | cls.ssh_proxy = ssh_proxy | 674 | cls.ssh_proxy = ssh_proxy |
| 671 | 675 | ||
| 676 | def _GetLongestSyncMessage(self): | ||
| 677 | if len(self._sync_dict) == 0: | ||
| 678 | return None | ||
| 679 | |||
| 680 | earliest_time = float("inf") | ||
| 681 | earliest_proj = None | ||
| 682 | for project, t in self._sync_dict.items(): | ||
| 683 | if t < earliest_time: | ||
| 684 | earliest_time = t | ||
| 685 | earliest_proj = project | ||
| 686 | |||
| 687 | elapsed = time.time() - earliest_time | ||
| 688 | return f"{elapsed_str(elapsed)} {earliest_proj}" | ||
| 689 | |||
| 672 | def _Fetch(self, projects, opt, err_event, ssh_proxy): | 690 | def _Fetch(self, projects, opt, err_event, ssh_proxy): |
| 673 | ret = True | 691 | ret = True |
| 674 | 692 | ||
| @@ -681,8 +699,22 @@ later is required to fix a server side protocol bug. | |||
| 681 | delay=False, | 699 | delay=False, |
| 682 | quiet=opt.quiet, | 700 | quiet=opt.quiet, |
| 683 | show_elapsed=True, | 701 | show_elapsed=True, |
| 702 | elide=True, | ||
| 684 | ) | 703 | ) |
| 685 | 704 | ||
| 705 | self._sync_dict = multiprocessing.Manager().dict() | ||
| 706 | sync_event = _threading.Event() | ||
| 707 | |||
| 708 | def _MonitorSyncLoop(): | ||
| 709 | while True: | ||
| 710 | pm.update(inc=0, msg=self._GetLongestSyncMessage()) | ||
| 711 | if sync_event.wait(timeout=1): | ||
| 712 | return | ||
| 713 | |||
| 714 | sync_progress_thread = _threading.Thread(target=_MonitorSyncLoop) | ||
| 715 | sync_progress_thread.daemon = True | ||
| 716 | sync_progress_thread.start() | ||
| 717 | |||
| 686 | objdir_project_map = dict() | 718 | objdir_project_map = dict() |
| 687 | for project in projects: | 719 | for project in projects: |
| 688 | objdir_project_map.setdefault(project.objdir, []).append(project) | 720 | objdir_project_map.setdefault(project.objdir, []).append(project) |
| @@ -712,7 +744,7 @@ later is required to fix a server side protocol bug. | |||
| 712 | ret = False | 744 | ret = False |
| 713 | else: | 745 | else: |
| 714 | fetched.add(project.gitdir) | 746 | fetched.add(project.gitdir) |
| 715 | pm.update(msg=f"Last synced: {project.name}") | 747 | pm.update() |
| 716 | if not ret and opt.fail_fast: | 748 | if not ret and opt.fail_fast: |
| 717 | break | 749 | break |
| 718 | return ret | 750 | return ret |
| @@ -764,6 +796,7 @@ later is required to fix a server side protocol bug. | |||
| 764 | # crash. | 796 | # crash. |
| 765 | del Sync.ssh_proxy | 797 | del Sync.ssh_proxy |
| 766 | 798 | ||
| 799 | sync_event.set() | ||
| 767 | pm.end() | 800 | pm.end() |
| 768 | self._fetch_times.Save() | 801 | self._fetch_times.Save() |
| 769 | 802 | ||
