diff options
| -rw-r--r-- | command.py | 9 | ||||
| -rw-r--r-- | subcmds/branches.py | 10 | ||||
| -rw-r--r-- | subcmds/status.py | 16 |
3 files changed, 22 insertions, 13 deletions
| @@ -23,6 +23,15 @@ from error import NoSuchProjectError | |||
| 23 | from error import InvalidProjectGroupsError | 23 | from error import InvalidProjectGroupsError |
| 24 | 24 | ||
| 25 | 25 | ||
| 26 | # Number of projects to submit to a single worker process at a time. | ||
| 27 | # This number represents a tradeoff between the overhead of IPC and finer | ||
| 28 | # grained opportunity for parallelism. This particular value was chosen by | ||
| 29 | # iterating through powers of two until the overall performance no longer | ||
| 30 | # improved. The performance of this batch size is not a function of the | ||
| 31 | # number of cores on the system. | ||
| 32 | WORKER_BATCH_SIZE = 32 | ||
| 33 | |||
| 34 | |||
| 26 | # How many jobs to run in parallel by default? This assumes the jobs are | 35 | # How many jobs to run in parallel by default? This assumes the jobs are |
| 27 | # largely I/O bound and do not hit the network. | 36 | # largely I/O bound and do not hit the network. |
| 28 | DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8) | 37 | DEFAULT_LOCAL_JOBS = min(os.cpu_count(), 8) |
diff --git a/subcmds/branches.py b/subcmds/branches.py index 9665e85f..d5ea580c 100644 --- a/subcmds/branches.py +++ b/subcmds/branches.py | |||
| @@ -16,15 +16,7 @@ import itertools | |||
| 16 | import multiprocessing | 16 | import multiprocessing |
| 17 | import sys | 17 | import sys |
| 18 | from color import Coloring | 18 | from color import Coloring |
| 19 | from command import Command, DEFAULT_LOCAL_JOBS | 19 | from command import Command, DEFAULT_LOCAL_JOBS, WORKER_BATCH_SIZE |
| 20 | |||
| 21 | # Number of projects to submit to a single worker process at a time. | ||
| 22 | # This number represents a tradeoff between the overhead of IPC and finer | ||
| 23 | # grained opportunity for parallelism. This particular value was chosen by | ||
| 24 | # iterating through powers of two until the overall performance no longer | ||
| 25 | # improved. The performance of this batch size is not a function of the | ||
| 26 | # number of cores on the system. | ||
| 27 | WORKER_BATCH_SIZE = 32 | ||
| 28 | 20 | ||
| 29 | 21 | ||
| 30 | class BranchColoring(Coloring): | 22 | class BranchColoring(Coloring): |
diff --git a/subcmds/status.py b/subcmds/status.py index f0f2e034..6c8e22e5 100644 --- a/subcmds/status.py +++ b/subcmds/status.py | |||
| @@ -14,10 +14,11 @@ | |||
| 14 | 14 | ||
| 15 | import functools | 15 | import functools |
| 16 | import glob | 16 | import glob |
| 17 | import io | ||
| 17 | import multiprocessing | 18 | import multiprocessing |
| 18 | import os | 19 | import os |
| 19 | 20 | ||
| 20 | from command import DEFAULT_LOCAL_JOBS, PagedCommand | 21 | from command import DEFAULT_LOCAL_JOBS, PagedCommand, WORKER_BATCH_SIZE |
| 21 | 22 | ||
| 22 | from color import Coloring | 23 | from color import Coloring |
| 23 | import platform_utils | 24 | import platform_utils |
| @@ -99,7 +100,9 @@ the following meanings: | |||
| 99 | Returns: | 100 | Returns: |
| 100 | The status of the project. | 101 | The status of the project. |
| 101 | """ | 102 | """ |
| 102 | return project.PrintWorkTreeStatus(quiet=quiet) | 103 | buf = io.StringIO() |
| 104 | ret = project.PrintWorkTreeStatus(quiet=quiet, output_redir=buf) | ||
| 105 | return (ret, buf.getvalue()) | ||
| 103 | 106 | ||
| 104 | def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring): | 107 | def _FindOrphans(self, dirs, proj_dirs, proj_dirs_parents, outstring): |
| 105 | """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'""" | 108 | """find 'dirs' that are present in 'proj_dirs_parents' but not in 'proj_dirs'""" |
| @@ -128,8 +131,13 @@ the following meanings: | |||
| 128 | counter += 1 | 131 | counter += 1 |
| 129 | else: | 132 | else: |
| 130 | with multiprocessing.Pool(opt.jobs) as pool: | 133 | with multiprocessing.Pool(opt.jobs) as pool: |
| 131 | states = pool.map(functools.partial(self._StatusHelper, opt.quiet), all_projects) | 134 | states = pool.imap(functools.partial(self._StatusHelper, opt.quiet), |
| 132 | counter += states.count('CLEAN') | 135 | all_projects, chunksize=WORKER_BATCH_SIZE) |
| 136 | for (state, output) in states: | ||
| 137 | if output: | ||
| 138 | print(output, end='') | ||
| 139 | if state == 'CLEAN': | ||
| 140 | counter += 1 | ||
| 133 | if not opt.quiet and len(all_projects) == counter: | 141 | if not opt.quiet and len(all_projects) == counter: |
| 134 | print('nothing to commit (working directory clean)') | 142 | print('nothing to commit (working directory clean)') |
| 135 | 143 | ||
