diff options
Diffstat (limited to 'subcmds')
| -rw-r--r-- | subcmds/sync.py | 83 |
1 files changed, 34 insertions, 49 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py index 27c8c728..a0a68960 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
| @@ -219,7 +219,7 @@ later is required to fix a server side protocol bug. | |||
| 219 | dest='repo_upgraded', action='store_true', | 219 | dest='repo_upgraded', action='store_true', |
| 220 | help=SUPPRESS_HELP) | 220 | help=SUPPRESS_HELP) |
| 221 | 221 | ||
| 222 | def _FetchProjectList(self, opt, projects, *args): | 222 | def _FetchProjectList(self, opt, projects, *args, **kwargs): |
| 223 | """Main function of the fetch threads when jobs are > 1. | 223 | """Main function of the fetch threads when jobs are > 1. |
| 224 | 224 | ||
| 225 | Delegates most of the work to _FetchHelper. | 225 | Delegates most of the work to _FetchHelper. |
| @@ -227,11 +227,11 @@ later is required to fix a server side protocol bug. | |||
| 227 | Args: | 227 | Args: |
| 228 | opt: Program options returned from optparse. See _Options(). | 228 | opt: Program options returned from optparse. See _Options(). |
| 229 | projects: Projects to fetch. | 229 | projects: Projects to fetch. |
| 230 | *args: Remaining arguments to pass to _FetchHelper. See the | 230 | *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the |
| 231 | _FetchHelper docstring for details. | 231 | _FetchHelper docstring for details. |
| 232 | """ | 232 | """ |
| 233 | for project in projects: | 233 | for project in projects: |
| 234 | success = self._FetchHelper(opt, project, *args) | 234 | success = self._FetchHelper(opt, project, *args, **kwargs) |
| 235 | if not success and not opt.force_broken: | 235 | if not success and not opt.force_broken: |
| 236 | break | 236 | break |
| 237 | 237 | ||
| @@ -304,62 +304,47 @@ later is required to fix a server side protocol bug. | |||
| 304 | 304 | ||
| 305 | def _Fetch(self, projects, opt): | 305 | def _Fetch(self, projects, opt): |
| 306 | fetched = set() | 306 | fetched = set() |
| 307 | lock = _threading.Lock() | ||
| 307 | pm = Progress('Fetching projects', len(projects)) | 308 | pm = Progress('Fetching projects', len(projects)) |
| 308 | 309 | ||
| 309 | if self.jobs == 1: | 310 | objdir_project_map = dict() |
| 310 | for project in projects: | 311 | for project in projects: |
| 311 | pm.update() | 312 | objdir_project_map.setdefault(project.objdir, []).append(project) |
| 312 | if not opt.quiet: | 313 | |
| 313 | print('Fetching project %s' % project.name) | 314 | threads = set() |
| 314 | if project.Sync_NetworkHalf( | 315 | sem = _threading.Semaphore(self.jobs) |
| 315 | quiet=opt.quiet, | 316 | err_event = _threading.Event() |
| 316 | current_branch_only=opt.current_branch_only, | 317 | for project_list in objdir_project_map.values(): |
| 317 | clone_bundle=not opt.no_clone_bundle, | 318 | # Check for any errors before running any more tasks. |
| 318 | no_tags=opt.no_tags, | 319 | # ...we'll let existing threads finish, though. |
| 319 | archive=self.manifest.IsArchive): | 320 | if err_event.isSet() and not opt.force_broken: |
| 320 | fetched.add(project.gitdir) | 321 | break |
| 321 | else: | ||
| 322 | print('error: Cannot fetch %s' % project.name, file=sys.stderr) | ||
| 323 | if opt.force_broken: | ||
| 324 | print('warn: --force-broken, continuing to sync', file=sys.stderr) | ||
| 325 | else: | ||
| 326 | sys.exit(1) | ||
| 327 | else: | ||
| 328 | objdir_project_map = dict() | ||
| 329 | for project in projects: | ||
| 330 | objdir_project_map.setdefault(project.objdir, []).append(project) | ||
| 331 | |||
| 332 | threads = set() | ||
| 333 | lock = _threading.Lock() | ||
| 334 | sem = _threading.Semaphore(self.jobs) | ||
| 335 | err_event = _threading.Event() | ||
| 336 | for project_list in objdir_project_map.values(): | ||
| 337 | # Check for any errors before starting any new threads. | ||
| 338 | # ...we'll let existing threads finish, though. | ||
| 339 | if err_event.isSet(): | ||
| 340 | break | ||
| 341 | 322 | ||
| 342 | sem.acquire() | 323 | sem.acquire() |
| 324 | kwargs = dict(opt=opt, | ||
| 325 | projects=project_list, | ||
| 326 | lock=lock, | ||
| 327 | fetched=fetched, | ||
| 328 | pm=pm, | ||
| 329 | sem=sem, | ||
| 330 | err_event=err_event) | ||
| 331 | if self.jobs > 1: | ||
| 343 | t = _threading.Thread(target = self._FetchProjectList, | 332 | t = _threading.Thread(target = self._FetchProjectList, |
| 344 | args = (opt, | 333 | kwargs = kwargs) |
| 345 | project_list, | ||
| 346 | lock, | ||
| 347 | fetched, | ||
| 348 | pm, | ||
| 349 | sem, | ||
| 350 | err_event)) | ||
| 351 | # Ensure that Ctrl-C will not freeze the repo process. | 334 | # Ensure that Ctrl-C will not freeze the repo process. |
| 352 | t.daemon = True | 335 | t.daemon = True |
| 353 | threads.add(t) | 336 | threads.add(t) |
| 354 | t.start() | 337 | t.start() |
| 338 | else: | ||
| 339 | self._FetchProjectList(**kwargs) | ||
| 355 | 340 | ||
| 356 | for t in threads: | 341 | for t in threads: |
| 357 | t.join() | 342 | t.join() |
| 358 | 343 | ||
| 359 | # If we saw an error, exit with code 1 so that other scripts can check. | 344 | # If we saw an error, exit with code 1 so that other scripts can check. |
| 360 | if err_event.isSet(): | 345 | if err_event.isSet(): |
| 361 | print('\nerror: Exited sync due to fetch errors', file=sys.stderr) | 346 | print('\nerror: Exited sync due to fetch errors', file=sys.stderr) |
| 362 | sys.exit(1) | 347 | sys.exit(1) |
| 363 | 348 | ||
| 364 | pm.end() | 349 | pm.end() |
| 365 | self._fetch_times.Save() | 350 | self._fetch_times.Save() |
