diff options
Diffstat (limited to 'subcmds')
| -rw-r--r-- | subcmds/sync.py | 55 |
1 files changed, 53 insertions, 2 deletions
diff --git a/subcmds/sync.py b/subcmds/sync.py index b83f2d4a..9e4a9754 100644 --- a/subcmds/sync.py +++ b/subcmds/sync.py | |||
| @@ -39,6 +39,11 @@ except ImportError: | |||
| 39 | def _rlimit_nofile(): | 39 | def _rlimit_nofile(): |
| 40 | return (256, 256) | 40 | return (256, 256) |
| 41 | 41 | ||
| 42 | try: | ||
| 43 | import multiprocessing | ||
| 44 | except ImportError: | ||
| 45 | multiprocessing = None | ||
| 46 | |||
| 42 | from git_command import GIT | 47 | from git_command import GIT |
| 43 | from git_refs import R_HEADS, HEAD | 48 | from git_refs import R_HEADS, HEAD |
| 44 | from project import Project | 49 | from project import Project |
| @@ -299,10 +304,56 @@ later is required to fix a server side protocol bug. | |||
| 299 | 304 | ||
| 300 | pm.end() | 305 | pm.end() |
| 301 | self._fetch_times.Save() | 306 | self._fetch_times.Save() |
| 302 | for project in projects: | 307 | |
| 303 | project.bare_git.gc('--auto') | 308 | self._GCProjects(projects) |
| 304 | return fetched | 309 | return fetched |
| 305 | 310 | ||
| 311 | def _GCProjects(self, projects): | ||
| 312 | if multiprocessing: | ||
| 313 | cpu_count = multiprocessing.cpu_count() | ||
| 314 | else: | ||
| 315 | cpu_count = 1 | ||
| 316 | jobs = min(self.jobs, cpu_count) | ||
| 317 | |||
| 318 | if jobs < 2: | ||
| 319 | for project in projects: | ||
| 320 | project.bare_git.gc('--auto') | ||
| 321 | return | ||
| 322 | |||
| 323 | config = {'pack.threads': cpu_count / jobs if cpu_count > jobs else 1} | ||
| 324 | |||
| 325 | threads = set() | ||
| 326 | sem = _threading.Semaphore(jobs) | ||
| 327 | err_event = _threading.Event() | ||
| 328 | |||
| 329 | def GC(project): | ||
| 330 | try: | ||
| 331 | try: | ||
| 332 | project.bare_git.gc('--auto', config=config) | ||
| 333 | except GitError: | ||
| 334 | err_event.set() | ||
| 335 | except: | ||
| 336 | err_event.set() | ||
| 337 | raise | ||
| 338 | finally: | ||
| 339 | sem.release() | ||
| 340 | |||
| 341 | for project in projects: | ||
| 342 | if err_event.isSet(): | ||
| 343 | break | ||
| 344 | sem.acquire() | ||
| 345 | t = _threading.Thread(target=GC, args=(project,)) | ||
| 346 | t.daemon = True | ||
| 347 | threads.add(t) | ||
| 348 | t.start() | ||
| 349 | |||
| 350 | for t in threads: | ||
| 351 | t.join() | ||
| 352 | |||
| 353 | if err_event.isSet(): | ||
| 354 | print >>sys.stderr, '\nerror: Exited sync due to gc errors' | ||
| 355 | sys.exit(1) | ||
| 356 | |||
| 306 | def UpdateProjectList(self): | 357 | def UpdateProjectList(self): |
| 307 | new_project_paths = [] | 358 | new_project_paths = [] |
| 308 | for project in self.GetProjects(None, missing_ok=True): | 359 | for project in self.GetProjects(None, missing_ok=True): |
