diff options
author | Shawn O. Pearce <sop@google.com> | 2011-09-19 14:50:58 -0700 |
---|---|---|
committer | Shawn O. Pearce <sop@google.com> | 2011-09-28 10:07:36 -0700 |
commit | f322b9abb4cadc67b991baf6ba1b9f2fbd5d7812 (patch) | |
tree | ce75a04fed2e84457800325d158de13645cef67e /repo | |
parent | db728cd866d4950779620993e12e76f09eb6e2ee (diff) | |
download | git-repo-f322b9abb4cadc67b991baf6ba1b9f2fbd5d7812.tar.gz |
sync: Support downloading bundle to initialize repositoryv1.7.7
An HTTP (or HTTPS) based remote server may now offer a 'clone.bundle'
file in each repository's Git directory. Over an http:// or https://
remote repo will first ask for '$URL/clone.bundle', and if present
download this to bootstrap the local client, rather than relying
on the native Git transport to initialize the new repository.
Bundles may be hosted elsewhere. The client automatically follows a
HTTP 302 redirect to acquire the bundle file. This allows servers
to direct clients to cached copies residing on content delivery
networks, where the bundle may be closer to the end-user.
Bundle downloads are resumeable from where they last left off,
allowing clients to initialize large repositories even when the
connection gets interrupted.
If a bundle does not exist for a repository (a HTTP 404 response
code is returned for '$URL/clone.bundle'), the native Git transport
is used instead. If the client is performing a shallow sync, the
bundle transport is not used, as there is no way to embed shallow
data into the bundle.
Change-Id: I05dad17792fd6fd20635a0f71589566e557cc743
Signed-off-by: Shawn O. Pearce <sop@google.com>
Diffstat (limited to 'repo')
-rwxr-xr-x | repo | 103 |
1 files changed, 93 insertions, 10 deletions
@@ -28,7 +28,7 @@ if __name__ == '__main__': | |||
28 | del magic | 28 | del magic |
29 | 29 | ||
30 | # increment this whenever we make important changes to this script | 30 | # increment this whenever we make important changes to this script |
31 | VERSION = (1, 12) | 31 | VERSION = (1, 13) |
32 | 32 | ||
33 | # increment this if the MAINTAINER_KEYS block is modified | 33 | # increment this if the MAINTAINER_KEYS block is modified |
34 | KEYRING_VERSION = (1,0) | 34 | KEYRING_VERSION = (1,0) |
@@ -91,6 +91,7 @@ import re | |||
91 | import readline | 91 | import readline |
92 | import subprocess | 92 | import subprocess |
93 | import sys | 93 | import sys |
94 | import urllib2 | ||
94 | 95 | ||
95 | home_dot_repo = os.path.expanduser('~/.repoconfig') | 96 | home_dot_repo = os.path.expanduser('~/.repoconfig') |
96 | gpg_dir = os.path.join(home_dot_repo, 'gnupg') | 97 | gpg_dir = os.path.join(home_dot_repo, 'gnupg') |
@@ -187,10 +188,6 @@ def _Init(args): | |||
187 | else: | 188 | else: |
188 | can_verify = True | 189 | can_verify = True |
189 | 190 | ||
190 | if not opt.quiet: | ||
191 | print >>sys.stderr, 'Getting repo ...' | ||
192 | print >>sys.stderr, ' from %s' % url | ||
193 | |||
194 | dst = os.path.abspath(os.path.join(repodir, S_repo)) | 191 | dst = os.path.abspath(os.path.join(repodir, S_repo)) |
195 | _Clone(url, dst, opt.quiet) | 192 | _Clone(url, dst, opt.quiet) |
196 | 193 | ||
@@ -300,15 +297,42 @@ def _SetConfig(local, name, value): | |||
300 | raise CloneFailure() | 297 | raise CloneFailure() |
301 | 298 | ||
302 | 299 | ||
303 | def _Fetch(local, quiet, *args): | 300 | def _InitHttp(): |
301 | handlers = [] | ||
302 | |||
303 | mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() | ||
304 | try: | ||
305 | import netrc | ||
306 | n = netrc.netrc() | ||
307 | for host in n.hosts: | ||
308 | p = n.hosts[host] | ||
309 | mgr.add_password(None, 'http://%s/' % host, p[0], p[2]) | ||
310 | mgr.add_password(None, 'https://%s/' % host, p[0], p[2]) | ||
311 | except: | ||
312 | pass | ||
313 | handlers.append(urllib2.HTTPBasicAuthHandler(mgr)) | ||
314 | |||
315 | if 'http_proxy' in os.environ: | ||
316 | url = os.environ['http_proxy'] | ||
317 | handlers.append(urllib2.ProxyHandler({'http': url, 'https': url})) | ||
318 | if 'REPO_CURL_VERBOSE' in os.environ: | ||
319 | handlers.append(urllib2.HTTPHandler(debuglevel=1)) | ||
320 | handlers.append(urllib2.HTTPSHandler(debuglevel=1)) | ||
321 | urllib2.install_opener(urllib2.build_opener(*handlers)) | ||
322 | |||
323 | def _Fetch(url, local, src, quiet): | ||
324 | if not quiet: | ||
325 | print >>sys.stderr, 'Get %s' % url | ||
326 | |||
304 | cmd = [GIT, 'fetch'] | 327 | cmd = [GIT, 'fetch'] |
305 | if quiet: | 328 | if quiet: |
306 | cmd.append('--quiet') | 329 | cmd.append('--quiet') |
307 | err = subprocess.PIPE | 330 | err = subprocess.PIPE |
308 | else: | 331 | else: |
309 | err = None | 332 | err = None |
310 | cmd.extend(args) | 333 | cmd.append(src) |
311 | cmd.append('origin') | 334 | cmd.append('+refs/heads/*:refs/remotes/origin/*') |
335 | cmd.append('refs/tags/*:refs/tags/*') | ||
312 | 336 | ||
313 | proc = subprocess.Popen(cmd, cwd = local, stderr = err) | 337 | proc = subprocess.Popen(cmd, cwd = local, stderr = err) |
314 | if err: | 338 | if err: |
@@ -317,6 +341,62 @@ def _Fetch(local, quiet, *args): | |||
317 | if proc.wait() != 0: | 341 | if proc.wait() != 0: |
318 | raise CloneFailure() | 342 | raise CloneFailure() |
319 | 343 | ||
344 | def _DownloadBundle(url, local, quiet): | ||
345 | if not url.endswith('/'): | ||
346 | url += '/' | ||
347 | url += 'clone.bundle' | ||
348 | |||
349 | proc = subprocess.Popen( | ||
350 | [GIT, 'config', '--get-regexp', 'url.*.insteadof'], | ||
351 | cwd = local, | ||
352 | stdout = subprocess.PIPE) | ||
353 | for line in proc.stdout: | ||
354 | m = re.compile(r'^url\.(.*)\.insteadof (.*)$').match(line) | ||
355 | if m: | ||
356 | new_url = m.group(1) | ||
357 | old_url = m.group(2) | ||
358 | if url.startswith(old_url): | ||
359 | url = new_url + url[len(old_url):] | ||
360 | break | ||
361 | proc.stdout.close() | ||
362 | proc.wait() | ||
363 | |||
364 | if not url.startswith('http:') and not url.startswith('https:'): | ||
365 | return False | ||
366 | |||
367 | dest = open(os.path.join(local, '.git', 'clone.bundle'), 'w+b') | ||
368 | try: | ||
369 | try: | ||
370 | r = urllib2.urlopen(url) | ||
371 | except urllib2.HTTPError, e: | ||
372 | if e.code == 404: | ||
373 | return False | ||
374 | print >>sys.stderr, 'fatal: Cannot get %s' % url | ||
375 | print >>sys.stderr, 'fatal: HTTP error %s' % e.code | ||
376 | raise CloneFailure() | ||
377 | except urllib2.URLError, e: | ||
378 | print >>sys.stderr, 'fatal: Cannot get %s' % url | ||
379 | print >>sys.stderr, 'fatal: error %s' % e.reason | ||
380 | raise CloneFailure() | ||
381 | try: | ||
382 | if not quiet: | ||
383 | print >>sys.stderr, 'Get %s' % url | ||
384 | while True: | ||
385 | buf = r.read(8192) | ||
386 | if buf == '': | ||
387 | return True | ||
388 | dest.write(buf) | ||
389 | finally: | ||
390 | r.close() | ||
391 | finally: | ||
392 | dest.close() | ||
393 | |||
394 | def _ImportBundle(local): | ||
395 | path = os.path.join(local, '.git', 'clone.bundle') | ||
396 | try: | ||
397 | _Fetch(local, local, path, True) | ||
398 | finally: | ||
399 | os.remove(path) | ||
320 | 400 | ||
321 | def _Clone(url, local, quiet): | 401 | def _Clone(url, local, quiet): |
322 | """Clones a git repository to a new subdirectory of repodir | 402 | """Clones a git repository to a new subdirectory of repodir |
@@ -344,11 +424,14 @@ def _Clone(url, local, quiet): | |||
344 | print >>sys.stderr, 'fatal: could not create %s' % local | 424 | print >>sys.stderr, 'fatal: could not create %s' % local |
345 | raise CloneFailure() | 425 | raise CloneFailure() |
346 | 426 | ||
427 | _InitHttp() | ||
347 | _SetConfig(local, 'remote.origin.url', url) | 428 | _SetConfig(local, 'remote.origin.url', url) |
348 | _SetConfig(local, 'remote.origin.fetch', | 429 | _SetConfig(local, 'remote.origin.fetch', |
349 | '+refs/heads/*:refs/remotes/origin/*') | 430 | '+refs/heads/*:refs/remotes/origin/*') |
350 | _Fetch(local, quiet) | 431 | if _DownloadBundle(url, local, quiet): |
351 | _Fetch(local, quiet, '--tags') | 432 | _ImportBundle(local) |
433 | else: | ||
434 | _Fetch(url, local, 'origin', quiet) | ||
352 | 435 | ||
353 | 436 | ||
354 | def _Verify(cwd, branch, quiet): | 437 | def _Verify(cwd, branch, quiet): |