diff options
author | Joshua Lock <josh@linux.intel.com> | 2010-02-01 16:56:16 +0000 |
---|---|---|
committer | Joshua Lock <josh@linux.intel.com> | 2010-02-04 00:18:29 +0000 |
commit | 0737552c1de18c64db8a5a52248a3e7b217f30aa (patch) | |
tree | d556e058c5e3b85dde0b3c13f5e9e25e472a5ed8 /bitbake/lib/bb/fetch/__init__.py | |
parent | 0b52537f5ce39a6025a2a8de381792accc333b8e (diff) | |
download | poky-0737552c1de18c64db8a5a52248a3e7b217f30aa.tar.gz |
bitbake: unify mirror support and make it independant of the fetcher
This patch serves two purposes. Firstly it unifies the concept of mirrors into
PREMIRRORS and MIRRORS. PREMIRRORS are tried before the SRC_URI defined in the
recipe whereas MIRRORS are tried only if that fails.
The tarball stash was conceptually inline with a PREMIRROR only with special
handling within the wget fetcher and therefore only worked with certain
fetch types.
Secondly the patch removes the need for individual fetch implementations to
worry about mirror handling.
With this patch, the base fetch implementation will first try to use a
PREMIRROR to fetch the desired object, if this fails the native fetch method
for the object will be tried and if this fails will try to fetch a copy from
one of the MIRRORS.
Signed-off-by: Joshua Lock <josh@linux.intel.com>
Diffstat (limited to 'bitbake/lib/bb/fetch/__init__.py')
-rw-r--r-- | bitbake/lib/bb/fetch/__init__.py | 94 |
1 files changed, 50 insertions, 44 deletions
diff --git a/bitbake/lib/bb/fetch/__init__.py b/bitbake/lib/bb/fetch/__init__.py index ab4658bc3b..ccb60de59c 100644 --- a/bitbake/lib/bb/fetch/__init__.py +++ b/bitbake/lib/bb/fetch/__init__.py | |||
@@ -185,7 +185,18 @@ def go(d, urls = None): | |||
185 | pass | 185 | pass |
186 | bb.utils.unlockfile(lf) | 186 | bb.utils.unlockfile(lf) |
187 | continue | 187 | continue |
188 | m.go(u, ud, d) | 188 | |
189 | # First try fetching uri, u, from PREMIRRORS | ||
190 | mirrors = [ i.split() for i in (bb.data.getVar('PREMIRRORS', d, 1) or "").split('\n') if i ] | ||
191 | if not try_mirrors(d, u, mirrors): | ||
192 | # Next try fetching from the original uri, u | ||
193 | try: | ||
194 | m.go(u, ud, d) | ||
195 | except: | ||
196 | # Finally, try fetching uri, u, from MIRRORS | ||
197 | mirrors = [ i.split() for i in (bb.data.getVar('MIRRORS', d, 1) or "").split('\n') if i ] | ||
198 | try_mirrors (d, u, mirrors) | ||
199 | |||
189 | if ud.localfile: | 200 | if ud.localfile: |
190 | if not m.forcefetch(u, ud, d): | 201 | if not m.forcefetch(u, ud, d): |
191 | Fetch.write_md5sum(u, ud, d) | 202 | Fetch.write_md5sum(u, ud, d) |
@@ -332,6 +343,44 @@ def runfetchcmd(cmd, d, quiet = False): | |||
332 | 343 | ||
333 | return output | 344 | return output |
334 | 345 | ||
346 | def try_mirrors(d, uri, mirrors): | ||
347 | """ | ||
348 | Try to use a mirrored version of the sources. | ||
349 | This method will be automatically called before the fetchers go. | ||
350 | |||
351 | d Is a bb.data instance | ||
352 | uri is the original uri we're trying to download | ||
353 | mirrors is the list of mirrors we're going to try | ||
354 | """ | ||
355 | fpath = os.path.join(data.getVar("DL_DIR", d, 1), os.path.basename(uri)) | ||
356 | if os.access(fpath, os.R_OK): | ||
357 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % fpath) | ||
358 | return True | ||
359 | |||
360 | ld = d.createCopy() | ||
361 | for (find, replace) in mirrors: | ||
362 | newuri = uri_replace(uri, find, replace, ld) | ||
363 | if newuri != uri: | ||
364 | try: | ||
365 | ud = FetchData(newuri, ld) | ||
366 | except bb.fetch.NoMethodError: | ||
367 | bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url) | ||
368 | continue | ||
369 | |||
370 | ud.setup_localpath(ld) | ||
371 | |||
372 | try: | ||
373 | ud.method.go(newuri, ud, ld) | ||
374 | return True | ||
375 | except (bb.fetch.MissingParameterError, | ||
376 | bb.fetch.FetchError, | ||
377 | bb.fetch.MD5SumError): | ||
378 | import sys | ||
379 | (type, value, traceback) = sys.exc_info() | ||
380 | bb.msg.debug(2, bb.msg.domain.Fetcher, "Mirror fetch failure: %s" % value) | ||
381 | return False | ||
382 | |||
383 | |||
335 | class FetchData(object): | 384 | class FetchData(object): |
336 | """ | 385 | """ |
337 | A class which represents the fetcher state for a given URI. | 386 | A class which represents the fetcher state for a given URI. |
@@ -489,49 +538,6 @@ class Fetch(object): | |||
489 | 538 | ||
490 | localcount_internal_helper = staticmethod(localcount_internal_helper) | 539 | localcount_internal_helper = staticmethod(localcount_internal_helper) |
491 | 540 | ||
492 | def try_mirror(d, tarfn): | ||
493 | """ | ||
494 | Try to use a mirrored version of the sources. We do this | ||
495 | to avoid massive loads on foreign cvs and svn servers. | ||
496 | This method will be used by the different fetcher | ||
497 | implementations. | ||
498 | |||
499 | d Is a bb.data instance | ||
500 | tarfn is the name of the tarball | ||
501 | """ | ||
502 | tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn) | ||
503 | if os.access(tarpath, os.R_OK): | ||
504 | bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn) | ||
505 | return True | ||
506 | |||
507 | pn = data.getVar('PN', d, True) | ||
508 | src_tarball_stash = None | ||
509 | if pn: | ||
510 | src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split() | ||
511 | |||
512 | ld = d.createCopy() | ||
513 | for stash in src_tarball_stash: | ||
514 | url = stash + tarfn | ||
515 | try: | ||
516 | ud = FetchData(url, ld) | ||
517 | except bb.fetch.NoMethodError: | ||
518 | bb.msg.debug(1, bb.msg.domain.Fetcher, "No method for %s" % url) | ||
519 | continue | ||
520 | |||
521 | ud.setup_localpath(ld) | ||
522 | |||
523 | try: | ||
524 | ud.method.go(url, ud, ld) | ||
525 | return True | ||
526 | except (bb.fetch.MissingParameterError, | ||
527 | bb.fetch.FetchError, | ||
528 | bb.fetch.MD5SumError): | ||
529 | import sys | ||
530 | (type, value, traceback) = sys.exc_info() | ||
531 | bb.msg.debug(2, bb.msg.domain.Fetcher, "Tarball stash fetch failure: %s" % value) | ||
532 | return False | ||
533 | try_mirror = staticmethod(try_mirror) | ||
534 | |||
535 | def verify_md5sum(ud, got_sum): | 541 | def verify_md5sum(ud, got_sum): |
536 | """ | 542 | """ |
537 | Verify the md5sum we wanted with the one we got | 543 | Verify the md5sum we wanted with the one we got |