From ab4e578b86efcf533c43dfa76e97ea98cd9a5808 Mon Sep 17 00:00:00 2001 From: Christopher Larson Date: Sat, 13 May 2017 02:46:26 +0500 Subject: bitbake: fetch: support multiple mirror tarball filenames Remove ud.mirrortarball in favor of ud.mirrortarballs. Each tarball will be attempted, in order, and the first available will be used. This is needed for git shallow mirror tarball support, as we want to be able to use either a shallow or full mirror tarball. (Bitbake rev: 02eebee6709e57b523862257f75929e64f16d6b0) Signed-off-by: Christopher Larson Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/__init__.py | 72 ++++++++++++++++++++------------------- bitbake/lib/bb/fetch2/git.py | 5 +-- bitbake/lib/bb/fetch2/hg.py | 5 +-- bitbake/lib/bb/fetch2/npm.py | 7 ++-- 4 files changed, 47 insertions(+), 42 deletions(-) (limited to 'bitbake/lib/bb/fetch2') diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 79a8906af7..1f63a045c1 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -425,7 +425,7 @@ def encodeurl(decoded): return url -def uri_replace(ud, uri_find, uri_replace, replacements, d): +def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None): if not ud.url or not uri_find or not uri_replace: logger.error("uri_replace: passed an undefined value, not replacing") return None @@ -464,9 +464,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d): if loc == 2: # Handle path manipulations basename = None - if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball: + if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball: # If the source and destination url types differ, must be a mirrortarball mapping - basename = os.path.basename(ud.mirrortarball) + basename = os.path.basename(mirrortarball) # Kill parameters, they make no sense for mirror tarballs uri_decoded[5] = {} elif ud.localpath and ud.method.supports_checksum(ud): @@ -892,45 +892,47 @@ def build_mirroruris(origud, mirrors, ld): replacements["BASENAME"] = origud.path.split("/")[-1] replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') - def adduri(ud, uris, uds, mirrors): + def adduri(ud, uris, uds, mirrors, tarballs): for line in mirrors: try: (find, replace) = line except ValueError: continue - newuri = uri_replace(ud, find, replace, replacements, ld) - if not newuri or newuri in uris or newuri == origud.url: - continue - if not trusted_network(ld, newuri): - logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) - continue + for tarball in tarballs: + newuri = uri_replace(ud, find, replace, replacements, ld, tarball) + if not newuri or newuri in uris or newuri == origud.url: + continue - # Create a local copy of the mirrors minus the current line - # this will prevent us from recursively processing the same line - # as well as indirect recursion A -> B -> C -> A - localmirrors = list(mirrors) - localmirrors.remove(line) + if not trusted_network(ld, newuri): + logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) + continue + + # Create a local copy of the mirrors minus the current line + # this will prevent us from recursively processing the same line + # as well as indirect recursion A -> B -> C -> A + localmirrors = list(mirrors) + localmirrors.remove(line) - try: - newud = FetchData(newuri, ld) - newud.setup_localpath(ld) - except bb.fetch2.BBFetchException as e: - logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) - logger.debug(1, str(e)) try: - # setup_localpath of file:// urls may fail, we should still see - # if mirrors of the url exist - adduri(newud, uris, uds, localmirrors) - except UnboundLocalError: - pass - continue - uris.append(newuri) - uds.append(newud) + newud = FetchData(newuri, ld) + newud.setup_localpath(ld) + except bb.fetch2.BBFetchException as e: + logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url)) + logger.debug(1, str(e)) + try: + # setup_localpath of file:// urls may fail, we should still see + # if mirrors of the url exist + adduri(newud, uris, uds, localmirrors, tarballs) + except UnboundLocalError: + pass + continue + uris.append(newuri) + uds.append(newud) - adduri(newud, uris, uds, localmirrors) + adduri(newud, uris, uds, localmirrors, tarballs) - adduri(origud, uris, uds, mirrors) + adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None]) return uris, uds @@ -975,8 +977,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): # We may be obtaining a mirror tarball which needs further processing by the real fetcher # If that tarball is a local file:// we need to provide a symlink to it dldir = ld.getVar("DL_DIR") - if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \ - and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): + + if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): # Create donestamp in old format to avoid triggering a re-download if ud.donestamp: bb.utils.mkdirhier(os.path.dirname(ud.donestamp)) @@ -993,7 +995,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False): pass if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld): origud.method.download(origud, ld) - if hasattr(origud.method,"build_mirror_data"): + if hasattr(origud.method, "build_mirror_data"): origud.method.build_mirror_data(origud, ld) return origud.localpath # Otherwise the result is a local file:// and we symlink to it @@ -1190,7 +1192,7 @@ class FetchData(object): self.localfile = "" self.localpath = None self.lockfile = None - self.mirrortarball = None + self.mirrortarballs = [] self.basename = None self.basepath = None (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url)) diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 2550bde838..01d4bbdc2e 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -205,8 +205,9 @@ class Git(FetchMethod): if ud.rebaseable: for name in ud.names: gitsrcname = gitsrcname + '_' + ud.revisions[name] - ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname - ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) + mirrortarball = 'git2_%s.tar.gz' % gitsrcname + ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) + ud.mirrortarballs = [mirrortarball] gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/") ud.clonedir = os.path.join(gitdir, gitsrcname) diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index b5f268601c..d0857e63f7 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py @@ -76,8 +76,9 @@ class Hg(FetchMethod): # Create paths to mercurial checkouts hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ ud.host, ud.path.replace('/', '.')) - ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname - ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) + mirrortarball = 'hg_%s.tar.gz' % hgsrcname + ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) + ud.mirrortarballs = [mirrortarball] hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/") ud.pkgdir = os.path.join(hgdir, hgsrcname) diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py index 73a75fe989..5f799724e7 100644 --- a/bitbake/lib/bb/fetch2/npm.py +++ b/bitbake/lib/bb/fetch2/npm.py @@ -91,9 +91,10 @@ class Npm(FetchMethod): ud.prefixdir = prefixdir ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") - ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) - ud.mirrortarball = ud.mirrortarball.replace('/', '-') - ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) + mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) + mirrortarball = ud.mirrortarball.replace('/', '-') + ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball) + ud.mirrortarballs = [mirrortarball] def need_update(self, ud, d): if os.path.exists(ud.localpath): -- cgit v1.2.3-54-g00ecf