summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristopher Larson <kergoth@gmail.com>2017-05-13 02:46:26 +0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-06-02 13:36:57 +0100
commitab4e578b86efcf533c43dfa76e97ea98cd9a5808 (patch)
treef8bbda21f1379d29ac0aa56efc444a16f9895389
parent60ade6074e8b1320f72c7d011b13bac9fa7f874e (diff)
downloadpoky-ab4e578b86efcf533c43dfa76e97ea98cd9a5808.tar.gz
bitbake: fetch: support multiple mirror tarball filenames
Remove ud.mirrortarball in favor of ud.mirrortarballs. Each tarball will be attempted, in order, and the first available will be used. This is needed for git shallow mirror tarball support, as we want to be able to use either a shallow or full mirror tarball. (Bitbake rev: 02eebee6709e57b523862257f75929e64f16d6b0) Signed-off-by: Christopher Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py72
-rw-r--r--bitbake/lib/bb/fetch2/git.py5
-rw-r--r--bitbake/lib/bb/fetch2/hg.py5
-rw-r--r--bitbake/lib/bb/fetch2/npm.py7
4 files changed, 47 insertions, 42 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 79a8906af7..1f63a045c1 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -425,7 +425,7 @@ def encodeurl(decoded):
425 425
426 return url 426 return url
427 427
428def uri_replace(ud, uri_find, uri_replace, replacements, d): 428def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
429 if not ud.url or not uri_find or not uri_replace: 429 if not ud.url or not uri_find or not uri_replace:
430 logger.error("uri_replace: passed an undefined value, not replacing") 430 logger.error("uri_replace: passed an undefined value, not replacing")
431 return None 431 return None
@@ -464,9 +464,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d):
464 if loc == 2: 464 if loc == 2:
465 # Handle path manipulations 465 # Handle path manipulations
466 basename = None 466 basename = None
467 if uri_decoded[0] != uri_replace_decoded[0] and ud.mirrortarball: 467 if uri_decoded[0] != uri_replace_decoded[0] and mirrortarball:
468 # If the source and destination url types differ, must be a mirrortarball mapping 468 # If the source and destination url types differ, must be a mirrortarball mapping
469 basename = os.path.basename(ud.mirrortarball) 469 basename = os.path.basename(mirrortarball)
470 # Kill parameters, they make no sense for mirror tarballs 470 # Kill parameters, they make no sense for mirror tarballs
471 uri_decoded[5] = {} 471 uri_decoded[5] = {}
472 elif ud.localpath and ud.method.supports_checksum(ud): 472 elif ud.localpath and ud.method.supports_checksum(ud):
@@ -892,45 +892,47 @@ def build_mirroruris(origud, mirrors, ld):
892 replacements["BASENAME"] = origud.path.split("/")[-1] 892 replacements["BASENAME"] = origud.path.split("/")[-1]
893 replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.') 893 replacements["MIRRORNAME"] = origud.host.replace(':','.') + origud.path.replace('/', '.').replace('*', '.')
894 894
895 def adduri(ud, uris, uds, mirrors): 895 def adduri(ud, uris, uds, mirrors, tarballs):
896 for line in mirrors: 896 for line in mirrors:
897 try: 897 try:
898 (find, replace) = line 898 (find, replace) = line
899 except ValueError: 899 except ValueError:
900 continue 900 continue
901 newuri = uri_replace(ud, find, replace, replacements, ld)
902 if not newuri or newuri in uris or newuri == origud.url:
903 continue
904 901
905 if not trusted_network(ld, newuri): 902 for tarball in tarballs:
906 logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri)) 903 newuri = uri_replace(ud, find, replace, replacements, ld, tarball)
907 continue 904 if not newuri or newuri in uris or newuri == origud.url:
905 continue
908 906
909 # Create a local copy of the mirrors minus the current line 907 if not trusted_network(ld, newuri):
910 # this will prevent us from recursively processing the same line 908 logger.debug(1, "Mirror %s not in the list of trusted networks, skipping" % (newuri))
911 # as well as indirect recursion A -> B -> C -> A 909 continue
912 localmirrors = list(mirrors) 910
913 localmirrors.remove(line) 911 # Create a local copy of the mirrors minus the current line
912 # this will prevent us from recursively processing the same line
913 # as well as indirect recursion A -> B -> C -> A
914 localmirrors = list(mirrors)
915 localmirrors.remove(line)
914 916
915 try:
916 newud = FetchData(newuri, ld)
917 newud.setup_localpath(ld)
918 except bb.fetch2.BBFetchException as e:
919 logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
920 logger.debug(1, str(e))
921 try: 917 try:
922 # setup_localpath of file:// urls may fail, we should still see 918 newud = FetchData(newuri, ld)
923 # if mirrors of the url exist 919 newud.setup_localpath(ld)
924 adduri(newud, uris, uds, localmirrors) 920 except bb.fetch2.BBFetchException as e:
925 except UnboundLocalError: 921 logger.debug(1, "Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
926 pass 922 logger.debug(1, str(e))
927 continue 923 try:
928 uris.append(newuri) 924 # setup_localpath of file:// urls may fail, we should still see
929 uds.append(newud) 925 # if mirrors of the url exist
926 adduri(newud, uris, uds, localmirrors, tarballs)
927 except UnboundLocalError:
928 pass
929 continue
930 uris.append(newuri)
931 uds.append(newud)
930 932
931 adduri(newud, uris, uds, localmirrors) 933 adduri(newud, uris, uds, localmirrors, tarballs)
932 934
933 adduri(origud, uris, uds, mirrors) 935 adduri(origud, uris, uds, mirrors, origud.mirrortarballs or [None])
934 936
935 return uris, uds 937 return uris, uds
936 938
@@ -975,8 +977,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
975 # We may be obtaining a mirror tarball which needs further processing by the real fetcher 977 # We may be obtaining a mirror tarball which needs further processing by the real fetcher
976 # If that tarball is a local file:// we need to provide a symlink to it 978 # If that tarball is a local file:// we need to provide a symlink to it
977 dldir = ld.getVar("DL_DIR") 979 dldir = ld.getVar("DL_DIR")
978 if origud.mirrortarball and os.path.basename(ud.localpath) == os.path.basename(origud.mirrortarball) \ 980
979 and os.path.basename(ud.localpath) != os.path.basename(origud.localpath): 981 if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
980 # Create donestamp in old format to avoid triggering a re-download 982 # Create donestamp in old format to avoid triggering a re-download
981 if ud.donestamp: 983 if ud.donestamp:
982 bb.utils.mkdirhier(os.path.dirname(ud.donestamp)) 984 bb.utils.mkdirhier(os.path.dirname(ud.donestamp))
@@ -993,7 +995,7 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
993 pass 995 pass
994 if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld): 996 if not verify_donestamp(origud, ld) or origud.method.need_update(origud, ld):
995 origud.method.download(origud, ld) 997 origud.method.download(origud, ld)
996 if hasattr(origud.method,"build_mirror_data"): 998 if hasattr(origud.method, "build_mirror_data"):
997 origud.method.build_mirror_data(origud, ld) 999 origud.method.build_mirror_data(origud, ld)
998 return origud.localpath 1000 return origud.localpath
999 # Otherwise the result is a local file:// and we symlink to it 1001 # Otherwise the result is a local file:// and we symlink to it
@@ -1190,7 +1192,7 @@ class FetchData(object):
1190 self.localfile = "" 1192 self.localfile = ""
1191 self.localpath = None 1193 self.localpath = None
1192 self.lockfile = None 1194 self.lockfile = None
1193 self.mirrortarball = None 1195 self.mirrortarballs = []
1194 self.basename = None 1196 self.basename = None
1195 self.basepath = None 1197 self.basepath = None
1196 (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url)) 1198 (self.type, self.host, self.path, self.user, self.pswd, self.parm) = decodeurl(d.expand(url))
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 2550bde838..01d4bbdc2e 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -205,8 +205,9 @@ class Git(FetchMethod):
205 if ud.rebaseable: 205 if ud.rebaseable:
206 for name in ud.names: 206 for name in ud.names:
207 gitsrcname = gitsrcname + '_' + ud.revisions[name] 207 gitsrcname = gitsrcname + '_' + ud.revisions[name]
208 ud.mirrortarball = 'git2_%s.tar.gz' % gitsrcname 208 mirrortarball = 'git2_%s.tar.gz' % gitsrcname
209 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) 209 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
210 ud.mirrortarballs = [mirrortarball]
210 gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/") 211 gitdir = d.getVar("GITDIR") or (d.getVar("DL_DIR") + "/git2/")
211 ud.clonedir = os.path.join(gitdir, gitsrcname) 212 ud.clonedir = os.path.join(gitdir, gitsrcname)
212 213
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index b5f268601c..d0857e63f7 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -76,8 +76,9 @@ class Hg(FetchMethod):
76 # Create paths to mercurial checkouts 76 # Create paths to mercurial checkouts
77 hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \ 77 hgsrcname = '%s_%s_%s' % (ud.module.replace('/', '.'), \
78 ud.host, ud.path.replace('/', '.')) 78 ud.host, ud.path.replace('/', '.'))
79 ud.mirrortarball = 'hg_%s.tar.gz' % hgsrcname 79 mirrortarball = 'hg_%s.tar.gz' % hgsrcname
80 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) 80 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
81 ud.mirrortarballs = [mirrortarball]
81 82
82 hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/") 83 hgdir = d.getVar("HGDIR") or (d.getVar("DL_DIR") + "/hg/")
83 ud.pkgdir = os.path.join(hgdir, hgsrcname) 84 ud.pkgdir = os.path.join(hgdir, hgsrcname)
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py
index 73a75fe989..5f799724e7 100644
--- a/bitbake/lib/bb/fetch2/npm.py
+++ b/bitbake/lib/bb/fetch2/npm.py
@@ -91,9 +91,10 @@ class Npm(FetchMethod):
91 ud.prefixdir = prefixdir 91 ud.prefixdir = prefixdir
92 92
93 ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0") 93 ud.write_tarballs = ((d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0") != "0")
94 ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version) 94 mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
95 ud.mirrortarball = ud.mirrortarball.replace('/', '-') 95 mirrortarball = ud.mirrortarball.replace('/', '-')
96 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), ud.mirrortarball) 96 ud.fullmirror = os.path.join(d.getVar("DL_DIR"), mirrortarball)
97 ud.mirrortarballs = [mirrortarball]
97 98
98 def need_update(self, ud, d): 99 def need_update(self, ud, d):
99 if os.path.exists(ud.localpath): 100 if os.path.exists(ud.localpath):