From 1251765fff1c12d943091a2981b347e721b93e4e Mon Sep 17 00:00:00 2001 From: Paul Barker Date: Wed, 3 Jun 2020 20:31:16 +0100 Subject: bitbake: fetch2: Add the ability to list expanded URL data Some fetchers may download additional sources along with those explicitly listed in SRC_URI. These "implicit URLs" will be needed by the archiver to ensure that all sources can be archived. We can't just return a list of URL strings since each URL may need its own SRCREV data so we return a list of FetchData objects. Each fetcher can override the implicit_urldata() function to provide the additional FetchData objects. For now this is just needed in the gitsm fetcher to walk git submodules recursively. (Bitbake rev: 1350f241b7d991bd191ce9e44f6662e4376c6e24) Signed-off-by: Paul Barker Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/__init__.py | 25 +++++++++++++++++++++++++ bitbake/lib/bb/fetch2/gitsm.py | 21 +++++++++++++++++++++ 2 files changed, 46 insertions(+) (limited to 'bitbake/lib') diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index eb112f069d..756f60212f 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -1617,6 +1617,13 @@ class FetchMethod(object): return True return False + def implicit_urldata(self, ud, d): + """ + Get a list of FetchData objects for any implicit URLs that will also + be downloaded when we fetch the given URL. + """ + return [] + class Fetch(object): def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None): if localonly and cache: @@ -1842,6 +1849,24 @@ class Fetch(object): if ud.lockfile: bb.utils.unlockfile(lf) + def expanded_urldata(self, urls=None): + """ + Get an expanded list of FetchData objects covering both the given + URLS and any additional implicit URLs that are added automatically by + the appropriate FetchMethod. + """ + + if not urls: + urls = self.urls + + urldata = [] + for url in urls: + ud = self.ud[url] + urldata.append(ud) + urldata += ud.method.implicit_urldata(ud, self.d) + + return urldata + class FetchConnectionCache(object): """ A class which represents an container for socket connections. diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py index e7083001da..56bd5f0480 100644 --- a/bitbake/lib/bb/fetch2/gitsm.py +++ b/bitbake/lib/bb/fetch2/gitsm.py @@ -223,3 +223,24 @@ class GitSM(Git): # up the configuration and checks out the files. The main project config should remain # unmodified, and no download from the internet should occur. runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir) + + def implicit_urldata(self, ud, d): + import shutil, subprocess, tempfile + + urldata = [] + def add_submodule(ud, url, module, modpath, workdir, d): + url += ";bareclone=1;nobranch=1" + newfetch = Fetch([url], d, cache=False) + urldata.extend(newfetch.expanded_urldata()) + + # If we're using a shallow mirror tarball it needs to be unpacked + # temporarily so that we can examine the .gitmodules file + if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d): + tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR")) + subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True) + self.process_submodules(ud, tmpdir, add_submodule, d) + shutil.rmtree(tmpdir) + else: + self.process_submodules(ud, ud.clonedir, add_submodule, d) + + return urldata -- cgit v1.2.3-54-g00ecf