From 45c02843d181b5f1872ad30f4d847c395233a676 Mon Sep 17 00:00:00 2001 From: Matt Hoosier Date: Fri, 22 Jan 2021 10:55:13 -0600 Subject: bitbake: fetch/git: download LFS content too during do_fetch Insert an explicit pass to fetch all blobs needed by Git LFS, during the fetch() function. This avoids the default behavior of Git LFS to wait until 'git checkout' to begin downloading the blobs pointed to by LFS records. Network access is not allowed at that point in the recipe's lifecycle. [YOCTO #14191] (Bitbake rev: 0efac66043662e7a2027192f50e92e982db2ba1c) Signed-off-by: Matt Hoosier Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/git.py | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) (limited to 'bitbake/lib/bb/fetch2/git.py') diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index df9538a60d..e3ba80a3f5 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -384,6 +384,35 @@ class Git(FetchMethod): if missing_rev: raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) + if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): + # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching + # of all LFS blobs needed at the the srcrev. + # + # It would be nice to just do this inline here by running 'git-lfs fetch' + # on the bare clonedir, but that operation requires a working copy on some + # releases of Git LFS. + tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) + try: + # Do the checkout. This implicitly involves a Git LFS fetch. + self.unpack(ud, tmpdir, d) + + # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into + # the bare clonedir. + # + # As this procedure is invoked repeatedly on incremental fetches as + # a recipe's SRCREV is bumped throughout its lifetime, this will + # result in a gradual accumulation of LFS blobs in /lfs + # corresponding to all the blobs reachable from the different revs + # fetched across time. + # + # Only do this if the unpack resulted in a .git/lfs directory being + # created; this only happens if at least one blob needed to be + # downloaded. + if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): + runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) + finally: + bb.utils.remove(tmpdir, recurse=True) + def build_mirror_data(self, ud, d): if ud.shallow and ud.write_shallow_tarballs: if not os.path.exists(ud.fullshallow): @@ -479,7 +508,7 @@ class Git(FetchMethod): if os.path.exists(destdir): bb.utils.prunedir(destdir) - need_lfs = ud.parm.get("lfs", "1") == "1" + need_lfs = self._need_lfs(ud) if not need_lfs: ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd @@ -568,6 +597,9 @@ class Git(FetchMethod): raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) return output.split()[0] != "0" + def _need_lfs(self, ud): + return ud.parm.get("lfs", "1") == "1" + def _contains_lfs(self, ud, d, wd): """ Check if the repository has 'lfs' (large file) content @@ -578,8 +610,14 @@ class Git(FetchMethod): else: branchname = "master" - cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % ( - ud.basecmd, ud.branches[ud.names[0]]) + # The bare clonedir doesn't use the remote names; it has the branch immediately. + if wd == ud.clonedir: + refname = ud.branches[ud.names[0]] + else: + refname = "origin/%s" % ud.branches[ud.names[0]] + + cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( + ud.basecmd, refname) try: output = runfetchcmd(cmd, d, quiet=True, workdir=wd) -- cgit v1.2.3-54-g00ecf