summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2/git.py
diff options
context:
space:
mode:
authorMatt Hoosier <matt.hoosier@garmin.com>2021-02-02 13:38:19 +0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-02-02 15:51:56 +0000
commit95057b9cde43445b5b96fc50b62d30416f0c41e8 (patch)
treeba2f31bec0ee703cd0d56b2e0fcb29c8cf0224d8 /bitbake/lib/bb/fetch2/git.py
parent9de9e2e3194525c8eea57ebc3c7396b4695549a7 (diff)
downloadpoky-95057b9cde43445b5b96fc50b62d30416f0c41e8.tar.gz
bitbake: fetch/git: download LFS content too during do_fetch
Insert an explicit pass to fetch all blobs needed by Git LFS, during the fetch() function. This avoids the default behavior of Git LFS to wait until 'git checkout' to begin downloading the blobs pointed to by LFS records. Network access is not allowed at that point in the recipe's lifecycle. [YOCTO #14191] (Bitbake rev: c73f8f2f4a6491c6bea54839630af6994c27ad24) Signed-off-by: Matt Hoosier <matt.hoosier@garmin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> (cherry picked from commit 0efac66043662e7a2027192f50e92e982db2ba1c) Signed-off-by: Anuj Mittal <anuj.mittal@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch2/git.py')
-rw-r--r--bitbake/lib/bb/fetch2/git.py44
1 files changed, 41 insertions, 3 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index b97967b487..b59a7cdc84 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -379,6 +379,35 @@ class Git(FetchMethod):
379 if missing_rev: 379 if missing_rev:
380 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) 380 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
381 381
382 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
383 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
384 # of all LFS blobs needed at the the srcrev.
385 #
386 # It would be nice to just do this inline here by running 'git-lfs fetch'
387 # on the bare clonedir, but that operation requires a working copy on some
388 # releases of Git LFS.
389 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
390 try:
391 # Do the checkout. This implicitly involves a Git LFS fetch.
392 self.unpack(ud, tmpdir, d)
393
394 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
395 # the bare clonedir.
396 #
397 # As this procedure is invoked repeatedly on incremental fetches as
398 # a recipe's SRCREV is bumped throughout its lifetime, this will
399 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
400 # corresponding to all the blobs reachable from the different revs
401 # fetched across time.
402 #
403 # Only do this if the unpack resulted in a .git/lfs directory being
404 # created; this only happens if at least one blob needed to be
405 # downloaded.
406 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
407 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
408 finally:
409 bb.utils.remove(tmpdir, recurse=True)
410
382 def build_mirror_data(self, ud, d): 411 def build_mirror_data(self, ud, d):
383 if ud.shallow and ud.write_shallow_tarballs: 412 if ud.shallow and ud.write_shallow_tarballs:
384 if not os.path.exists(ud.fullshallow): 413 if not os.path.exists(ud.fullshallow):
@@ -474,7 +503,7 @@ class Git(FetchMethod):
474 if os.path.exists(destdir): 503 if os.path.exists(destdir):
475 bb.utils.prunedir(destdir) 504 bb.utils.prunedir(destdir)
476 505
477 need_lfs = ud.parm.get("lfs", "1") == "1" 506 need_lfs = self._need_lfs(ud)
478 507
479 if not need_lfs: 508 if not need_lfs:
480 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd 509 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
@@ -563,6 +592,9 @@ class Git(FetchMethod):
563 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) 592 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
564 return output.split()[0] != "0" 593 return output.split()[0] != "0"
565 594
595 def _need_lfs(self, ud):
596 return ud.parm.get("lfs", "1") == "1"
597
566 def _contains_lfs(self, ud, d, wd): 598 def _contains_lfs(self, ud, d, wd):
567 """ 599 """
568 Check if the repository has 'lfs' (large file) content 600 Check if the repository has 'lfs' (large file) content
@@ -573,8 +605,14 @@ class Git(FetchMethod):
573 else: 605 else:
574 branchname = "master" 606 branchname = "master"
575 607
576 cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % ( 608 # The bare clonedir doesn't use the remote names; it has the branch immediately.
577 ud.basecmd, ud.branches[ud.names[0]]) 609 if wd == ud.clonedir:
610 refname = ud.branches[ud.names[0]]
611 else:
612 refname = "origin/%s" % ud.branches[ud.names[0]]
613
614 cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
615 ud.basecmd, refname)
578 616
579 try: 617 try:
580 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 618 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)