summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorMatt Hoosier <matt.hoosier@garmin.com>2021-01-22 10:55:13 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-01-23 17:10:46 +0000
commit45c02843d181b5f1872ad30f4d847c395233a676 (patch)
tree1b61559ebf14f9b5a97b283b6860dd795024b674 /bitbake/lib
parent8e92ec78322909ca5ad6ceda582b611fa77a9964 (diff)
downloadpoky-45c02843d181b5f1872ad30f4d847c395233a676.tar.gz
bitbake: fetch/git: download LFS content too during do_fetch
Insert an explicit pass to fetch all blobs needed by Git LFS, during the fetch() function. This avoids the default behavior of Git LFS to wait until 'git checkout' to begin downloading the blobs pointed to by LFS records. Network access is not allowed at that point in the recipe's lifecycle. [YOCTO #14191] (Bitbake rev: 0efac66043662e7a2027192f50e92e982db2ba1c) Signed-off-by: Matt Hoosier <matt.hoosier@garmin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/fetch2/git.py44
-rw-r--r--bitbake/lib/bb/tests/fetch.py28
2 files changed, 61 insertions, 11 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index df9538a60d..e3ba80a3f5 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -384,6 +384,35 @@ class Git(FetchMethod):
384 if missing_rev: 384 if missing_rev:
385 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) 385 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
386 386
387 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
388 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
389 # of all LFS blobs needed at the the srcrev.
390 #
391 # It would be nice to just do this inline here by running 'git-lfs fetch'
392 # on the bare clonedir, but that operation requires a working copy on some
393 # releases of Git LFS.
394 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
395 try:
396 # Do the checkout. This implicitly involves a Git LFS fetch.
397 self.unpack(ud, tmpdir, d)
398
399 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
400 # the bare clonedir.
401 #
402 # As this procedure is invoked repeatedly on incremental fetches as
403 # a recipe's SRCREV is bumped throughout its lifetime, this will
404 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
405 # corresponding to all the blobs reachable from the different revs
406 # fetched across time.
407 #
408 # Only do this if the unpack resulted in a .git/lfs directory being
409 # created; this only happens if at least one blob needed to be
410 # downloaded.
411 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
412 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
413 finally:
414 bb.utils.remove(tmpdir, recurse=True)
415
387 def build_mirror_data(self, ud, d): 416 def build_mirror_data(self, ud, d):
388 if ud.shallow and ud.write_shallow_tarballs: 417 if ud.shallow and ud.write_shallow_tarballs:
389 if not os.path.exists(ud.fullshallow): 418 if not os.path.exists(ud.fullshallow):
@@ -479,7 +508,7 @@ class Git(FetchMethod):
479 if os.path.exists(destdir): 508 if os.path.exists(destdir):
480 bb.utils.prunedir(destdir) 509 bb.utils.prunedir(destdir)
481 510
482 need_lfs = ud.parm.get("lfs", "1") == "1" 511 need_lfs = self._need_lfs(ud)
483 512
484 if not need_lfs: 513 if not need_lfs:
485 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd 514 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
@@ -568,6 +597,9 @@ class Git(FetchMethod):
568 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) 597 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
569 return output.split()[0] != "0" 598 return output.split()[0] != "0"
570 599
600 def _need_lfs(self, ud):
601 return ud.parm.get("lfs", "1") == "1"
602
571 def _contains_lfs(self, ud, d, wd): 603 def _contains_lfs(self, ud, d, wd):
572 """ 604 """
573 Check if the repository has 'lfs' (large file) content 605 Check if the repository has 'lfs' (large file) content
@@ -578,8 +610,14 @@ class Git(FetchMethod):
578 else: 610 else:
579 branchname = "master" 611 branchname = "master"
580 612
581 cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % ( 613 # The bare clonedir doesn't use the remote names; it has the branch immediately.
582 ud.basecmd, ud.branches[ud.names[0]]) 614 if wd == ud.clonedir:
615 refname = ud.branches[ud.names[0]]
616 else:
617 refname = "origin/%s" % ud.branches[ud.names[0]]
618
619 cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
620 ud.basecmd, refname)
583 621
584 try: 622 try:
585 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 623 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 20beab3855..7b2dac7b86 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -2108,13 +2108,14 @@ class GitLfsTest(FetcherTest):
2108 cwd = self.gitdir 2108 cwd = self.gitdir
2109 return bb.process.run(cmd, cwd=cwd)[0] 2109 return bb.process.run(cmd, cwd=cwd)[0]
2110 2110
2111 def fetch(self, uri=None): 2111 def fetch(self, uri=None, download=True):
2112 uris = self.d.getVar('SRC_URI').split() 2112 uris = self.d.getVar('SRC_URI').split()
2113 uri = uris[0] 2113 uri = uris[0]
2114 d = self.d 2114 d = self.d
2115 2115
2116 fetcher = bb.fetch2.Fetch(uris, d) 2116 fetcher = bb.fetch2.Fetch(uris, d)
2117 fetcher.download() 2117 if download:
2118 fetcher.download()
2118 ud = fetcher.ud[uri] 2119 ud = fetcher.ud[uri]
2119 return fetcher, ud 2120 return fetcher, ud
2120 2121
@@ -2124,16 +2125,21 @@ class GitLfsTest(FetcherTest):
2124 uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir 2125 uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
2125 self.d.setVar('SRC_URI', uri) 2126 self.d.setVar('SRC_URI', uri)
2126 2127
2127 fetcher, ud = self.fetch() 2128 # Careful: suppress initial attempt at downloading until
2129 # we know whether git-lfs is installed.
2130 fetcher, ud = self.fetch(uri=None, download=False)
2128 self.assertIsNotNone(ud.method._find_git_lfs) 2131 self.assertIsNotNone(ud.method._find_git_lfs)
2129 2132
2130 # If git-lfs can be found, the unpack should be successful 2133 # If git-lfs can be found, the unpack should be successful. Only
2131 ud.method._find_git_lfs = lambda d: True 2134 # attempt this with the real live copy of git-lfs installed.
2132 shutil.rmtree(self.gitdir, ignore_errors=True) 2135 if ud.method._find_git_lfs(self.d):
2133 fetcher.unpack(self.d.getVar('WORKDIR')) 2136 fetcher.download()
2137 shutil.rmtree(self.gitdir, ignore_errors=True)
2138 fetcher.unpack(self.d.getVar('WORKDIR'))
2134 2139
2135 # If git-lfs cannot be found, the unpack should throw an error 2140 # If git-lfs cannot be found, the unpack should throw an error
2136 with self.assertRaises(bb.fetch2.FetchError): 2141 with self.assertRaises(bb.fetch2.FetchError):
2142 fetcher.download()
2137 ud.method._find_git_lfs = lambda d: False 2143 ud.method._find_git_lfs = lambda d: False
2138 shutil.rmtree(self.gitdir, ignore_errors=True) 2144 shutil.rmtree(self.gitdir, ignore_errors=True)
2139 fetcher.unpack(self.d.getVar('WORKDIR')) 2145 fetcher.unpack(self.d.getVar('WORKDIR'))
@@ -2144,10 +2150,16 @@ class GitLfsTest(FetcherTest):
2144 uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir 2150 uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
2145 self.d.setVar('SRC_URI', uri) 2151 self.d.setVar('SRC_URI', uri)
2146 2152
2153 # In contrast to test_lfs_enabled(), allow the implicit download
2154 # done by self.fetch() to occur here. The point of this test case
2155 # is to verify that the fetcher can survive even if the source
2156 # repository has Git LFS usage configured.
2147 fetcher, ud = self.fetch() 2157 fetcher, ud = self.fetch()
2148 self.assertIsNotNone(ud.method._find_git_lfs) 2158 self.assertIsNotNone(ud.method._find_git_lfs)
2149 2159
2150 # If git-lfs can be found, the unpack should be successful 2160 # If git-lfs can be found, the unpack should be successful. A
2161 # live copy of git-lfs is not required for this case, so
2162 # unconditionally forge its presence.
2151 ud.method._find_git_lfs = lambda d: True 2163 ud.method._find_git_lfs = lambda d: True
2152 shutil.rmtree(self.gitdir, ignore_errors=True) 2164 shutil.rmtree(self.gitdir, ignore_errors=True)
2153 fetcher.unpack(self.d.getVar('WORKDIR')) 2165 fetcher.unpack(self.d.getVar('WORKDIR'))