summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/fetch2/git.py44
-rw-r--r--bitbake/lib/bb/tests/fetch.py28
2 files changed, 61 insertions, 11 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index df9538a60d..e3ba80a3f5 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -384,6 +384,35 @@ class Git(FetchMethod):
384 if missing_rev: 384 if missing_rev:
385 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) 385 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
386 386
387 if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
388 # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
389 # of all LFS blobs needed at the the srcrev.
390 #
391 # It would be nice to just do this inline here by running 'git-lfs fetch'
392 # on the bare clonedir, but that operation requires a working copy on some
393 # releases of Git LFS.
394 tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
395 try:
396 # Do the checkout. This implicitly involves a Git LFS fetch.
397 self.unpack(ud, tmpdir, d)
398
399 # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
400 # the bare clonedir.
401 #
402 # As this procedure is invoked repeatedly on incremental fetches as
403 # a recipe's SRCREV is bumped throughout its lifetime, this will
404 # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
405 # corresponding to all the blobs reachable from the different revs
406 # fetched across time.
407 #
408 # Only do this if the unpack resulted in a .git/lfs directory being
409 # created; this only happens if at least one blob needed to be
410 # downloaded.
411 if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
412 runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
413 finally:
414 bb.utils.remove(tmpdir, recurse=True)
415
387 def build_mirror_data(self, ud, d): 416 def build_mirror_data(self, ud, d):
388 if ud.shallow and ud.write_shallow_tarballs: 417 if ud.shallow and ud.write_shallow_tarballs:
389 if not os.path.exists(ud.fullshallow): 418 if not os.path.exists(ud.fullshallow):
@@ -479,7 +508,7 @@ class Git(FetchMethod):
479 if os.path.exists(destdir): 508 if os.path.exists(destdir):
480 bb.utils.prunedir(destdir) 509 bb.utils.prunedir(destdir)
481 510
482 need_lfs = ud.parm.get("lfs", "1") == "1" 511 need_lfs = self._need_lfs(ud)
483 512
484 if not need_lfs: 513 if not need_lfs:
485 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd 514 ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd
@@ -568,6 +597,9 @@ class Git(FetchMethod):
568 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) 597 raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
569 return output.split()[0] != "0" 598 return output.split()[0] != "0"
570 599
600 def _need_lfs(self, ud):
601 return ud.parm.get("lfs", "1") == "1"
602
571 def _contains_lfs(self, ud, d, wd): 603 def _contains_lfs(self, ud, d, wd):
572 """ 604 """
573 Check if the repository has 'lfs' (large file) content 605 Check if the repository has 'lfs' (large file) content
@@ -578,8 +610,14 @@ class Git(FetchMethod):
578 else: 610 else:
579 branchname = "master" 611 branchname = "master"
580 612
581 cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % ( 613 # The bare clonedir doesn't use the remote names; it has the branch immediately.
582 ud.basecmd, ud.branches[ud.names[0]]) 614 if wd == ud.clonedir:
615 refname = ud.branches[ud.names[0]]
616 else:
617 refname = "origin/%s" % ud.branches[ud.names[0]]
618
619 cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
620 ud.basecmd, refname)
583 621
584 try: 622 try:
585 output = runfetchcmd(cmd, d, quiet=True, workdir=wd) 623 output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 20beab3855..7b2dac7b86 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -2108,13 +2108,14 @@ class GitLfsTest(FetcherTest):
2108 cwd = self.gitdir 2108 cwd = self.gitdir
2109 return bb.process.run(cmd, cwd=cwd)[0] 2109 return bb.process.run(cmd, cwd=cwd)[0]
2110 2110
2111 def fetch(self, uri=None): 2111 def fetch(self, uri=None, download=True):
2112 uris = self.d.getVar('SRC_URI').split() 2112 uris = self.d.getVar('SRC_URI').split()
2113 uri = uris[0] 2113 uri = uris[0]
2114 d = self.d 2114 d = self.d
2115 2115
2116 fetcher = bb.fetch2.Fetch(uris, d) 2116 fetcher = bb.fetch2.Fetch(uris, d)
2117 fetcher.download() 2117 if download:
2118 fetcher.download()
2118 ud = fetcher.ud[uri] 2119 ud = fetcher.ud[uri]
2119 return fetcher, ud 2120 return fetcher, ud
2120 2121
@@ -2124,16 +2125,21 @@ class GitLfsTest(FetcherTest):
2124 uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir 2125 uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
2125 self.d.setVar('SRC_URI', uri) 2126 self.d.setVar('SRC_URI', uri)
2126 2127
2127 fetcher, ud = self.fetch() 2128 # Careful: suppress initial attempt at downloading until
2129 # we know whether git-lfs is installed.
2130 fetcher, ud = self.fetch(uri=None, download=False)
2128 self.assertIsNotNone(ud.method._find_git_lfs) 2131 self.assertIsNotNone(ud.method._find_git_lfs)
2129 2132
2130 # If git-lfs can be found, the unpack should be successful 2133 # If git-lfs can be found, the unpack should be successful. Only
2131 ud.method._find_git_lfs = lambda d: True 2134 # attempt this with the real live copy of git-lfs installed.
2132 shutil.rmtree(self.gitdir, ignore_errors=True) 2135 if ud.method._find_git_lfs(self.d):
2133 fetcher.unpack(self.d.getVar('WORKDIR')) 2136 fetcher.download()
2137 shutil.rmtree(self.gitdir, ignore_errors=True)
2138 fetcher.unpack(self.d.getVar('WORKDIR'))
2134 2139
2135 # If git-lfs cannot be found, the unpack should throw an error 2140 # If git-lfs cannot be found, the unpack should throw an error
2136 with self.assertRaises(bb.fetch2.FetchError): 2141 with self.assertRaises(bb.fetch2.FetchError):
2142 fetcher.download()
2137 ud.method._find_git_lfs = lambda d: False 2143 ud.method._find_git_lfs = lambda d: False
2138 shutil.rmtree(self.gitdir, ignore_errors=True) 2144 shutil.rmtree(self.gitdir, ignore_errors=True)
2139 fetcher.unpack(self.d.getVar('WORKDIR')) 2145 fetcher.unpack(self.d.getVar('WORKDIR'))
@@ -2144,10 +2150,16 @@ class GitLfsTest(FetcherTest):
2144 uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir 2150 uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
2145 self.d.setVar('SRC_URI', uri) 2151 self.d.setVar('SRC_URI', uri)
2146 2152
2153 # In contrast to test_lfs_enabled(), allow the implicit download
2154 # done by self.fetch() to occur here. The point of this test case
2155 # is to verify that the fetcher can survive even if the source
2156 # repository has Git LFS usage configured.
2147 fetcher, ud = self.fetch() 2157 fetcher, ud = self.fetch()
2148 self.assertIsNotNone(ud.method._find_git_lfs) 2158 self.assertIsNotNone(ud.method._find_git_lfs)
2149 2159
2150 # If git-lfs can be found, the unpack should be successful 2160 # If git-lfs can be found, the unpack should be successful. A
2161 # live copy of git-lfs is not required for this case, so
2162 # unconditionally forge its presence.
2151 ud.method._find_git_lfs = lambda d: True 2163 ud.method._find_git_lfs = lambda d: True
2152 shutil.rmtree(self.gitdir, ignore_errors=True) 2164 shutil.rmtree(self.gitdir, ignore_errors=True)
2153 fetcher.unpack(self.d.getVar('WORKDIR')) 2165 fetcher.unpack(self.d.getVar('WORKDIR'))