diff options
Diffstat (limited to 'bitbake')
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 44 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/fetch.py | 28 |
2 files changed, 61 insertions, 11 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 07064c694f..8740e9c05f 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -378,6 +378,35 @@ class Git(FetchMethod): | |||
378 | if missing_rev: | 378 | if missing_rev: |
379 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) | 379 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) |
380 | 380 | ||
381 | if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): | ||
382 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching | ||
383 | # of all LFS blobs needed at the the srcrev. | ||
384 | # | ||
385 | # It would be nice to just do this inline here by running 'git-lfs fetch' | ||
386 | # on the bare clonedir, but that operation requires a working copy on some | ||
387 | # releases of Git LFS. | ||
388 | tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR')) | ||
389 | try: | ||
390 | # Do the checkout. This implicitly involves a Git LFS fetch. | ||
391 | self.unpack(ud, tmpdir, d) | ||
392 | |||
393 | # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into | ||
394 | # the bare clonedir. | ||
395 | # | ||
396 | # As this procedure is invoked repeatedly on incremental fetches as | ||
397 | # a recipe's SRCREV is bumped throughout its lifetime, this will | ||
398 | # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs | ||
399 | # corresponding to all the blobs reachable from the different revs | ||
400 | # fetched across time. | ||
401 | # | ||
402 | # Only do this if the unpack resulted in a .git/lfs directory being | ||
403 | # created; this only happens if at least one blob needed to be | ||
404 | # downloaded. | ||
405 | if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")): | ||
406 | runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir) | ||
407 | finally: | ||
408 | bb.utils.remove(tmpdir, recurse=True) | ||
409 | |||
381 | def build_mirror_data(self, ud, d): | 410 | def build_mirror_data(self, ud, d): |
382 | if ud.shallow and ud.write_shallow_tarballs: | 411 | if ud.shallow and ud.write_shallow_tarballs: |
383 | if not os.path.exists(ud.fullshallow): | 412 | if not os.path.exists(ud.fullshallow): |
@@ -473,7 +502,7 @@ class Git(FetchMethod): | |||
473 | if os.path.exists(destdir): | 502 | if os.path.exists(destdir): |
474 | bb.utils.prunedir(destdir) | 503 | bb.utils.prunedir(destdir) |
475 | 504 | ||
476 | need_lfs = ud.parm.get("lfs", "1") == "1" | 505 | need_lfs = self._need_lfs(ud) |
477 | 506 | ||
478 | if not need_lfs: | 507 | if not need_lfs: |
479 | ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd | 508 | ud.basecmd = "GIT_LFS_SKIP_SMUDGE=1 " + ud.basecmd |
@@ -562,6 +591,9 @@ class Git(FetchMethod): | |||
562 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) | 591 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) |
563 | return output.split()[0] != "0" | 592 | return output.split()[0] != "0" |
564 | 593 | ||
594 | def _need_lfs(self, ud): | ||
595 | return ud.parm.get("lfs", "1") == "1" | ||
596 | |||
565 | def _contains_lfs(self, ud, d, wd): | 597 | def _contains_lfs(self, ud, d, wd): |
566 | """ | 598 | """ |
567 | Check if the repository has 'lfs' (large file) content | 599 | Check if the repository has 'lfs' (large file) content |
@@ -572,8 +604,14 @@ class Git(FetchMethod): | |||
572 | else: | 604 | else: |
573 | branchname = "master" | 605 | branchname = "master" |
574 | 606 | ||
575 | cmd = "%s grep lfs origin/%s:.gitattributes | wc -l" % ( | 607 | # The bare clonedir doesn't use the remote names; it has the branch immediately. |
576 | ud.basecmd, ud.branches[ud.names[0]]) | 608 | if wd == ud.clonedir: |
609 | refname = ud.branches[ud.names[0]] | ||
610 | else: | ||
611 | refname = "origin/%s" % ud.branches[ud.names[0]] | ||
612 | |||
613 | cmd = "%s grep lfs %s:.gitattributes | wc -l" % ( | ||
614 | ud.basecmd, refname) | ||
577 | 615 | ||
578 | try: | 616 | try: |
579 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) | 617 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd) |
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 4702c99a7e..9453c90d2b 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py | |||
@@ -2046,13 +2046,14 @@ class GitLfsTest(FetcherTest): | |||
2046 | cwd = self.gitdir | 2046 | cwd = self.gitdir |
2047 | return bb.process.run(cmd, cwd=cwd)[0] | 2047 | return bb.process.run(cmd, cwd=cwd)[0] |
2048 | 2048 | ||
2049 | def fetch(self, uri=None): | 2049 | def fetch(self, uri=None, download=True): |
2050 | uris = self.d.getVar('SRC_URI').split() | 2050 | uris = self.d.getVar('SRC_URI').split() |
2051 | uri = uris[0] | 2051 | uri = uris[0] |
2052 | d = self.d | 2052 | d = self.d |
2053 | 2053 | ||
2054 | fetcher = bb.fetch2.Fetch(uris, d) | 2054 | fetcher = bb.fetch2.Fetch(uris, d) |
2055 | fetcher.download() | 2055 | if download: |
2056 | fetcher.download() | ||
2056 | ud = fetcher.ud[uri] | 2057 | ud = fetcher.ud[uri] |
2057 | return fetcher, ud | 2058 | return fetcher, ud |
2058 | 2059 | ||
@@ -2062,16 +2063,21 @@ class GitLfsTest(FetcherTest): | |||
2062 | uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir | 2063 | uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir |
2063 | self.d.setVar('SRC_URI', uri) | 2064 | self.d.setVar('SRC_URI', uri) |
2064 | 2065 | ||
2065 | fetcher, ud = self.fetch() | 2066 | # Careful: suppress initial attempt at downloading until |
2067 | # we know whether git-lfs is installed. | ||
2068 | fetcher, ud = self.fetch(uri=None, download=False) | ||
2066 | self.assertIsNotNone(ud.method._find_git_lfs) | 2069 | self.assertIsNotNone(ud.method._find_git_lfs) |
2067 | 2070 | ||
2068 | # If git-lfs can be found, the unpack should be successful | 2071 | # If git-lfs can be found, the unpack should be successful. Only |
2069 | ud.method._find_git_lfs = lambda d: True | 2072 | # attempt this with the real live copy of git-lfs installed. |
2070 | shutil.rmtree(self.gitdir, ignore_errors=True) | 2073 | if ud.method._find_git_lfs(self.d): |
2071 | fetcher.unpack(self.d.getVar('WORKDIR')) | 2074 | fetcher.download() |
2075 | shutil.rmtree(self.gitdir, ignore_errors=True) | ||
2076 | fetcher.unpack(self.d.getVar('WORKDIR')) | ||
2072 | 2077 | ||
2073 | # If git-lfs cannot be found, the unpack should throw an error | 2078 | # If git-lfs cannot be found, the unpack should throw an error |
2074 | with self.assertRaises(bb.fetch2.FetchError): | 2079 | with self.assertRaises(bb.fetch2.FetchError): |
2080 | fetcher.download() | ||
2075 | ud.method._find_git_lfs = lambda d: False | 2081 | ud.method._find_git_lfs = lambda d: False |
2076 | shutil.rmtree(self.gitdir, ignore_errors=True) | 2082 | shutil.rmtree(self.gitdir, ignore_errors=True) |
2077 | fetcher.unpack(self.d.getVar('WORKDIR')) | 2083 | fetcher.unpack(self.d.getVar('WORKDIR')) |
@@ -2082,10 +2088,16 @@ class GitLfsTest(FetcherTest): | |||
2082 | uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir | 2088 | uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir |
2083 | self.d.setVar('SRC_URI', uri) | 2089 | self.d.setVar('SRC_URI', uri) |
2084 | 2090 | ||
2091 | # In contrast to test_lfs_enabled(), allow the implicit download | ||
2092 | # done by self.fetch() to occur here. The point of this test case | ||
2093 | # is to verify that the fetcher can survive even if the source | ||
2094 | # repository has Git LFS usage configured. | ||
2085 | fetcher, ud = self.fetch() | 2095 | fetcher, ud = self.fetch() |
2086 | self.assertIsNotNone(ud.method._find_git_lfs) | 2096 | self.assertIsNotNone(ud.method._find_git_lfs) |
2087 | 2097 | ||
2088 | # If git-lfs can be found, the unpack should be successful | 2098 | # If git-lfs can be found, the unpack should be successful. A |
2099 | # live copy of git-lfs is not required for this case, so | ||
2100 | # unconditionally forge its presence. | ||
2089 | ud.method._find_git_lfs = lambda d: True | 2101 | ud.method._find_git_lfs = lambda d: True |
2090 | shutil.rmtree(self.gitdir, ignore_errors=True) | 2102 | shutil.rmtree(self.gitdir, ignore_errors=True) |
2091 | fetcher.unpack(self.d.getVar('WORKDIR')) | 2103 | fetcher.unpack(self.d.getVar('WORKDIR')) |