diff options
-rw-r--r-- | bitbake/lib/bb/fetch2/git.py | 45 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/fetch.py | 51 |
2 files changed, 90 insertions, 6 deletions
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index df33fb6aeb..b43ee0da39 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py | |||
@@ -325,7 +325,10 @@ class Git(FetchMethod): | |||
325 | return ud.clonedir | 325 | return ud.clonedir |
326 | 326 | ||
327 | def need_update(self, ud, d): | 327 | def need_update(self, ud, d): |
328 | return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud) | 328 | return self.clonedir_need_update(ud, d) \ |
329 | or self.shallow_tarball_need_update(ud) \ | ||
330 | or self.tarball_need_update(ud) \ | ||
331 | or self.lfs_need_update(ud, d) | ||
329 | 332 | ||
330 | def clonedir_need_update(self, ud, d): | 333 | def clonedir_need_update(self, ud, d): |
331 | if not os.path.exists(ud.clonedir): | 334 | if not os.path.exists(ud.clonedir): |
@@ -337,6 +340,15 @@ class Git(FetchMethod): | |||
337 | return True | 340 | return True |
338 | return False | 341 | return False |
339 | 342 | ||
343 | def lfs_need_update(self, ud, d): | ||
344 | if self.clonedir_need_update(ud, d): | ||
345 | return True | ||
346 | |||
347 | for name in ud.names: | ||
348 | if not self._lfs_objects_downloaded(ud, d, name, ud.clonedir): | ||
349 | return True | ||
350 | return False | ||
351 | |||
340 | def clonedir_need_shallow_revs(self, ud, d): | 352 | def clonedir_need_shallow_revs(self, ud, d): |
341 | for rev in ud.shallow_revs: | 353 | for rev in ud.shallow_revs: |
342 | try: | 354 | try: |
@@ -467,7 +479,7 @@ class Git(FetchMethod): | |||
467 | if missing_rev: | 479 | if missing_rev: |
468 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) | 480 | raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev) |
469 | 481 | ||
470 | if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud): | 482 | if self.lfs_need_update(ud, d): |
471 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching | 483 | # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching |
472 | # of all LFS blobs needed at the srcrev. | 484 | # of all LFS blobs needed at the srcrev. |
473 | # | 485 | # |
@@ -710,6 +722,35 @@ class Git(FetchMethod): | |||
710 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) | 722 | raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output)) |
711 | return output.split()[0] != "0" | 723 | return output.split()[0] != "0" |
712 | 724 | ||
725 | def _lfs_objects_downloaded(self, ud, d, name, wd): | ||
726 | """ | ||
727 | Verifies whether the LFS objects for requested revisions have already been downloaded | ||
728 | """ | ||
729 | # Bail out early if this repository doesn't use LFS | ||
730 | if not self._need_lfs(ud) or not self._contains_lfs(ud, d, wd): | ||
731 | return True | ||
732 | |||
733 | # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file | ||
734 | # existence. | ||
735 | # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git | ||
736 | cmd = "%s lfs ls-files -l %s" \ | ||
737 | % (ud.basecmd, ud.revisions[name]) | ||
738 | output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip() | ||
739 | # Do not do any further matching if no objects are managed by LFS | ||
740 | if not output: | ||
741 | return True | ||
742 | |||
743 | # Match all lines beginning with the hexadecimal OID | ||
744 | oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)") | ||
745 | for line in output.split("\n"): | ||
746 | oid = re.search(oid_regex, line) | ||
747 | if not oid: | ||
748 | bb.warn("git lfs ls-files output '%s' did not match expected format." % line) | ||
749 | if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))): | ||
750 | return False | ||
751 | |||
752 | return True | ||
753 | |||
713 | def _need_lfs(self, ud): | 754 | def _need_lfs(self, ud): |
714 | return ud.parm.get("lfs", "1") == "1" | 755 | return ud.parm.get("lfs", "1") == "1" |
715 | 756 | ||
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py index 5ed5b5607f..e988e26c0a 100644 --- a/bitbake/lib/bb/tests/fetch.py +++ b/bitbake/lib/bb/tests/fetch.py | |||
@@ -6,6 +6,7 @@ | |||
6 | # SPDX-License-Identifier: GPL-2.0-only | 6 | # SPDX-License-Identifier: GPL-2.0-only |
7 | # | 7 | # |
8 | 8 | ||
9 | import contextlib | ||
9 | import unittest | 10 | import unittest |
10 | import hashlib | 11 | import hashlib |
11 | import tempfile | 12 | import tempfile |
@@ -2261,10 +2262,14 @@ class GitLfsTest(FetcherTest): | |||
2261 | 2262 | ||
2262 | bb.utils.mkdirhier(self.srcdir) | 2263 | bb.utils.mkdirhier(self.srcdir) |
2263 | self.git_init(cwd=self.srcdir) | 2264 | self.git_init(cwd=self.srcdir) |
2264 | with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs: | 2265 | self.commit_file('.gitattributes', '*.mp3 filter=lfs -text') |
2265 | attrs.write('*.mp3 filter=lfs -text') | 2266 | |
2266 | self.git(['add', '.gitattributes'], cwd=self.srcdir) | 2267 | def commit_file(self, filename, content): |
2267 | self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir) | 2268 | with open(os.path.join(self.srcdir, filename), "w") as f: |
2269 | f.write(content) | ||
2270 | self.git(["add", filename], cwd=self.srcdir) | ||
2271 | self.git(["commit", "-m", "Change"], cwd=self.srcdir) | ||
2272 | return self.git(["rev-parse", "HEAD"], cwd=self.srcdir).strip() | ||
2268 | 2273 | ||
2269 | def fetch(self, uri=None, download=True): | 2274 | def fetch(self, uri=None, download=True): |
2270 | uris = self.d.getVar('SRC_URI').split() | 2275 | uris = self.d.getVar('SRC_URI').split() |
@@ -2285,6 +2290,44 @@ class GitLfsTest(FetcherTest): | |||
2285 | return unpacked_lfs_file | 2290 | return unpacked_lfs_file |
2286 | 2291 | ||
2287 | @skipIfNoGitLFS() | 2292 | @skipIfNoGitLFS() |
2293 | def test_fetch_lfs_on_srcrev_change(self): | ||
2294 | """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested""" | ||
2295 | self.git(["lfs", "install", "--local"], cwd=self.srcdir) | ||
2296 | |||
2297 | @contextlib.contextmanager | ||
2298 | def hide_upstream_repository(): | ||
2299 | """Hide the upstream repository to make sure that git lfs cannot pull from it""" | ||
2300 | temp_name = self.srcdir + ".bak" | ||
2301 | os.rename(self.srcdir, temp_name) | ||
2302 | try: | ||
2303 | yield | ||
2304 | finally: | ||
2305 | os.rename(temp_name, self.srcdir) | ||
2306 | |||
2307 | def fetch_and_verify(revision, filename, content): | ||
2308 | self.d.setVar('SRCREV', revision) | ||
2309 | fetcher, ud = self.fetch() | ||
2310 | |||
2311 | with hide_upstream_repository(): | ||
2312 | workdir = self.d.getVar('WORKDIR') | ||
2313 | fetcher.unpack(workdir) | ||
2314 | |||
2315 | with open(os.path.join(workdir, "git", filename)) as f: | ||
2316 | self.assertEqual(f.read(), content) | ||
2317 | |||
2318 | commit_1 = self.commit_file("a.mp3", "version 1") | ||
2319 | commit_2 = self.commit_file("a.mp3", "version 2") | ||
2320 | |||
2321 | self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir) | ||
2322 | |||
2323 | # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are | ||
2324 | # available even when the upstream repository disappears. | ||
2325 | fetch_and_verify(commit_2, "a.mp3", "version 2") | ||
2326 | # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download | ||
2327 | # folder. | ||
2328 | fetch_and_verify(commit_1, "a.mp3", "version 1") | ||
2329 | |||
2330 | @skipIfNoGitLFS() | ||
2288 | @skipIfNoNetwork() | 2331 | @skipIfNoNetwork() |
2289 | def test_real_git_lfs_repo_succeeds_without_lfs_param(self): | 2332 | def test_real_git_lfs_repo_succeeds_without_lfs_param(self): |
2290 | self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") | 2333 | self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master") |