summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2022-02-07 13:16:39 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-02-17 17:51:03 +0000
commitebfa1700f41b3411aec040144605166c35b8dd14 (patch)
tree37e9b4aba5a8c07e677bf1b982091ad835479821
parentd09e2e05a0391ad4d718214516d92a160f89767e (diff)
downloadpoky-ebfa1700f41b3411aec040144605166c35b8dd14.tar.gz
bitbake: fetch2/cooker: Fix source revision handling with floating upstreams
Where a git url uses a tag instead of a full source revision, breakage can currently occur in builds. Issues include: * the revision being looked up in multiple tasks (fetch and unpack) * the risk a different revision may be obtained in those tasks * that some tasks may not be allowed to access the network * that a revision may not be consistent throughout a given build * rerunning a specific task may given inconsistent results To fix this, stop the workers from cleaning out the source revision store. This should only be done in the cooker itself (based on current policy). Also, where the code "sees" an upstream access, mark the recipe as not to be cached. The reparse re-triggers the upstream lookup by the server. Add a test to ensure that if get_srcrev isn't called, the user is told they're using a configuration that is known to break. (Bitbake rev: 4b5eed1626709ef3dc06b32fd55d40a2a6edd179) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xbitbake/bin/bitbake-worker2
-rw-r--r--bitbake/lib/bb/cookerdata.py4
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py1
-rw-r--r--bitbake/lib/bb/fetch2/git.py6
-rw-r--r--bitbake/lib/bb/tests/fetch.py4
5 files changed, 14 insertions, 3 deletions
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index 063cf37926..9d850ec77c 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -440,7 +440,7 @@ class BitbakeWorker(object):
440 def handle_cookercfg(self, data): 440 def handle_cookercfg(self, data):
441 self.cookercfg = pickle.loads(data) 441 self.cookercfg = pickle.loads(data)
442 self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True) 442 self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
443 self.databuilder.parseBaseConfiguration() 443 self.databuilder.parseBaseConfiguration(worker=True)
444 self.data = self.databuilder.data 444 self.data = self.databuilder.data
445 445
446 def handle_extraconfigdata(self, data): 446 def handle_extraconfigdata(self, data):
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 397b43dfa7..a961f36574 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -258,12 +258,12 @@ class CookerDataBuilder(object):
258 self.data = self.basedata 258 self.data = self.basedata
259 self.mcdata = {} 259 self.mcdata = {}
260 260
261 def parseBaseConfiguration(self): 261 def parseBaseConfiguration(self, worker=False):
262 data_hash = hashlib.sha256() 262 data_hash = hashlib.sha256()
263 try: 263 try:
264 self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles) 264 self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
265 265
266 if self.data.getVar("BB_WORKERCONTEXT", False) is None: 266 if self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker:
267 bb.fetch.fetcher_init(self.data) 267 bb.fetch.fetcher_init(self.data)
268 bb.parse.init_parser(self.data) 268 bb.parse.init_parser(self.data)
269 bb.codeparser.parser_cache_init(self.data) 269 bb.codeparser.parser_cache_init(self.data)
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 28a3e54c7f..d099cd1092 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -765,6 +765,7 @@ def get_srcrev(d, method_name='sortable_revision'):
765 that fetcher provides a method with the given name and the same signature as sortable_revision. 765 that fetcher provides a method with the given name and the same signature as sortable_revision.
766 """ 766 """
767 767
768 d.setVar("__BBSEENSRCREV", "1")
768 recursion = d.getVar("__BBINSRCREV") 769 recursion = d.getVar("__BBINSRCREV")
769 if recursion: 770 if recursion:
770 raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI") 771 raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI")
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index 30da8e95b7..836ef1c49a 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -727,6 +727,12 @@ class Git(FetchMethod):
727 """ 727 """
728 Compute the HEAD revision for the url 728 Compute the HEAD revision for the url
729 """ 729 """
730 if not d.getVar("__BBSEENSRCREV"):
731 raise bb.fetch2.FetchError("Recipe uses a floating tag/branch without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE).")
732
733 # Ensure we mark as not cached
734 bb.fetch2.get_autorev(d)
735
730 output = self._lsremote(ud, d, "") 736 output = self._lsremote(ud, d, "")
731 # Tags of the form ^{} may not work, need to fallback to other form 737 # Tags of the form ^{} may not work, need to fallback to other form
732 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead: 738 if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index f368cd02b0..f0d417a50d 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -728,6 +728,7 @@ class FetcherLocalTest(FetcherTest):
728 728
729 # Fetch and check revision 729 # Fetch and check revision
730 self.d.setVar("SRCREV", "AUTOINC") 730 self.d.setVar("SRCREV", "AUTOINC")
731 self.d.setVar("__BBSEENSRCREV", "1")
731 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix 732 url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix
732 fetcher = bb.fetch.Fetch([url], self.d) 733 fetcher = bb.fetch.Fetch([url], self.d)
733 fetcher.download() 734 fetcher.download()
@@ -1581,6 +1582,7 @@ class GitShallowTest(FetcherTest):
1581 self.d.setVar('BB_GIT_SHALLOW', '1') 1582 self.d.setVar('BB_GIT_SHALLOW', '1')
1582 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0') 1583 self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
1583 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1') 1584 self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
1585 self.d.setVar("__BBSEENSRCREV", "1")
1584 1586
1585 def assertRefs(self, expected_refs, cwd=None): 1587 def assertRefs(self, expected_refs, cwd=None):
1586 if cwd is None: 1588 if cwd is None:
@@ -2138,6 +2140,7 @@ class GitLfsTest(FetcherTest):
2138 2140
2139 self.d.setVar('SRCREV', '${AUTOREV}') 2141 self.d.setVar('SRCREV', '${AUTOREV}')
2140 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}') 2142 self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
2143 self.d.setVar("__BBSEENSRCREV", "1")
2141 2144
2142 bb.utils.mkdirhier(self.srcdir) 2145 bb.utils.mkdirhier(self.srcdir)
2143 self.git_init(cwd=self.srcdir) 2146 self.git_init(cwd=self.srcdir)
@@ -2746,6 +2749,7 @@ class GitSharedTest(FetcherTest):
2746 super(GitSharedTest, self).setUp() 2749 super(GitSharedTest, self).setUp()
2747 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master" 2750 self.recipe_url = "git://git.openembedded.org/bitbake;branch=master"
2748 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40') 2751 self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
2752 self.d.setVar("__BBSEENSRCREV", "1")
2749 2753
2750 @skipIfNoNetwork() 2754 @skipIfNoNetwork()
2751 def test_shared_unpack(self): 2755 def test_shared_unpack(self):