summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2022-09-01 10:06:34 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-09-05 12:54:25 +0100
commit58c42911d452e57505b08915618723961df5ef89 (patch)
tree9dcf92f76035e745bc3af153df49b405e083a0d2 /bitbake
parentd2420daacfefba20d26500e0b053855caff62238 (diff)
downloadpoky-58c42911d452e57505b08915618723961df5ef89.tar.gz
bitbake: wget: Avoid bad checksum race issues
If two recipes have conflicting checksums for a file, the code will currently remove the existing file when a mismatch is downloaded, even if another task successfully fetched it. This changes the code to verify the checksum (if possible) before replacing the file. This removes a potential race window and stops builds failing everywhere from one incorrect checksum. To make this work, we need to be able to override localpath and avoid NoChecksum errors being logged. (Bitbake rev: 4b8de2e7d12667d69d86ffe6e9f85a7932c4c9a5) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py11
-rw-r--r--bitbake/lib/bb/fetch2/wget.py5
2 files changed, 12 insertions, 4 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 0fb718b23e..e6dd79c4aa 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -545,7 +545,7 @@ def mirror_from_string(data):
545 bb.warn('Invalid mirror data %s, should have paired members.' % data) 545 bb.warn('Invalid mirror data %s, should have paired members.' % data)
546 return list(zip(*[iter(mirrors)]*2)) 546 return list(zip(*[iter(mirrors)]*2))
547 547
548def verify_checksum(ud, d, precomputed={}): 548def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
549 """ 549 """
550 verify the MD5 and SHA256 checksum for downloaded src 550 verify the MD5 and SHA256 checksum for downloaded src
551 551
@@ -563,13 +563,16 @@ def verify_checksum(ud, d, precomputed={}):
563 if ud.ignore_checksums or not ud.method.supports_checksum(ud): 563 if ud.ignore_checksums or not ud.method.supports_checksum(ud):
564 return {} 564 return {}
565 565
566 if localpath is None:
567 localpath = ud.localpath
568
566 def compute_checksum_info(checksum_id): 569 def compute_checksum_info(checksum_id):
567 checksum_name = getattr(ud, "%s_name" % checksum_id) 570 checksum_name = getattr(ud, "%s_name" % checksum_id)
568 571
569 if checksum_id in precomputed: 572 if checksum_id in precomputed:
570 checksum_data = precomputed[checksum_id] 573 checksum_data = precomputed[checksum_id]
571 else: 574 else:
572 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath) 575 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath)
573 576
574 checksum_expected = getattr(ud, "%s_expected" % checksum_id) 577 checksum_expected = getattr(ud, "%s_expected" % checksum_id)
575 578
@@ -595,7 +598,7 @@ def verify_checksum(ud, d, precomputed={}):
595 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])] 598 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
596 599
597 # If no checksum has been provided 600 # If no checksum has been provided
598 if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos): 601 if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
599 messages = [] 602 messages = []
600 strict = d.getVar("BB_STRICT_CHECKSUM") or "0" 603 strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
601 604
@@ -627,7 +630,7 @@ def verify_checksum(ud, d, precomputed={}):
627 for ci in checksum_infos: 630 for ci in checksum_infos:
628 if ci["expected"] and ci["expected"] != ci["data"]: 631 if ci["expected"] and ci["expected"] != ci["data"]:
629 messages.append("File: '%s' has %s checksum '%s' when '%s' was " \ 632 messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
630 "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"])) 633 "expected" % (localpath, ci["id"], ci["data"], ci["expected"]))
631 bad_checksum = ci["data"] 634 bad_checksum = ci["data"]
632 635
633 if bad_checksum: 636 if bad_checksum:
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index b2b542e1dc..821afa5b58 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -132,6 +132,11 @@ class Wget(FetchMethod):
132 132
133 self._runwget(ud, d, fetchcmd, False) 133 self._runwget(ud, d, fetchcmd, False)
134 134
135 # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
136 # original file, which might be a race (imagine two recipes referencing the same
137 # source, one with an incorrect checksum)
138 bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
139
135 # Remove the ".tmp" and move the file into position atomically 140 # Remove the ".tmp" and move the file into position atomically
136 # Our lock prevents multiple writers but mirroring code may grab incomplete files 141 # Our lock prevents multiple writers but mirroring code may grab incomplete files
137 os.rename(localpath, localpath[:-4]) 142 os.rename(localpath, localpath[:-4])