summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/fetch2
diff options
context:
space:
mode:
authorJean-Marie LEMETAYER <jean-marie.lemetayer@savoirfairelinux.com>2020-01-24 18:08:05 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-01-27 16:48:10 +0000
commitd6bd2c1a4bdae66006a0b189e381eee2cc2744f0 (patch)
tree68175d5ee31538ac6e6e71ee2b8a340bedf3581a /bitbake/lib/bb/fetch2
parentbd184ea6ff09f9d5b47ac1b3d1140bc34aa420f2 (diff)
downloadpoky-d6bd2c1a4bdae66006a0b189e381eee2cc2744f0.tar.gz
bitbake: fetch2: refactor checksum verification
This commit refactors the way checksums are verified to be more generic. The support of new hash functions is now limited to the update of the CHECKSUM_LIST variable. (Bitbake rev: debd9eeaf5638755d8956b2d65b904fe02826966) Signed-off-by: Jean-Marie LEMETAYER <jean-marie.lemetayer@savoirfairelinux.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/fetch2')
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py156
1 files changed, 85 insertions, 71 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 406a91cce4..9762c36045 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -33,6 +33,8 @@ _checksum_cache = bb.checksum.FileChecksumCache()
33 33
34logger = logging.getLogger("BitBake.Fetcher") 34logger = logging.getLogger("BitBake.Fetcher")
35 35
36CHECKSUM_LIST = [ "md5", "sha256" ]
37
36class BBFetchException(Exception): 38class BBFetchException(Exception):
37 """Class all fetch exceptions inherit from""" 39 """Class all fetch exceptions inherit from"""
38 def __init__(self, message): 40 def __init__(self, message):
@@ -131,10 +133,9 @@ class NonLocalMethod(Exception):
131 Exception.__init__(self) 133 Exception.__init__(self)
132 134
133class MissingChecksumEvent(bb.event.Event): 135class MissingChecksumEvent(bb.event.Event):
134 def __init__(self, url, md5sum, sha256sum): 136 def __init__(self, url, **checksums):
135 self.url = url 137 self.url = url
136 self.checksums = {'md5sum': md5sum, 138 self.checksums = checksums
137 'sha256sum': sha256sum}
138 bb.event.Event.__init__(self) 139 bb.event.Event.__init__(self)
139 140
140 141
@@ -552,71 +553,82 @@ def verify_checksum(ud, d, precomputed={}):
552 downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571. 553 downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
553 """ 554 """
554 555
555 _MD5_KEY = "md5"
556 _SHA256_KEY = "sha256"
557
558 if ud.ignore_checksums or not ud.method.supports_checksum(ud): 556 if ud.ignore_checksums or not ud.method.supports_checksum(ud):
559 return {} 557 return {}
560 558
561 if _MD5_KEY in precomputed: 559 def compute_checksum_info(checksum_id):
562 md5data = precomputed[_MD5_KEY] 560 checksum_name = getattr(ud, "%s_name" % checksum_id)
563 else:
564 md5data = bb.utils.md5_file(ud.localpath)
565 561
566 if _SHA256_KEY in precomputed: 562 if checksum_id in precomputed:
567 sha256data = precomputed[_SHA256_KEY] 563 checksum_data = precomputed[checksum_id]
568 else: 564 else:
569 sha256data = bb.utils.sha256_file(ud.localpath) 565 checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath)
570 566
571 if ud.method.recommends_checksum(ud) and not ud.md5_expected and not ud.sha256_expected: 567 checksum_expected = getattr(ud, "%s_expected" % checksum_id)
572 # If strict checking enabled and neither sum defined, raise error 568
569 return {
570 "id": checksum_id,
571 "name": checksum_name,
572 "data": checksum_data,
573 "expected": checksum_expected
574 }
575
576 checksum_infos = []
577 for checksum_id in CHECKSUM_LIST:
578 checksum_infos.append(compute_checksum_info(checksum_id))
579
580 checksum_dict = {ci["id"] : ci["data"] for ci in checksum_infos}
581 checksum_event = {"%ssum" % ci["id"] : ci["data"] for ci in checksum_infos}
582
583 checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"]) for ci in checksum_infos]
584
585 # If no checksum has been provided
586 if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
587 messages = []
573 strict = d.getVar("BB_STRICT_CHECKSUM") or "0" 588 strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
589
590 # If strict checking enabled and neither sum defined, raise error
574 if strict == "1": 591 if strict == "1":
575 logger.error('No checksum specified for %s, please add at least one to the recipe:\n' 592 messages.append("No checksum specified for '%s', please add at " \
576 'SRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"' % 593 "least one to the recipe:" % ud.localpath)
577 (ud.localpath, ud.md5_name, md5data, 594 messages.extend(checksum_lines)
578 ud.sha256_name, sha256data)) 595 logger.error("\n".join(messages))
579 raise NoChecksumError('Missing SRC_URI checksum', ud.url) 596 raise NoChecksumError("Missing SRC_URI checksum", ud.url)
580 597
581 bb.event.fire(MissingChecksumEvent(ud.url, md5data, sha256data), d) 598 bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
582 599
583 if strict == "ignore": 600 if strict == "ignore":
584 return { 601 return checksum_dict
585 _MD5_KEY: md5data,
586 _SHA256_KEY: sha256data
587 }
588 602
589 # Log missing sums so user can more easily add them 603 # Log missing sums so user can more easily add them
590 logger.warning('Missing md5 SRC_URI checksum for %s, consider adding to the recipe:\n' 604 messages.append("Missing checksum for '%s', consider adding at " \
591 'SRC_URI[%s] = "%s"', 605 "least one to the recipe:" % ud.localpath)
592 ud.localpath, ud.md5_name, md5data) 606 messages.extend(checksum_lines)
593 logger.warning('Missing sha256 SRC_URI checksum for %s, consider adding to the recipe:\n' 607 logger.warning("\n".join(messages))
594 'SRC_URI[%s] = "%s"',
595 ud.localpath, ud.sha256_name, sha256data)
596 608
597 # We want to alert the user if a checksum is defined in the recipe but 609 # We want to alert the user if a checksum is defined in the recipe but
598 # it does not match. 610 # it does not match.
599 msg = "" 611 messages = []
600 mismatch = False 612 messages.append("Checksum mismatch!")
601 if ud.md5_expected and ud.md5_expected != md5data: 613 bad_checksum = None
602 msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'md5', md5data, ud.md5_expected) 614
603 mismatch = True; 615 for ci in checksum_infos:
604 616 if ci["expected"] and ci["expected"] != ci["data"]:
605 if ud.sha256_expected and ud.sha256_expected != sha256data: 617 messages.append("File: '%s' has %s checksum %s when %s was " \
606 msg = msg + "\nFile: '%s' has %s checksum %s when %s was expected" % (ud.localpath, 'sha256', sha256data, ud.sha256_expected) 618 "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
607 mismatch = True; 619 bad_checksum = ci["data"]
608 620
609 if mismatch: 621 if bad_checksum:
610 msg = msg + '\nIf this change is expected (e.g. you have upgraded to a new version without updating the checksums) then you can use these lines within the recipe:\nSRC_URI[%s] = "%s"\nSRC_URI[%s] = "%s"\nOtherwise you should retry the download and/or check with upstream to determine if the file has become corrupted or otherwise unexpectedly modified.\n' % (ud.md5_name, md5data, ud.sha256_name, sha256data) 622 messages.append("If this change is expected (e.g. you have upgraded " \
611 623 "to a new version without updating the checksums) " \
612 if len(msg): 624 "then you can use these lines within the recipe:")
613 raise ChecksumError('Checksum mismatch!%s' % msg, ud.url, md5data) 625 messages.extend(checksum_lines)
614 626 messages.append("Otherwise you should retry the download and/or " \
615 return { 627 "check with upstream to determine if the file has " \
616 _MD5_KEY: md5data, 628 "become corrupted or otherwise unexpectedly modified.")
617 _SHA256_KEY: sha256data 629 raise ChecksumError("\n".join(messages), ud.url, bad_checksum)
618 } 630
619 631 return checksum_dict
620 632
621def verify_donestamp(ud, d, origud=None): 633def verify_donestamp(ud, d, origud=None):
622 """ 634 """
@@ -1230,24 +1242,26 @@ class FetchData(object):
1230 self.pswd = self.parm["pswd"] 1242 self.pswd = self.parm["pswd"]
1231 self.setup = False 1243 self.setup = False
1232 1244
1233 if "name" in self.parm: 1245 def configure_checksum(checksum_id):
1234 self.md5_name = "%s.md5sum" % self.parm["name"] 1246 if "name" in self.parm:
1235 self.sha256_name = "%s.sha256sum" % self.parm["name"] 1247 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id)
1236 else: 1248 else:
1237 self.md5_name = "md5sum" 1249 checksum_name = "%ssum" % checksum_id
1238 self.sha256_name = "sha256sum" 1250
1239 if self.md5_name in self.parm: 1251 setattr(self, "%s_name" % checksum_id, checksum_name)
1240 self.md5_expected = self.parm[self.md5_name] 1252
1241 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: 1253 if checksum_name in self.parm:
1242 self.md5_expected = None 1254 checksum_expected = self.parm[checksum_name]
1243 else: 1255 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
1244 self.md5_expected = d.getVarFlag("SRC_URI", self.md5_name) 1256 checksum_expected = None
1245 if self.sha256_name in self.parm: 1257 else:
1246 self.sha256_expected = self.parm[self.sha256_name] 1258 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
1247 elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: 1259
1248 self.sha256_expected = None 1260 setattr(self, "%s_expected" % checksum_id, checksum_expected)
1249 else: 1261
1250 self.sha256_expected = d.getVarFlag("SRC_URI", self.sha256_name) 1262 for checksum_id in CHECKSUM_LIST:
1263 configure_checksum(checksum_id)
1264
1251 self.ignore_checksums = False 1265 self.ignore_checksums = False
1252 1266
1253 self.names = self.parm.get("name",'default').split(',') 1267 self.names = self.parm.get("name",'default').split(',')