summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRoss Burton <ross.burton@intel.com>2019-03-21 12:30:42 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-03-24 17:00:52 +0000
commit24b9bcf043adb020fef58ee8a7dc2a95d11b8bf0 (patch)
tree9ebfd77045f7b57d45ca7d6ebe3ae845a845d928
parent2187d06166ed6819a1d07c3f8a3be7a256fab18c (diff)
downloadpoky-24b9bcf043adb020fef58ee8a7dc2a95d11b8bf0.tar.gz
insane: improve license checksumming logic
Instead of opening files as bytes and battling decoding to UTF-8 which can throw exceptions, open directly as strings and replace invalid codepoints. This handles licenses in encodings which are not UTF-8 but are based on ASCII much better. Also instead of extracting the license lines, writing them to a file, and then hashing the file, hash the lines directly. (From OE-Core rev: 63ef9d342277c4ba541b78cbb45ef181f071f495) Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/insane.bbclass81
1 files changed, 32 insertions, 49 deletions
diff --git a/meta/classes/insane.bbclass b/meta/classes/insane.bbclass
index a2adfee835..37b8bb0032 100644
--- a/meta/classes/insane.bbclass
+++ b/meta/classes/insane.bbclass
@@ -458,7 +458,6 @@ python populate_lic_qa_checksum() {
458 """ 458 """
459 Check for changes in the license files. 459 Check for changes in the license files.
460 """ 460 """
461 import tempfile
462 sane = True 461 sane = True
463 462
464 lic_files = d.getVar('LIC_FILES_CHKSUM') or '' 463 lic_files = d.getVar('LIC_FILES_CHKSUM') or ''
@@ -496,61 +495,45 @@ python populate_lic_qa_checksum() {
496 495
497 if (not beginline) and (not endline): 496 if (not beginline) and (not endline):
498 md5chksum = bb.utils.md5_file(srclicfile) 497 md5chksum = bb.utils.md5_file(srclicfile)
499 with open(srclicfile, 'rb') as f: 498 with open(srclicfile, 'r', errors='replace') as f:
500 license = f.read() 499 license = f.read().splitlines()
501 else: 500 else:
502 fi = open(srclicfile, 'rb') 501 with open(srclicfile, 'rb') as f:
503 fo = tempfile.NamedTemporaryFile(mode='wb', prefix='poky.', suffix='.tmp', delete=False) 502 import hashlib
504 tmplicfile = fo.name; 503 lineno = 0
505 lineno = 0 504 license = []
506 linesout = 0 505 m = hashlib.md5()
507 license = [] 506 for line in f:
508 for line in fi: 507 lineno += 1
509 lineno += 1 508 if (lineno >= beginline):
510 if (lineno >= beginline): 509 if ((lineno <= endline) or not endline):
511 if ((lineno <= endline) or not endline): 510 m.update(line)
512 fo.write(line) 511 license.append(line.decode('utf-8', errors='replace').rstrip())
513 license.append(line) 512 else:
514 linesout += 1 513 break
515 else: 514 md5chksum = m.hexdigest()
516 break
517 fo.flush()
518 fo.close()
519 fi.close()
520 md5chksum = bb.utils.md5_file(tmplicfile)
521 license = b''.join(license)
522 os.unlink(tmplicfile)
523
524 if recipemd5 == md5chksum: 515 if recipemd5 == md5chksum:
525 bb.note (pn + ": md5 checksum matched for ", url) 516 bb.note (pn + ": md5 checksum matched for ", url)
526 else: 517 else:
527 if recipemd5: 518 if recipemd5:
528 msg = pn + ": The LIC_FILES_CHKSUM does not match for " + url 519 msg = pn + ": The LIC_FILES_CHKSUM does not match for " + url
529 msg = msg + "\n" + pn + ": The new md5 checksum is " + md5chksum 520 msg = msg + "\n" + pn + ": The new md5 checksum is " + md5chksum
530 try: 521 max_lines = int(d.getVar('QA_MAX_LICENSE_LINES') or 20)
531 license_lines = license.decode('utf-8').split('\n') 522 if not license or license[-1] != '':
532 except: 523 # Ensure that our license text ends with a line break
533 # License text might not be valid UTF-8, in which 524 # (will be added with join() below).
534 # case we don't know how to include it in our output 525 license.append('')
535 # and have to skip it. 526 remove = len(license) - max_lines
536 pass 527 if remove > 0:
537 else: 528 start = max_lines // 2
538 max_lines = int(d.getVar('QA_MAX_LICENSE_LINES') or 20) 529 end = start + remove - 1
539 if not license_lines or license_lines[-1] != '': 530 del license[start:end]
540 # Ensure that our license text ends with a line break 531 license.insert(start, '...')
541 # (will be added with join() below). 532 msg = msg + "\n" + pn + ": Here is the selected license text:" + \
542 license_lines.append('') 533 "\n" + \
543 remove = len(license_lines) - max_lines 534 "{:v^70}".format(" beginline=%d " % beginline if beginline else "") + \
544 if remove > 0: 535 "\n" + "\n".join(license) + \
545 start = max_lines // 2 536 "{:^^70}".format(" endline=%d " % endline if endline else "")
546 end = start + remove - 1
547 del license_lines[start:end]
548 license_lines.insert(start, '...')
549 msg = msg + "\n" + pn + ": Here is the selected license text:" + \
550 "\n" + \
551 "{:v^70}".format(" beginline=%d " % beginline if beginline else "") + \
552 "\n" + "\n".join(license_lines) + \
553 "{:^^70}".format(" endline=%d " % endline if endline else "")
554 if beginline: 537 if beginline:
555 if endline: 538 if endline:
556 srcfiledesc = "%s (lines %d through to %d)" % (srclicfile, beginline, endline) 539 srcfiledesc = "%s (lines %d through to %d)" % (srclicfile, beginline, endline)