summaryrefslogtreecommitdiffstats
path: root/meta/lib
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2021-09-29 14:02:08 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-10-04 14:03:53 (GMT)
commit86c7d3e0310f1602327365a360199efe5ea05fc8 (patch)
tree6ca83ac3466048b8b915a522f327c25810f2d7fe /meta/lib
parent0b4198eeac06d40861256496169a2e6bb275916c (diff)
downloadpoky-86c7d3e0310f1602327365a360199efe5ea05fc8.tar.gz
sstatesig: Add processing for full build paths in sysroot files
Some files in the populate_sysroot tasks have hardcoded paths in them, particularly if they are postinst-useradd- files or crossscripts. Add some filtering logic to remove these paths. This means that the hashequiv "outhash" matches correcting in more cases allowing for better build artefact reuse. To make this work a new variable is added SSTATE_HASHEQUIV_FILEMAP which maps file globbing to replacement patterns (paths or regex) on a per sstate task basis. It is hoped this shouldn't be needed in many cases. We are in the process to developing QA tests which will better detect issues in this area to allow optimal sstate reuse. (From OE-Core rev: d9852ffbbe728dac33dc081538a08af98f52fd4a) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib')
-rw-r--r--meta/lib/oe/sstatesig.py39
1 files changed, 36 insertions, 3 deletions
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 2457724..0c3b458 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -470,6 +470,8 @@ def OEOuthashBasic(path, sigfile, task, d):
470 import stat 470 import stat
471 import pwd 471 import pwd
472 import grp 472 import grp
473 import re
474 import fnmatch
473 475
474 def update_hash(s): 476 def update_hash(s):
475 s = s.encode('utf-8') 477 s = s.encode('utf-8')
@@ -479,6 +481,8 @@ def OEOuthashBasic(path, sigfile, task, d):
479 481
480 h = hashlib.sha256() 482 h = hashlib.sha256()
481 prev_dir = os.getcwd() 483 prev_dir = os.getcwd()
484 corebase = d.getVar("COREBASE")
485 tmpdir = d.getVar("TMPDIR")
482 include_owners = os.environ.get('PSEUDO_DISABLED') == '0' 486 include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
483 if "package_write_" in task or task == "package_qa": 487 if "package_write_" in task or task == "package_qa":
484 include_owners = False 488 include_owners = False
@@ -489,8 +493,17 @@ def OEOuthashBasic(path, sigfile, task, d):
489 include_root = False 493 include_root = False
490 extra_content = d.getVar('HASHEQUIV_HASH_VERSION') 494 extra_content = d.getVar('HASHEQUIV_HASH_VERSION')
491 495
496 filemaps = {}
497 for m in (d.getVar('SSTATE_HASHEQUIV_FILEMAP') or '').split():
498 entry = m.split(":")
499 if len(entry) != 3 or entry[0] != task:
500 continue
501 filemaps.setdefault(entry[1], [])
502 filemaps[entry[1]].append(entry[2])
503
492 try: 504 try:
493 os.chdir(path) 505 os.chdir(path)
506 basepath = os.path.normpath(path)
494 507
495 update_hash("OEOuthashBasic\n") 508 update_hash("OEOuthashBasic\n")
496 if extra_content: 509 if extra_content:
@@ -572,8 +585,13 @@ def OEOuthashBasic(path, sigfile, task, d):
572 else: 585 else:
573 update_hash(" " * 9) 586 update_hash(" " * 9)
574 587
588 filterfile = False
589 for entry in filemaps:
590 if fnmatch.fnmatch(path, entry):
591 filterfile = True
592
575 update_hash(" ") 593 update_hash(" ")
576 if stat.S_ISREG(s.st_mode): 594 if stat.S_ISREG(s.st_mode) and not filterfile:
577 update_hash("%10d" % s.st_size) 595 update_hash("%10d" % s.st_size)
578 else: 596 else:
579 update_hash(" " * 10) 597 update_hash(" " * 10)
@@ -582,9 +600,24 @@ def OEOuthashBasic(path, sigfile, task, d):
582 fh = hashlib.sha256() 600 fh = hashlib.sha256()
583 if stat.S_ISREG(s.st_mode): 601 if stat.S_ISREG(s.st_mode):
584 # Hash file contents 602 # Hash file contents
585 with open(path, 'rb') as d: 603 if filterfile:
586 for chunk in iter(lambda: d.read(4096), b""): 604 # Need to ignore paths in crossscripts and postinst-useradd files.
605 with open(path, 'rb') as d:
606 chunk = d.read()
607 chunk = chunk.replace(bytes(basepath, encoding='utf8'), b'')
608 for entry in filemaps:
609 if not fnmatch.fnmatch(path, entry):
610 continue
611 for r in filemaps[entry]:
612 if r.startswith("regex-"):
613 chunk = re.sub(bytes(r[6:], encoding='utf8'), b'', chunk)
614 else:
615 chunk = chunk.replace(bytes(r, encoding='utf8'), b'')
587 fh.update(chunk) 616 fh.update(chunk)
617 else:
618 with open(path, 'rb') as d:
619 for chunk in iter(lambda: d.read(4096), b""):
620 fh.update(chunk)
588 update_hash(fh.hexdigest()) 621 update_hash(fh.hexdigest())
589 else: 622 else:
590 update_hash(" " * len(fh.hexdigest())) 623 update_hash(" " * len(fh.hexdigest()))