summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe
diff options
context:
space:
mode:
authorJoshua Watt <jpewhacker@gmail.com>2019-01-21 16:39:19 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-22 14:35:58 +0000
commit1d86f65ff54164f23d96d76dec4b1f468f4bab06 (patch)
tree7b4cf11cc005b6cdc45e56cdb0d664a5375da20e /meta/lib/oe
parent6fd870e6a1f61d17e43cf30db4259a939db93820 (diff)
downloadpoky-1d86f65ff54164f23d96d76dec4b1f468f4bab06.tar.gz
classes/sstate: Update output hash
Updates the output hash calculation for determining if tasks are equivalent. The new algorithm does the following based on feedback: 1) The output hash function was moved to the OE library. 2) All files are printed in a single line tabular format 3) Prints the file type and mode in a user-friendly ls-like format 4) Includes the file owner and group (by name, not ID). These are only included if the task is run under pseudo since that is the only time they can be consistently determined. 5) File size is included for regular files (From OE-Core rev: 4bd297dfe92851f3b44f6b5560bac9d8f9ccf9f2) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oe')
-rw-r--r--meta/lib/oe/sstatesig.py139
1 files changed, 136 insertions, 3 deletions
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index e0eb87e29f..a83af519ec 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -270,7 +270,7 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
270 super().init_rundepcheck(data) 270 super().init_rundepcheck(data)
271 self.server = data.getVar('SSTATE_HASHEQUIV_SERVER') 271 self.server = data.getVar('SSTATE_HASHEQUIV_SERVER')
272 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') 272 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD')
273 self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method, data) 273 self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data)
274 274
275 def get_taskdata(self): 275 def get_taskdata(self):
276 return (self.server, self.method) + super().get_taskdata() 276 return (self.server, self.method) + super().get_taskdata()
@@ -355,6 +355,7 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
355 import json 355 import json
356 import tempfile 356 import tempfile
357 import base64 357 import base64
358 import importlib
358 359
359 taskhash = d.getVar('BB_TASKHASH') 360 taskhash = d.getVar('BB_TASKHASH')
360 unihash = d.getVar('BB_UNIHASH') 361 unihash = d.getVar('BB_UNIHASH')
@@ -376,11 +377,14 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
376 sigfile_link = "depsig.do_%s" % task 377 sigfile_link = "depsig.do_%s" % task
377 378
378 try: 379 try:
379 call = self.method + '(path, sigfile, task, d)'
380 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') 380 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
381
381 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} 382 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
382 383
383 outhash = bb.utils.better_eval(call, locs) 384 (module, method) = self.method.rsplit('.', 1)
385 locs['method'] = getattr(importlib.import_module(module), method)
386
387 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
384 388
385 try: 389 try:
386 url = '%s/v1/equivalent' % self.server 390 url = '%s/v1/equivalent' % self.server
@@ -581,4 +585,133 @@ def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache):
581 bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant)) 585 bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant))
582 return None, d2 586 return None, d2
583 587
588def OEOuthashBasic(path, sigfile, task, d):
589 """
590 Basic output hash function
591
592 Calculates the output hash of a task by hashing all output file metadata,
593 and file contents.
594 """
595 import hashlib
596 import stat
597 import pwd
598 import grp
599
600 def update_hash(s):
601 s = s.encode('utf-8')
602 h.update(s)
603 if sigfile:
604 sigfile.write(s)
605
606 h = hashlib.sha256()
607 prev_dir = os.getcwd()
608 include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
609
610 try:
611 os.chdir(path)
612
613 update_hash("OEOuthashBasic\n")
614
615 # It is only currently useful to get equivalent hashes for things that
616 # can be restored from sstate. Since the sstate object is named using
617 # SSTATE_PKGSPEC and the task name, those should be included in the
618 # output hash calculation.
619 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC'))
620 update_hash("task=%s\n" % task)
621
622 for root, dirs, files in os.walk('.', topdown=True):
623 # Sort directories to ensure consistent ordering when recursing
624 dirs.sort()
625 files.sort()
626
627 def process(path):
628 s = os.lstat(path)
629
630 if stat.S_ISDIR(s.st_mode):
631 update_hash('d')
632 elif stat.S_ISCHR(s.st_mode):
633 update_hash('c')
634 elif stat.S_ISBLK(s.st_mode):
635 update_hash('b')
636 elif stat.S_ISSOCK(s.st_mode):
637 update_hash('s')
638 elif stat.S_ISLNK(s.st_mode):
639 update_hash('l')
640 elif stat.S_ISFIFO(s.st_mode):
641 update_hash('p')
642 else:
643 update_hash('-')
644
645 def add_perm(mask, on, off='-'):
646 if mask & s.st_mode:
647 update_hash(on)
648 else:
649 update_hash(off)
650
651 add_perm(stat.S_IRUSR, 'r')
652 add_perm(stat.S_IWUSR, 'w')
653 if stat.S_ISUID & s.st_mode:
654 add_perm(stat.S_IXUSR, 's', 'S')
655 else:
656 add_perm(stat.S_IXUSR, 'x')
657
658 add_perm(stat.S_IRGRP, 'r')
659 add_perm(stat.S_IWGRP, 'w')
660 if stat.S_ISGID & s.st_mode:
661 add_perm(stat.S_IXGRP, 's', 'S')
662 else:
663 add_perm(stat.S_IXGRP, 'x')
664
665 add_perm(stat.S_IROTH, 'r')
666 add_perm(stat.S_IWOTH, 'w')
667 if stat.S_ISVTX & s.st_mode:
668 update_hash('t')
669 else:
670 add_perm(stat.S_IXOTH, 'x')
671
672 if include_owners:
673 update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
674 update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
675
676 update_hash(" ")
677 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
678 update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
679 else:
680 update_hash(" " * 9)
681
682 update_hash(" ")
683 if stat.S_ISREG(s.st_mode):
684 update_hash("%10d" % s.st_size)
685 else:
686 update_hash(" " * 10)
687
688 update_hash(" ")
689 fh = hashlib.sha256()
690 if stat.S_ISREG(s.st_mode):
691 # Hash file contents
692 with open(path, 'rb') as d:
693 for chunk in iter(lambda: d.read(4096), b""):
694 fh.update(chunk)
695 update_hash(fh.hexdigest())
696 else:
697 update_hash(" " * len(fh.hexdigest()))
698
699 update_hash(" %s" % path)
700
701 if stat.S_ISLNK(s.st_mode):
702 update_hash(" -> %s" % os.readlink(path))
703
704 update_hash("\n")
705
706 # Process this directory and all its child files
707 process(root)
708 for f in files:
709 if f == 'fixmepath':
710 continue
711 process(os.path.join(root, f))
712 finally:
713 os.chdir(prev_dir)
714
715 return h.hexdigest()
716
584 717