summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--meta/classes/sstate.bbclass65
-rw-r--r--meta/lib/oe/sstatesig.py139
2 files changed, 139 insertions, 65 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 763fce07f9..2f0bbd2d7d 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -83,9 +83,9 @@ SSTATE_SIG_PASSPHRASE ?= ""
83# Whether to verify the GnUPG signatures when extracting sstate archives 83# Whether to verify the GnUPG signatures when extracting sstate archives
84SSTATE_VERIFY_SIG ?= "0" 84SSTATE_VERIFY_SIG ?= "0"
85 85
86SSTATE_HASHEQUIV_METHOD ?= "OEOuthashBasic" 86SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic"
87SSTATE_HASHEQUIV_METHOD[doc] = "The function used to calculate the output hash \ 87SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \
88 for a task, which in turn is used to determine equivalency. \ 88 the output hash for a task, which in turn is used to determine equivalency. \
89 " 89 "
90 90
91SSTATE_HASHEQUIV_SERVER ?= "" 91SSTATE_HASHEQUIV_SERVER ?= ""
@@ -782,65 +782,6 @@ python sstate_sign_package () {
782 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) 782 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
783} 783}
784 784
785def OEOuthashBasic(path, sigfile, task, d):
786 import hashlib
787 import stat
788
789 def update_hash(s):
790 s = s.encode('utf-8')
791 h.update(s)
792 if sigfile:
793 sigfile.write(s)
794
795 h = hashlib.sha256()
796 prev_dir = os.getcwd()
797
798 try:
799 os.chdir(path)
800
801 update_hash("OEOuthashBasic\n")
802
803 # It is only currently useful to get equivalent hashes for things that
804 # can be restored from sstate. Since the sstate object is named using
805 # SSTATE_PKGSPEC and the task name, those should be included in the
806 # output hash calculation.
807 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC'))
808 update_hash("task=%s\n" % task)
809
810 for root, dirs, files in os.walk('.', topdown=True):
811 # Sort directories and files to ensure consistent ordering
812 dirs.sort()
813 files.sort()
814
815 for f in files:
816 path = os.path.join(root, f)
817 s = os.lstat(path)
818
819 # Hash file path
820 update_hash(path + '\n')
821
822 # Hash file mode
823 update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode))
824 update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode))
825
826 if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode):
827 # Hash device major and minor
828 update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev)))
829 elif stat.S_ISLNK(s.st_mode):
830 # Hash symbolic link
831 update_hash("\tsymlink=%s\n" % os.readlink(path))
832 else:
833 fh = hashlib.sha256()
834 # Hash file contents
835 with open(path, 'rb') as d:
836 for chunk in iter(lambda: d.read(4096), b""):
837 fh.update(chunk)
838 update_hash("\tdigest=%s\n" % fh.hexdigest())
839 finally:
840 os.chdir(prev_dir)
841
842 return h.hexdigest()
843
844python sstate_report_unihash() { 785python sstate_report_unihash() {
845 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None) 786 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
846 787
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index e0eb87e29f..a83af519ec 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -270,7 +270,7 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
270 super().init_rundepcheck(data) 270 super().init_rundepcheck(data)
271 self.server = data.getVar('SSTATE_HASHEQUIV_SERVER') 271 self.server = data.getVar('SSTATE_HASHEQUIV_SERVER')
272 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') 272 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD')
273 self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method, data) 273 self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data)
274 274
275 def get_taskdata(self): 275 def get_taskdata(self):
276 return (self.server, self.method) + super().get_taskdata() 276 return (self.server, self.method) + super().get_taskdata()
@@ -355,6 +355,7 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
355 import json 355 import json
356 import tempfile 356 import tempfile
357 import base64 357 import base64
358 import importlib
358 359
359 taskhash = d.getVar('BB_TASKHASH') 360 taskhash = d.getVar('BB_TASKHASH')
360 unihash = d.getVar('BB_UNIHASH') 361 unihash = d.getVar('BB_UNIHASH')
@@ -376,11 +377,14 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
376 sigfile_link = "depsig.do_%s" % task 377 sigfile_link = "depsig.do_%s" % task
377 378
378 try: 379 try:
379 call = self.method + '(path, sigfile, task, d)'
380 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') 380 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
381
381 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} 382 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
382 383
383 outhash = bb.utils.better_eval(call, locs) 384 (module, method) = self.method.rsplit('.', 1)
385 locs['method'] = getattr(importlib.import_module(module), method)
386
387 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
384 388
385 try: 389 try:
386 url = '%s/v1/equivalent' % self.server 390 url = '%s/v1/equivalent' % self.server
@@ -581,4 +585,133 @@ def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache):
581 bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant)) 585 bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant))
582 return None, d2 586 return None, d2
583 587
588def OEOuthashBasic(path, sigfile, task, d):
589 """
590 Basic output hash function
591
592 Calculates the output hash of a task by hashing all output file metadata,
593 and file contents.
594 """
595 import hashlib
596 import stat
597 import pwd
598 import grp
599
600 def update_hash(s):
601 s = s.encode('utf-8')
602 h.update(s)
603 if sigfile:
604 sigfile.write(s)
605
606 h = hashlib.sha256()
607 prev_dir = os.getcwd()
608 include_owners = os.environ.get('PSEUDO_DISABLED') == '0'
609
610 try:
611 os.chdir(path)
612
613 update_hash("OEOuthashBasic\n")
614
615 # It is only currently useful to get equivalent hashes for things that
616 # can be restored from sstate. Since the sstate object is named using
617 # SSTATE_PKGSPEC and the task name, those should be included in the
618 # output hash calculation.
619 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC'))
620 update_hash("task=%s\n" % task)
621
622 for root, dirs, files in os.walk('.', topdown=True):
623 # Sort directories to ensure consistent ordering when recursing
624 dirs.sort()
625 files.sort()
626
627 def process(path):
628 s = os.lstat(path)
629
630 if stat.S_ISDIR(s.st_mode):
631 update_hash('d')
632 elif stat.S_ISCHR(s.st_mode):
633 update_hash('c')
634 elif stat.S_ISBLK(s.st_mode):
635 update_hash('b')
636 elif stat.S_ISSOCK(s.st_mode):
637 update_hash('s')
638 elif stat.S_ISLNK(s.st_mode):
639 update_hash('l')
640 elif stat.S_ISFIFO(s.st_mode):
641 update_hash('p')
642 else:
643 update_hash('-')
644
645 def add_perm(mask, on, off='-'):
646 if mask & s.st_mode:
647 update_hash(on)
648 else:
649 update_hash(off)
650
651 add_perm(stat.S_IRUSR, 'r')
652 add_perm(stat.S_IWUSR, 'w')
653 if stat.S_ISUID & s.st_mode:
654 add_perm(stat.S_IXUSR, 's', 'S')
655 else:
656 add_perm(stat.S_IXUSR, 'x')
657
658 add_perm(stat.S_IRGRP, 'r')
659 add_perm(stat.S_IWGRP, 'w')
660 if stat.S_ISGID & s.st_mode:
661 add_perm(stat.S_IXGRP, 's', 'S')
662 else:
663 add_perm(stat.S_IXGRP, 'x')
664
665 add_perm(stat.S_IROTH, 'r')
666 add_perm(stat.S_IWOTH, 'w')
667 if stat.S_ISVTX & s.st_mode:
668 update_hash('t')
669 else:
670 add_perm(stat.S_IXOTH, 'x')
671
672 if include_owners:
673 update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name)
674 update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name)
675
676 update_hash(" ")
677 if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode):
678 update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))))
679 else:
680 update_hash(" " * 9)
681
682 update_hash(" ")
683 if stat.S_ISREG(s.st_mode):
684 update_hash("%10d" % s.st_size)
685 else:
686 update_hash(" " * 10)
687
688 update_hash(" ")
689 fh = hashlib.sha256()
690 if stat.S_ISREG(s.st_mode):
691 # Hash file contents
692 with open(path, 'rb') as d:
693 for chunk in iter(lambda: d.read(4096), b""):
694 fh.update(chunk)
695 update_hash(fh.hexdigest())
696 else:
697 update_hash(" " * len(fh.hexdigest()))
698
699 update_hash(" %s" % path)
700
701 if stat.S_ISLNK(s.st_mode):
702 update_hash(" -> %s" % os.readlink(path))
703
704 update_hash("\n")
705
706 # Process this directory and all its child files
707 process(root)
708 for f in files:
709 if f == 'fixmepath':
710 continue
711 process(os.path.join(root, f))
712 finally:
713 os.chdir(prev_dir)
714
715 return h.hexdigest()
716
584 717