diff options
-rw-r--r-- | meta/classes/sstate.bbclass | 65 | ||||
-rw-r--r-- | meta/lib/oe/sstatesig.py | 139 |
2 files changed, 139 insertions, 65 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass index 763fce07f9..2f0bbd2d7d 100644 --- a/meta/classes/sstate.bbclass +++ b/meta/classes/sstate.bbclass | |||
@@ -83,9 +83,9 @@ SSTATE_SIG_PASSPHRASE ?= "" | |||
83 | # Whether to verify the GnUPG signatures when extracting sstate archives | 83 | # Whether to verify the GnUPG signatures when extracting sstate archives |
84 | SSTATE_VERIFY_SIG ?= "0" | 84 | SSTATE_VERIFY_SIG ?= "0" |
85 | 85 | ||
86 | SSTATE_HASHEQUIV_METHOD ?= "OEOuthashBasic" | 86 | SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic" |
87 | SSTATE_HASHEQUIV_METHOD[doc] = "The function used to calculate the output hash \ | 87 | SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \ |
88 | for a task, which in turn is used to determine equivalency. \ | 88 | the output hash for a task, which in turn is used to determine equivalency. \ |
89 | " | 89 | " |
90 | 90 | ||
91 | SSTATE_HASHEQUIV_SERVER ?= "" | 91 | SSTATE_HASHEQUIV_SERVER ?= "" |
@@ -782,65 +782,6 @@ python sstate_sign_package () { | |||
782 | d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) | 782 | d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) |
783 | } | 783 | } |
784 | 784 | ||
785 | def OEOuthashBasic(path, sigfile, task, d): | ||
786 | import hashlib | ||
787 | import stat | ||
788 | |||
789 | def update_hash(s): | ||
790 | s = s.encode('utf-8') | ||
791 | h.update(s) | ||
792 | if sigfile: | ||
793 | sigfile.write(s) | ||
794 | |||
795 | h = hashlib.sha256() | ||
796 | prev_dir = os.getcwd() | ||
797 | |||
798 | try: | ||
799 | os.chdir(path) | ||
800 | |||
801 | update_hash("OEOuthashBasic\n") | ||
802 | |||
803 | # It is only currently useful to get equivalent hashes for things that | ||
804 | # can be restored from sstate. Since the sstate object is named using | ||
805 | # SSTATE_PKGSPEC and the task name, those should be included in the | ||
806 | # output hash calculation. | ||
807 | update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) | ||
808 | update_hash("task=%s\n" % task) | ||
809 | |||
810 | for root, dirs, files in os.walk('.', topdown=True): | ||
811 | # Sort directories and files to ensure consistent ordering | ||
812 | dirs.sort() | ||
813 | files.sort() | ||
814 | |||
815 | for f in files: | ||
816 | path = os.path.join(root, f) | ||
817 | s = os.lstat(path) | ||
818 | |||
819 | # Hash file path | ||
820 | update_hash(path + '\n') | ||
821 | |||
822 | # Hash file mode | ||
823 | update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode)) | ||
824 | update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode)) | ||
825 | |||
826 | if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode): | ||
827 | # Hash device major and minor | ||
828 | update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev))) | ||
829 | elif stat.S_ISLNK(s.st_mode): | ||
830 | # Hash symbolic link | ||
831 | update_hash("\tsymlink=%s\n" % os.readlink(path)) | ||
832 | else: | ||
833 | fh = hashlib.sha256() | ||
834 | # Hash file contents | ||
835 | with open(path, 'rb') as d: | ||
836 | for chunk in iter(lambda: d.read(4096), b""): | ||
837 | fh.update(chunk) | ||
838 | update_hash("\tdigest=%s\n" % fh.hexdigest()) | ||
839 | finally: | ||
840 | os.chdir(prev_dir) | ||
841 | |||
842 | return h.hexdigest() | ||
843 | |||
844 | python sstate_report_unihash() { | 785 | python sstate_report_unihash() { |
845 | report_unihash = getattr(bb.parse.siggen, 'report_unihash', None) | 786 | report_unihash = getattr(bb.parse.siggen, 'report_unihash', None) |
846 | 787 | ||
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py index e0eb87e29f..a83af519ec 100644 --- a/meta/lib/oe/sstatesig.py +++ b/meta/lib/oe/sstatesig.py | |||
@@ -270,7 +270,7 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | |||
270 | super().init_rundepcheck(data) | 270 | super().init_rundepcheck(data) |
271 | self.server = data.getVar('SSTATE_HASHEQUIV_SERVER') | 271 | self.server = data.getVar('SSTATE_HASHEQUIV_SERVER') |
272 | self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') | 272 | self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') |
273 | self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method, data) | 273 | self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) |
274 | 274 | ||
275 | def get_taskdata(self): | 275 | def get_taskdata(self): |
276 | return (self.server, self.method) + super().get_taskdata() | 276 | return (self.server, self.method) + super().get_taskdata() |
@@ -355,6 +355,7 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | |||
355 | import json | 355 | import json |
356 | import tempfile | 356 | import tempfile |
357 | import base64 | 357 | import base64 |
358 | import importlib | ||
358 | 359 | ||
359 | taskhash = d.getVar('BB_TASKHASH') | 360 | taskhash = d.getVar('BB_TASKHASH') |
360 | unihash = d.getVar('BB_UNIHASH') | 361 | unihash = d.getVar('BB_UNIHASH') |
@@ -376,11 +377,14 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | |||
376 | sigfile_link = "depsig.do_%s" % task | 377 | sigfile_link = "depsig.do_%s" % task |
377 | 378 | ||
378 | try: | 379 | try: |
379 | call = self.method + '(path, sigfile, task, d)' | ||
380 | sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') | 380 | sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') |
381 | |||
381 | locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} | 382 | locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} |
382 | 383 | ||
383 | outhash = bb.utils.better_eval(call, locs) | 384 | (module, method) = self.method.rsplit('.', 1) |
385 | locs['method'] = getattr(importlib.import_module(module), method) | ||
386 | |||
387 | outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) | ||
384 | 388 | ||
385 | try: | 389 | try: |
386 | url = '%s/v1/equivalent' % self.server | 390 | url = '%s/v1/equivalent' % self.server |
@@ -581,4 +585,133 @@ def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache): | |||
581 | bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant)) | 585 | bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant)) |
582 | return None, d2 | 586 | return None, d2 |
583 | 587 | ||
588 | def OEOuthashBasic(path, sigfile, task, d): | ||
589 | """ | ||
590 | Basic output hash function | ||
591 | |||
592 | Calculates the output hash of a task by hashing all output file metadata, | ||
593 | and file contents. | ||
594 | """ | ||
595 | import hashlib | ||
596 | import stat | ||
597 | import pwd | ||
598 | import grp | ||
599 | |||
600 | def update_hash(s): | ||
601 | s = s.encode('utf-8') | ||
602 | h.update(s) | ||
603 | if sigfile: | ||
604 | sigfile.write(s) | ||
605 | |||
606 | h = hashlib.sha256() | ||
607 | prev_dir = os.getcwd() | ||
608 | include_owners = os.environ.get('PSEUDO_DISABLED') == '0' | ||
609 | |||
610 | try: | ||
611 | os.chdir(path) | ||
612 | |||
613 | update_hash("OEOuthashBasic\n") | ||
614 | |||
615 | # It is only currently useful to get equivalent hashes for things that | ||
616 | # can be restored from sstate. Since the sstate object is named using | ||
617 | # SSTATE_PKGSPEC and the task name, those should be included in the | ||
618 | # output hash calculation. | ||
619 | update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) | ||
620 | update_hash("task=%s\n" % task) | ||
621 | |||
622 | for root, dirs, files in os.walk('.', topdown=True): | ||
623 | # Sort directories to ensure consistent ordering when recursing | ||
624 | dirs.sort() | ||
625 | files.sort() | ||
626 | |||
627 | def process(path): | ||
628 | s = os.lstat(path) | ||
629 | |||
630 | if stat.S_ISDIR(s.st_mode): | ||
631 | update_hash('d') | ||
632 | elif stat.S_ISCHR(s.st_mode): | ||
633 | update_hash('c') | ||
634 | elif stat.S_ISBLK(s.st_mode): | ||
635 | update_hash('b') | ||
636 | elif stat.S_ISSOCK(s.st_mode): | ||
637 | update_hash('s') | ||
638 | elif stat.S_ISLNK(s.st_mode): | ||
639 | update_hash('l') | ||
640 | elif stat.S_ISFIFO(s.st_mode): | ||
641 | update_hash('p') | ||
642 | else: | ||
643 | update_hash('-') | ||
644 | |||
645 | def add_perm(mask, on, off='-'): | ||
646 | if mask & s.st_mode: | ||
647 | update_hash(on) | ||
648 | else: | ||
649 | update_hash(off) | ||
650 | |||
651 | add_perm(stat.S_IRUSR, 'r') | ||
652 | add_perm(stat.S_IWUSR, 'w') | ||
653 | if stat.S_ISUID & s.st_mode: | ||
654 | add_perm(stat.S_IXUSR, 's', 'S') | ||
655 | else: | ||
656 | add_perm(stat.S_IXUSR, 'x') | ||
657 | |||
658 | add_perm(stat.S_IRGRP, 'r') | ||
659 | add_perm(stat.S_IWGRP, 'w') | ||
660 | if stat.S_ISGID & s.st_mode: | ||
661 | add_perm(stat.S_IXGRP, 's', 'S') | ||
662 | else: | ||
663 | add_perm(stat.S_IXGRP, 'x') | ||
664 | |||
665 | add_perm(stat.S_IROTH, 'r') | ||
666 | add_perm(stat.S_IWOTH, 'w') | ||
667 | if stat.S_ISVTX & s.st_mode: | ||
668 | update_hash('t') | ||
669 | else: | ||
670 | add_perm(stat.S_IXOTH, 'x') | ||
671 | |||
672 | if include_owners: | ||
673 | update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) | ||
674 | update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) | ||
675 | |||
676 | update_hash(" ") | ||
677 | if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode): | ||
678 | update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev)))) | ||
679 | else: | ||
680 | update_hash(" " * 9) | ||
681 | |||
682 | update_hash(" ") | ||
683 | if stat.S_ISREG(s.st_mode): | ||
684 | update_hash("%10d" % s.st_size) | ||
685 | else: | ||
686 | update_hash(" " * 10) | ||
687 | |||
688 | update_hash(" ") | ||
689 | fh = hashlib.sha256() | ||
690 | if stat.S_ISREG(s.st_mode): | ||
691 | # Hash file contents | ||
692 | with open(path, 'rb') as d: | ||
693 | for chunk in iter(lambda: d.read(4096), b""): | ||
694 | fh.update(chunk) | ||
695 | update_hash(fh.hexdigest()) | ||
696 | else: | ||
697 | update_hash(" " * len(fh.hexdigest())) | ||
698 | |||
699 | update_hash(" %s" % path) | ||
700 | |||
701 | if stat.S_ISLNK(s.st_mode): | ||
702 | update_hash(" -> %s" % os.readlink(path)) | ||
703 | |||
704 | update_hash("\n") | ||
705 | |||
706 | # Process this directory and all its child files | ||
707 | process(root) | ||
708 | for f in files: | ||
709 | if f == 'fixmepath': | ||
710 | continue | ||
711 | process(os.path.join(root, f)) | ||
712 | finally: | ||
713 | os.chdir(prev_dir) | ||
714 | |||
715 | return h.hexdigest() | ||
716 | |||
584 | 717 | ||