diff options
| -rw-r--r-- | meta/classes/sstate.bbclass | 65 | ||||
| -rw-r--r-- | meta/lib/oe/sstatesig.py | 139 |
2 files changed, 139 insertions, 65 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass index 763fce07f9..2f0bbd2d7d 100644 --- a/meta/classes/sstate.bbclass +++ b/meta/classes/sstate.bbclass | |||
| @@ -83,9 +83,9 @@ SSTATE_SIG_PASSPHRASE ?= "" | |||
| 83 | # Whether to verify the GnUPG signatures when extracting sstate archives | 83 | # Whether to verify the GnUPG signatures when extracting sstate archives |
| 84 | SSTATE_VERIFY_SIG ?= "0" | 84 | SSTATE_VERIFY_SIG ?= "0" |
| 85 | 85 | ||
| 86 | SSTATE_HASHEQUIV_METHOD ?= "OEOuthashBasic" | 86 | SSTATE_HASHEQUIV_METHOD ?= "oe.sstatesig.OEOuthashBasic" |
| 87 | SSTATE_HASHEQUIV_METHOD[doc] = "The function used to calculate the output hash \ | 87 | SSTATE_HASHEQUIV_METHOD[doc] = "The fully-qualified function used to calculate \ |
| 88 | for a task, which in turn is used to determine equivalency. \ | 88 | the output hash for a task, which in turn is used to determine equivalency. \ |
| 89 | " | 89 | " |
| 90 | 90 | ||
| 91 | SSTATE_HASHEQUIV_SERVER ?= "" | 91 | SSTATE_HASHEQUIV_SERVER ?= "" |
| @@ -782,65 +782,6 @@ python sstate_sign_package () { | |||
| 782 | d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) | 782 | d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) |
| 783 | } | 783 | } |
| 784 | 784 | ||
| 785 | def OEOuthashBasic(path, sigfile, task, d): | ||
| 786 | import hashlib | ||
| 787 | import stat | ||
| 788 | |||
| 789 | def update_hash(s): | ||
| 790 | s = s.encode('utf-8') | ||
| 791 | h.update(s) | ||
| 792 | if sigfile: | ||
| 793 | sigfile.write(s) | ||
| 794 | |||
| 795 | h = hashlib.sha256() | ||
| 796 | prev_dir = os.getcwd() | ||
| 797 | |||
| 798 | try: | ||
| 799 | os.chdir(path) | ||
| 800 | |||
| 801 | update_hash("OEOuthashBasic\n") | ||
| 802 | |||
| 803 | # It is only currently useful to get equivalent hashes for things that | ||
| 804 | # can be restored from sstate. Since the sstate object is named using | ||
| 805 | # SSTATE_PKGSPEC and the task name, those should be included in the | ||
| 806 | # output hash calculation. | ||
| 807 | update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) | ||
| 808 | update_hash("task=%s\n" % task) | ||
| 809 | |||
| 810 | for root, dirs, files in os.walk('.', topdown=True): | ||
| 811 | # Sort directories and files to ensure consistent ordering | ||
| 812 | dirs.sort() | ||
| 813 | files.sort() | ||
| 814 | |||
| 815 | for f in files: | ||
| 816 | path = os.path.join(root, f) | ||
| 817 | s = os.lstat(path) | ||
| 818 | |||
| 819 | # Hash file path | ||
| 820 | update_hash(path + '\n') | ||
| 821 | |||
| 822 | # Hash file mode | ||
| 823 | update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode)) | ||
| 824 | update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode)) | ||
| 825 | |||
| 826 | if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode): | ||
| 827 | # Hash device major and minor | ||
| 828 | update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev))) | ||
| 829 | elif stat.S_ISLNK(s.st_mode): | ||
| 830 | # Hash symbolic link | ||
| 831 | update_hash("\tsymlink=%s\n" % os.readlink(path)) | ||
| 832 | else: | ||
| 833 | fh = hashlib.sha256() | ||
| 834 | # Hash file contents | ||
| 835 | with open(path, 'rb') as d: | ||
| 836 | for chunk in iter(lambda: d.read(4096), b""): | ||
| 837 | fh.update(chunk) | ||
| 838 | update_hash("\tdigest=%s\n" % fh.hexdigest()) | ||
| 839 | finally: | ||
| 840 | os.chdir(prev_dir) | ||
| 841 | |||
| 842 | return h.hexdigest() | ||
| 843 | |||
| 844 | python sstate_report_unihash() { | 785 | python sstate_report_unihash() { |
| 845 | report_unihash = getattr(bb.parse.siggen, 'report_unihash', None) | 786 | report_unihash = getattr(bb.parse.siggen, 'report_unihash', None) |
| 846 | 787 | ||
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py index e0eb87e29f..a83af519ec 100644 --- a/meta/lib/oe/sstatesig.py +++ b/meta/lib/oe/sstatesig.py | |||
| @@ -270,7 +270,7 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | |||
| 270 | super().init_rundepcheck(data) | 270 | super().init_rundepcheck(data) |
| 271 | self.server = data.getVar('SSTATE_HASHEQUIV_SERVER') | 271 | self.server = data.getVar('SSTATE_HASHEQUIV_SERVER') |
| 272 | self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') | 272 | self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') |
| 273 | self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method, data) | 273 | self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) |
| 274 | 274 | ||
| 275 | def get_taskdata(self): | 275 | def get_taskdata(self): |
| 276 | return (self.server, self.method) + super().get_taskdata() | 276 | return (self.server, self.method) + super().get_taskdata() |
| @@ -355,6 +355,7 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | |||
| 355 | import json | 355 | import json |
| 356 | import tempfile | 356 | import tempfile |
| 357 | import base64 | 357 | import base64 |
| 358 | import importlib | ||
| 358 | 359 | ||
| 359 | taskhash = d.getVar('BB_TASKHASH') | 360 | taskhash = d.getVar('BB_TASKHASH') |
| 360 | unihash = d.getVar('BB_UNIHASH') | 361 | unihash = d.getVar('BB_UNIHASH') |
| @@ -376,11 +377,14 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | |||
| 376 | sigfile_link = "depsig.do_%s" % task | 377 | sigfile_link = "depsig.do_%s" % task |
| 377 | 378 | ||
| 378 | try: | 379 | try: |
| 379 | call = self.method + '(path, sigfile, task, d)' | ||
| 380 | sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') | 380 | sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') |
| 381 | |||
| 381 | locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} | 382 | locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} |
| 382 | 383 | ||
| 383 | outhash = bb.utils.better_eval(call, locs) | 384 | (module, method) = self.method.rsplit('.', 1) |
| 385 | locs['method'] = getattr(importlib.import_module(module), method) | ||
| 386 | |||
| 387 | outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) | ||
| 384 | 388 | ||
| 385 | try: | 389 | try: |
| 386 | url = '%s/v1/equivalent' % self.server | 390 | url = '%s/v1/equivalent' % self.server |
| @@ -581,4 +585,133 @@ def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache): | |||
| 581 | bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant)) | 585 | bb.warn("Manifest %s not found in %s (variant '%s')?" % (manifest, d2.expand(" ".join(pkgarchs)), variant)) |
| 582 | return None, d2 | 586 | return None, d2 |
| 583 | 587 | ||
| 588 | def OEOuthashBasic(path, sigfile, task, d): | ||
| 589 | """ | ||
| 590 | Basic output hash function | ||
| 591 | |||
| 592 | Calculates the output hash of a task by hashing all output file metadata, | ||
| 593 | and file contents. | ||
| 594 | """ | ||
| 595 | import hashlib | ||
| 596 | import stat | ||
| 597 | import pwd | ||
| 598 | import grp | ||
| 599 | |||
| 600 | def update_hash(s): | ||
| 601 | s = s.encode('utf-8') | ||
| 602 | h.update(s) | ||
| 603 | if sigfile: | ||
| 604 | sigfile.write(s) | ||
| 605 | |||
| 606 | h = hashlib.sha256() | ||
| 607 | prev_dir = os.getcwd() | ||
| 608 | include_owners = os.environ.get('PSEUDO_DISABLED') == '0' | ||
| 609 | |||
| 610 | try: | ||
| 611 | os.chdir(path) | ||
| 612 | |||
| 613 | update_hash("OEOuthashBasic\n") | ||
| 614 | |||
| 615 | # It is only currently useful to get equivalent hashes for things that | ||
| 616 | # can be restored from sstate. Since the sstate object is named using | ||
| 617 | # SSTATE_PKGSPEC and the task name, those should be included in the | ||
| 618 | # output hash calculation. | ||
| 619 | update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) | ||
| 620 | update_hash("task=%s\n" % task) | ||
| 621 | |||
| 622 | for root, dirs, files in os.walk('.', topdown=True): | ||
| 623 | # Sort directories to ensure consistent ordering when recursing | ||
| 624 | dirs.sort() | ||
| 625 | files.sort() | ||
| 626 | |||
| 627 | def process(path): | ||
| 628 | s = os.lstat(path) | ||
| 629 | |||
| 630 | if stat.S_ISDIR(s.st_mode): | ||
| 631 | update_hash('d') | ||
| 632 | elif stat.S_ISCHR(s.st_mode): | ||
| 633 | update_hash('c') | ||
| 634 | elif stat.S_ISBLK(s.st_mode): | ||
| 635 | update_hash('b') | ||
| 636 | elif stat.S_ISSOCK(s.st_mode): | ||
| 637 | update_hash('s') | ||
| 638 | elif stat.S_ISLNK(s.st_mode): | ||
| 639 | update_hash('l') | ||
| 640 | elif stat.S_ISFIFO(s.st_mode): | ||
| 641 | update_hash('p') | ||
| 642 | else: | ||
| 643 | update_hash('-') | ||
| 644 | |||
| 645 | def add_perm(mask, on, off='-'): | ||
| 646 | if mask & s.st_mode: | ||
| 647 | update_hash(on) | ||
| 648 | else: | ||
| 649 | update_hash(off) | ||
| 650 | |||
| 651 | add_perm(stat.S_IRUSR, 'r') | ||
| 652 | add_perm(stat.S_IWUSR, 'w') | ||
| 653 | if stat.S_ISUID & s.st_mode: | ||
| 654 | add_perm(stat.S_IXUSR, 's', 'S') | ||
| 655 | else: | ||
| 656 | add_perm(stat.S_IXUSR, 'x') | ||
| 657 | |||
| 658 | add_perm(stat.S_IRGRP, 'r') | ||
| 659 | add_perm(stat.S_IWGRP, 'w') | ||
| 660 | if stat.S_ISGID & s.st_mode: | ||
| 661 | add_perm(stat.S_IXGRP, 's', 'S') | ||
| 662 | else: | ||
| 663 | add_perm(stat.S_IXGRP, 'x') | ||
| 664 | |||
| 665 | add_perm(stat.S_IROTH, 'r') | ||
| 666 | add_perm(stat.S_IWOTH, 'w') | ||
| 667 | if stat.S_ISVTX & s.st_mode: | ||
| 668 | update_hash('t') | ||
| 669 | else: | ||
| 670 | add_perm(stat.S_IXOTH, 'x') | ||
| 671 | |||
| 672 | if include_owners: | ||
| 673 | update_hash(" %10s" % pwd.getpwuid(s.st_uid).pw_name) | ||
| 674 | update_hash(" %10s" % grp.getgrgid(s.st_gid).gr_name) | ||
| 675 | |||
| 676 | update_hash(" ") | ||
| 677 | if stat.S_ISBLK(s.st_mode) or stat.S_ISCHR(s.st_mode): | ||
| 678 | update_hash("%9s" % ("%d.%d" % (os.major(s.st_rdev), os.minor(s.st_rdev)))) | ||
| 679 | else: | ||
| 680 | update_hash(" " * 9) | ||
| 681 | |||
| 682 | update_hash(" ") | ||
| 683 | if stat.S_ISREG(s.st_mode): | ||
| 684 | update_hash("%10d" % s.st_size) | ||
| 685 | else: | ||
| 686 | update_hash(" " * 10) | ||
| 687 | |||
| 688 | update_hash(" ") | ||
| 689 | fh = hashlib.sha256() | ||
| 690 | if stat.S_ISREG(s.st_mode): | ||
| 691 | # Hash file contents | ||
| 692 | with open(path, 'rb') as d: | ||
| 693 | for chunk in iter(lambda: d.read(4096), b""): | ||
| 694 | fh.update(chunk) | ||
| 695 | update_hash(fh.hexdigest()) | ||
| 696 | else: | ||
| 697 | update_hash(" " * len(fh.hexdigest())) | ||
| 698 | |||
| 699 | update_hash(" %s" % path) | ||
| 700 | |||
| 701 | if stat.S_ISLNK(s.st_mode): | ||
| 702 | update_hash(" -> %s" % os.readlink(path)) | ||
| 703 | |||
| 704 | update_hash("\n") | ||
| 705 | |||
| 706 | # Process this directory and all its child files | ||
| 707 | process(root) | ||
| 708 | for f in files: | ||
| 709 | if f == 'fixmepath': | ||
| 710 | continue | ||
| 711 | process(os.path.join(root, f)) | ||
| 712 | finally: | ||
| 713 | os.chdir(prev_dir) | ||
| 714 | |||
| 715 | return h.hexdigest() | ||
| 716 | |||
| 584 | 717 | ||
