summaryrefslogtreecommitdiffstats
path: root/meta/classes
diff options
context:
space:
mode:
authorJoshua Watt <jpewhacker@gmail.com>2019-01-04 10:20:15 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-08 11:16:44 +0000
commitadc37721a86ce44c0223b7b03aabd7deceefe57d (patch)
treef917b9bdf3f9d5fa3c53bfce68cc947a44ebc1fa /meta/classes
parentcbdfa376633d4cf2d86a0f6953d5b0e3a076e06d (diff)
downloadpoky-adc37721a86ce44c0223b7b03aabd7deceefe57d.tar.gz
sstate: Implement hash equivalence sstate
Converts sstate so that it can use a hash equivalence server to determine if a task really needs to be rebuilt, or if it can be restored from a different (equivalent) sstate object. The unique hashes are cached persistently using persist_data. This has a number of advantages: 1) Unique hashes can be cached between invocations of bitbake to prevent needing to contact the server every time (which is slow) 2) The value of each tasks unique hash can easily be synchronized between different threads, which will be useful if bitbake is updated to do on the fly task re-hashing. [YOCTO #13030] (From OE-Core rev: d889acb4f8f06f09cece80fa12661725e6e5f037) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/classes')
-rw-r--r--meta/classes/sstate.bbclass105
1 files changed, 97 insertions, 8 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 59ebc3ab5c..da0807d6e9 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -11,7 +11,7 @@ def generate_sstatefn(spec, hash, d):
11SSTATE_PKGARCH = "${PACKAGE_ARCH}" 11SSTATE_PKGARCH = "${PACKAGE_ARCH}"
12SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" 12SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
13SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:" 13SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:"
14SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_TASKHASH'), d)}" 14SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d)}"
15SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}" 15SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}"
16SSTATE_EXTRAPATH = "" 16SSTATE_EXTRAPATH = ""
17SSTATE_EXTRAPATHWILDCARD = "" 17SSTATE_EXTRAPATHWILDCARD = ""
@@ -82,6 +82,23 @@ SSTATE_SIG_PASSPHRASE ?= ""
82# Whether to verify the GnUPG signatures when extracting sstate archives 82# Whether to verify the GnUPG signatures when extracting sstate archives
83SSTATE_VERIFY_SIG ?= "0" 83SSTATE_VERIFY_SIG ?= "0"
84 84
85SSTATE_HASHEQUIV_METHOD ?= "OEOuthashBasic"
86SSTATE_HASHEQUIV_METHOD[doc] = "The function used to calculate the output hash \
87 for a task, which in turn is used to determine equivalency. \
88 "
89
90SSTATE_HASHEQUIV_SERVER ?= ""
91SSTATE_HASHEQUIV_SERVER[doc] = "The hash equivalence sever. For example, \
92 'http://192.168.0.1:5000'. Do not include a trailing slash \
93 "
94
95SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0"
96SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \
97 hash equivalency server, such as PN, PV, taskname, etc. This information \
98 is very useful for developers looking at task data, but may leak sensitive \
99 data if the equivalence server is public. \
100 "
101
85python () { 102python () {
86 if bb.data.inherits_class('native', d): 103 if bb.data.inherits_class('native', d):
87 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False)) 104 d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False))
@@ -640,7 +657,7 @@ def sstate_package(ss, d):
640 return 657 return
641 658
642 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \ 659 for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
643 ['sstate_create_package', 'sstate_sign_package'] + \ 660 ['sstate_report_unihash', 'sstate_create_package', 'sstate_sign_package'] + \
644 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split(): 661 (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
645 # All hooks should run in SSTATE_BUILDDIR. 662 # All hooks should run in SSTATE_BUILDDIR.
646 bb.build.exec_func(f, d, (sstatebuild,)) 663 bb.build.exec_func(f, d, (sstatebuild,))
@@ -764,6 +781,73 @@ python sstate_sign_package () {
764 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) 781 d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False)
765} 782}
766 783
784def OEOuthashBasic(path, sigfile, task, d):
785 import hashlib
786 import stat
787
788 def update_hash(s):
789 s = s.encode('utf-8')
790 h.update(s)
791 if sigfile:
792 sigfile.write(s)
793
794 h = hashlib.sha256()
795 prev_dir = os.getcwd()
796
797 try:
798 os.chdir(path)
799
800 update_hash("OEOuthashBasic\n")
801
802 # It is only currently useful to get equivalent hashes for things that
803 # can be restored from sstate. Since the sstate object is named using
804 # SSTATE_PKGSPEC and the task name, those should be included in the
805 # output hash calculation.
806 update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC'))
807 update_hash("task=%s\n" % task)
808
809 for root, dirs, files in os.walk('.', topdown=True):
810 # Sort directories and files to ensure consistent ordering
811 dirs.sort()
812 files.sort()
813
814 for f in files:
815 path = os.path.join(root, f)
816 s = os.lstat(path)
817
818 # Hash file path
819 update_hash(path + '\n')
820
821 # Hash file mode
822 update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode))
823 update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode))
824
825 if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode):
826 # Hash device major and minor
827 update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev)))
828 elif stat.S_ISLNK(s.st_mode):
829 # Hash symbolic link
830 update_hash("\tsymlink=%s\n" % os.readlink(path))
831 else:
832 fh = hashlib.sha256()
833 # Hash file contents
834 with open(path, 'rb') as d:
835 for chunk in iter(lambda: d.read(4096), b""):
836 fh.update(chunk)
837 update_hash("\tdigest=%s\n" % fh.hexdigest())
838 finally:
839 os.chdir(prev_dir)
840
841 return h.hexdigest()
842
843python sstate_report_unihash() {
844 report_unihash = getattr(bb.parse.siggen, 'report_unihash', None)
845
846 if report_unihash:
847 ss = sstate_state_fromvars(d)
848 report_unihash(os.getcwd(), ss['task'], d)
849}
850
767# 851#
768# Shell function to decompress and prepare a package for installation 852# Shell function to decompress and prepare a package for installation
769# Will be run from within SSTATE_INSTDIR. 853# Will be run from within SSTATE_INSTDIR.
@@ -788,6 +872,11 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
788 if siginfo: 872 if siginfo:
789 extension = extension + ".siginfo" 873 extension = extension + ".siginfo"
790 874
875 def gethash(task):
876 if sq_unihash is not None:
877 return sq_unihash[task]
878 return sq_hash[task]
879
791 def getpathcomponents(task, d): 880 def getpathcomponents(task, d):
792 # Magic data from BB_HASHFILENAME 881 # Magic data from BB_HASHFILENAME
793 splithashfn = sq_hashfn[task].split(" ") 882 splithashfn = sq_hashfn[task].split(" ")
@@ -810,7 +899,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
810 899
811 spec, extrapath, tname = getpathcomponents(task, d) 900 spec, extrapath, tname = getpathcomponents(task, d)
812 901
813 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + extension) 902 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension)
814 903
815 if os.path.exists(sstatefile): 904 if os.path.exists(sstatefile):
816 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) 905 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
@@ -872,7 +961,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
872 if task in ret: 961 if task in ret:
873 continue 962 continue
874 spec, extrapath, tname = getpathcomponents(task, d) 963 spec, extrapath, tname = getpathcomponents(task, d)
875 sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + extension) 964 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension)
876 tasklist.append((task, sstatefile)) 965 tasklist.append((task, sstatefile))
877 966
878 if tasklist: 967 if tasklist:
@@ -898,12 +987,12 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
898 evdata = {'missed': [], 'found': []}; 987 evdata = {'missed': [], 'found': []};
899 for task in missed: 988 for task in missed:
900 spec, extrapath, tname = getpathcomponents(task, d) 989 spec, extrapath, tname = getpathcomponents(task, d)
901 sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz") 990 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz")
902 evdata['missed'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) ) 991 evdata['missed'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) )
903 for task in ret: 992 for task in ret:
904 spec, extrapath, tname = getpathcomponents(task, d) 993 spec, extrapath, tname = getpathcomponents(task, d)
905 sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz") 994 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz")
906 evdata['found'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) ) 995 evdata['found'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) )
907 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d) 996 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
908 997
909 # Print some summary statistics about the current task completion and how much sstate 998 # Print some summary statistics about the current task completion and how much sstate