From adc37721a86ce44c0223b7b03aabd7deceefe57d Mon Sep 17 00:00:00 2001 From: Joshua Watt Date: Fri, 4 Jan 2019 10:20:15 -0600 Subject: sstate: Implement hash equivalence sstate Converts sstate so that it can use a hash equivalence server to determine if a task really needs to be rebuilt, or if it can be restored from a different (equivalent) sstate object. The unique hashes are cached persistently using persist_data. This has a number of advantages: 1) Unique hashes can be cached between invocations of bitbake to prevent needing to contact the server every time (which is slow) 2) The value of each tasks unique hash can easily be synchronized between different threads, which will be useful if bitbake is updated to do on the fly task re-hashing. [YOCTO #13030] (From OE-Core rev: d889acb4f8f06f09cece80fa12661725e6e5f037) Signed-off-by: Joshua Watt Signed-off-by: Richard Purdie --- meta/classes/sstate.bbclass | 105 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 97 insertions(+), 8 deletions(-) (limited to 'meta/classes/sstate.bbclass') diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass index 59ebc3ab5c..da0807d6e9 100644 --- a/meta/classes/sstate.bbclass +++ b/meta/classes/sstate.bbclass @@ -11,7 +11,7 @@ def generate_sstatefn(spec, hash, d): SSTATE_PKGARCH = "${PACKAGE_ARCH}" SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" SSTATE_SWSPEC = "sstate:${PN}::${PV}:${PR}::${SSTATE_VERSION}:" -SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_TASKHASH'), d)}" +SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC'), d.getVar('BB_UNIHASH'), d)}" SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}" SSTATE_EXTRAPATH = "" SSTATE_EXTRAPATHWILDCARD = "" @@ -82,6 +82,23 @@ SSTATE_SIG_PASSPHRASE ?= "" # Whether to verify the GnUPG signatures when extracting sstate archives SSTATE_VERIFY_SIG ?= "0" +SSTATE_HASHEQUIV_METHOD ?= "OEOuthashBasic" +SSTATE_HASHEQUIV_METHOD[doc] = "The function used to calculate the output hash \ + for a task, which in turn is used to determine equivalency. \ + " + +SSTATE_HASHEQUIV_SERVER ?= "" +SSTATE_HASHEQUIV_SERVER[doc] = "The hash equivalence sever. For example, \ + 'http://192.168.0.1:5000'. Do not include a trailing slash \ + " + +SSTATE_HASHEQUIV_REPORT_TASKDATA ?= "0" +SSTATE_HASHEQUIV_REPORT_TASKDATA[doc] = "Report additional useful data to the \ + hash equivalency server, such as PN, PV, taskname, etc. This information \ + is very useful for developers looking at task data, but may leak sensitive \ + data if the equivalence server is public. \ + " + python () { if bb.data.inherits_class('native', d): d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH', False)) @@ -640,7 +657,7 @@ def sstate_package(ss, d): return for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \ - ['sstate_create_package', 'sstate_sign_package'] + \ + ['sstate_report_unihash', 'sstate_create_package', 'sstate_sign_package'] + \ (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split(): # All hooks should run in SSTATE_BUILDDIR. bb.build.exec_func(f, d, (sstatebuild,)) @@ -764,6 +781,73 @@ python sstate_sign_package () { d.getVar('SSTATE_SIG_PASSPHRASE'), armor=False) } +def OEOuthashBasic(path, sigfile, task, d): + import hashlib + import stat + + def update_hash(s): + s = s.encode('utf-8') + h.update(s) + if sigfile: + sigfile.write(s) + + h = hashlib.sha256() + prev_dir = os.getcwd() + + try: + os.chdir(path) + + update_hash("OEOuthashBasic\n") + + # It is only currently useful to get equivalent hashes for things that + # can be restored from sstate. Since the sstate object is named using + # SSTATE_PKGSPEC and the task name, those should be included in the + # output hash calculation. + update_hash("SSTATE_PKGSPEC=%s\n" % d.getVar('SSTATE_PKGSPEC')) + update_hash("task=%s\n" % task) + + for root, dirs, files in os.walk('.', topdown=True): + # Sort directories and files to ensure consistent ordering + dirs.sort() + files.sort() + + for f in files: + path = os.path.join(root, f) + s = os.lstat(path) + + # Hash file path + update_hash(path + '\n') + + # Hash file mode + update_hash("\tmode=0x%x\n" % stat.S_IMODE(s.st_mode)) + update_hash("\ttype=0x%x\n" % stat.S_IFMT(s.st_mode)) + + if stat.S_ISBLK(s.st_mode) or stat.S_ISBLK(s.st_mode): + # Hash device major and minor + update_hash("\tdev=%d,%d\n" % (os.major(s.st_rdev), os.minor(s.st_rdev))) + elif stat.S_ISLNK(s.st_mode): + # Hash symbolic link + update_hash("\tsymlink=%s\n" % os.readlink(path)) + else: + fh = hashlib.sha256() + # Hash file contents + with open(path, 'rb') as d: + for chunk in iter(lambda: d.read(4096), b""): + fh.update(chunk) + update_hash("\tdigest=%s\n" % fh.hexdigest()) + finally: + os.chdir(prev_dir) + + return h.hexdigest() + +python sstate_report_unihash() { + report_unihash = getattr(bb.parse.siggen, 'report_unihash', None) + + if report_unihash: + ss = sstate_state_fromvars(d) + report_unihash(os.getcwd(), ss['task'], d) +} + # # Shell function to decompress and prepare a package for installation # Will be run from within SSTATE_INSTDIR. @@ -788,6 +872,11 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, if siginfo: extension = extension + ".siginfo" + def gethash(task): + if sq_unihash is not None: + return sq_unihash[task] + return sq_hash[task] + def getpathcomponents(task, d): # Magic data from BB_HASHFILENAME splithashfn = sq_hashfn[task].split(" ") @@ -810,7 +899,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, spec, extrapath, tname = getpathcomponents(task, d) - sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + extension) + sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension) if os.path.exists(sstatefile): bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) @@ -872,7 +961,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, if task in ret: continue spec, extrapath, tname = getpathcomponents(task, d) - sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + extension) + sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension) tasklist.append((task, sstatefile)) if tasklist: @@ -898,12 +987,12 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, evdata = {'missed': [], 'found': []}; for task in missed: spec, extrapath, tname = getpathcomponents(task, d) - sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz") - evdata['missed'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) ) + sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz") + evdata['missed'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) ) for task in ret: spec, extrapath, tname = getpathcomponents(task, d) - sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz") - evdata['found'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) ) + sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz") + evdata['found'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) ) bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d) # Print some summary statistics about the current task completion and how much sstate -- cgit v1.2.3-54-g00ecf