From c3c14808dc4ff62196ef22bfb4984369e4740647 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Wed, 2 Nov 2016 15:06:50 +0000 Subject: bitbake: siggen: Pass basehash to worker processes and sanity check reparsing result Bitbake can parse metadata in the cooker and in the worker during builds. If the metadata isn't deterministic, it can change between these two parses and this confuses things a lot. It turns out to be hard to debug these issues currently. This patch ensures the basehashes from the original parsing are passed into the workers and that these are checked when reparsing for consistency. The user is shown an error message if inconsistencies are found. There is debug code in siggen.py (see the "Slow but can be useful for debugging mismatched basehashes" commented code), we don't enable this by default due to performance issues. If you run into this message, enable this code and you will find "sigbasedata" files in tmp/stamps which should correspond to the hashes shown in this error message. bitbake-diffsigs on the files should show which variables are changing. (Bitbake rev: 46207262ee6cdd2e49c4765481a6a24702ca4843) (Bitbake rev: aa873f982ae4a56b135abd9eee169794e4c3aadd) Signed-off-by: Richard Purdie Fixed up do to python3 changes not being in krogoth. Signed-off-by: Armin Kuster Signed-off-by: Richard Purdie --- bitbake/lib/bb/siggen.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 88fc0f1d5c..7d72f0c3ba 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -35,6 +35,7 @@ class SignatureGenerator(object): name = "noop" def __init__(self, data): + self.basehash = {} self.taskhash = {} self.runtaskdeps = {} self.file_checksum_values = {} @@ -66,11 +67,10 @@ class SignatureGenerator(object): return def get_taskdata(self): - return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints) + return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash) def set_taskdata(self, data): - self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints = data - + self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data class SignatureGeneratorBasic(SignatureGenerator): """ @@ -138,7 +138,11 @@ class SignatureGeneratorBasic(SignatureGenerator): var = lookupcache[dep] if var is not None: data = data + str(var) - self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest() + datahash = hashlib.md5(data).hexdigest() + k = fn + "." + task + if k in self.basehash and self.basehash[k] != datahash: + bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash)) + self.basehash[k] = datahash taskdeps[task] = alldeps self.taskdeps[fn] = taskdeps @@ -186,6 +190,7 @@ class SignatureGeneratorBasic(SignatureGenerator): def get_taskhash(self, fn, task, deps, dataCache): k = fn + "." + task data = dataCache.basetaskhash[k] + self.basehash[k] = data self.runtaskdeps[k] = [] self.file_checksum_values[k] = [] recipename = dataCache.pkg_fn[fn] -- cgit v1.2.3-54-g00ecf