summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2016-11-02 15:06:50 +0000
committerSona Sarmadi <sona.sarmadi@enea.com>2017-02-10 12:21:35 +0100
commit79a3cf30ee62bee1a96f2310b7c75a811f252ec1 (patch)
treed2c7289639beb184a6566a9cd7acc4beee488031
parent1188de6e65fb93fd15db5bab68d8a2226c1478c7 (diff)
downloadpoky-79a3cf30ee62bee1a96f2310b7c75a811f252ec1.tar.gz
bitbake: siggen: Pass basehash to worker processes and sanity check reparsing result
Bitbake can parse metadata in the cooker and in the worker during builds. If the metadata isn't deterministic, it can change between these two parses and this confuses things a lot. It turns out to be hard to debug these issues currently. This patch ensures the basehashes from the original parsing are passed into the workers and that these are checked when reparsing for consistency. The user is shown an error message if inconsistencies are found. There is debug code in siggen.py (see the "Slow but can be useful for debugging mismatched basehashes" commented code), we don't enable this by default due to performance issues. If you run into this message, enable this code and you will find "sigbasedata" files in tmp/stamps which should correspond to the hashes shown in this error message. bitbake-diffsigs on the files should show which variables are changing. (Bitbake rev: 46207262ee6cdd2e49c4765481a6a24702ca4843) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> Fixed up do to python3 changes not being in krogoth. Signed-off-by: Armin Kuster <akuster808@gmail.com> Signed-off-by: Sona Sarmadi <sona.sarmadi@enea.com>
-rw-r--r--bitbake/lib/bb/siggen.py13
1 files changed, 9 insertions, 4 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 88fc0f1d5c..7d72f0c3ba 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -35,6 +35,7 @@ class SignatureGenerator(object):
35 name = "noop" 35 name = "noop"
36 36
37 def __init__(self, data): 37 def __init__(self, data):
38 self.basehash = {}
38 self.taskhash = {} 39 self.taskhash = {}
39 self.runtaskdeps = {} 40 self.runtaskdeps = {}
40 self.file_checksum_values = {} 41 self.file_checksum_values = {}
@@ -66,11 +67,10 @@ class SignatureGenerator(object):
66 return 67 return
67 68
68 def get_taskdata(self): 69 def get_taskdata(self):
69 return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints) 70 return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash)
70 71
71 def set_taskdata(self, data): 72 def set_taskdata(self, data):
72 self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints = data 73 self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash = data
73
74 74
75class SignatureGeneratorBasic(SignatureGenerator): 75class SignatureGeneratorBasic(SignatureGenerator):
76 """ 76 """
@@ -138,7 +138,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
138 var = lookupcache[dep] 138 var = lookupcache[dep]
139 if var is not None: 139 if var is not None:
140 data = data + str(var) 140 data = data + str(var)
141 self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest() 141 datahash = hashlib.md5(data).hexdigest()
142 k = fn + "." + task
143 if k in self.basehash and self.basehash[k] != datahash:
144 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (k, self.basehash[k], datahash))
145 self.basehash[k] = datahash
142 taskdeps[task] = alldeps 146 taskdeps[task] = alldeps
143 147
144 self.taskdeps[fn] = taskdeps 148 self.taskdeps[fn] = taskdeps
@@ -186,6 +190,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
186 def get_taskhash(self, fn, task, deps, dataCache): 190 def get_taskhash(self, fn, task, deps, dataCache):
187 k = fn + "." + task 191 k = fn + "." + task
188 data = dataCache.basetaskhash[k] 192 data = dataCache.basetaskhash[k]
193 self.basehash[k] = data
189 self.runtaskdeps[k] = [] 194 self.runtaskdeps[k] = []
190 self.file_checksum_values[k] = [] 195 self.file_checksum_values[k] = []
191 recipename = dataCache.pkg_fn[fn] 196 recipename = dataCache.pkg_fn[fn]