summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-12-13 16:07:28 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-12-16 23:27:14 +0000
commit884463c8d1518d6bf1884e9526b4a50384290fe7 (patch)
tree538be6559af8c48b67d406a2c937f523ef139d25 /bitbake
parent73896a7a0e0b7fa1c6ee8a72940aefdba641ea5e (diff)
downloadpoky-884463c8d1518d6bf1884e9526b4a50384290fe7.tar.gz
bitbake: siggen: Split get_tashhash for performance
There are two operations happening in get_taskhash, the building of the underlying data and the calculation of the hash. Split these into two funtions since the preparation part doesn't need to rerun when unihash changes, only the calculation does. This split allows sigificant performance improvements for hashequiv in builds where many hashes are equivalent and many hashes are changing. (Bitbake rev: 6a32af2808d748819f4af55c443578c8a63062b3) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/runqueue.py1
-rw-r--r--bitbake/lib/bb/siggen.py33
2 files changed, 25 insertions, 9 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index b3648ddb54..515e9d4314 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1185,6 +1185,7 @@ class RunQueueData:
1185 procdep = [] 1185 procdep = []
1186 for dep in self.runtaskentries[tid].depends: 1186 for dep in self.runtaskentries[tid].depends:
1187 procdep.append(dep) 1187 procdep.append(dep)
1188 bb.parse.siggen.prep_taskhash(tid, procdep, self.dataCaches[mc_from_tid(tid)])
1188 self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.dataCaches[mc_from_tid(tid)]) 1189 self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, procdep, self.dataCaches[mc_from_tid(tid)])
1189 self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid) 1190 self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
1190 1191
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 2fec8599b3..e484e5e37d 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -52,6 +52,9 @@ class SignatureGenerator(object):
52 def get_unihash(self, tid): 52 def get_unihash(self, tid):
53 return self.taskhash[tid] 53 return self.taskhash[tid]
54 54
55 def prep_taskhash(self, tid, deps, dataCache):
56 return
57
55 def get_taskhash(self, tid, deps, dataCache): 58 def get_taskhash(self, tid, deps, dataCache):
56 self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest() 59 self.taskhash[tid] = hashlib.sha256(tid.encode("utf-8")).hexdigest()
57 return self.taskhash[tid] 60 return self.taskhash[tid]
@@ -198,12 +201,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
198 pass 201 pass
199 return taint 202 return taint
200 203
201 def get_taskhash(self, tid, deps, dataCache): 204 def prep_taskhash(self, tid, deps, dataCache):
202 205
203 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid) 206 (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
204 207
205 data = dataCache.basetaskhash[tid] 208 self.basehash[tid] = dataCache.basetaskhash[tid]
206 self.basehash[tid] = data
207 self.runtaskdeps[tid] = [] 209 self.runtaskdeps[tid] = []
208 self.file_checksum_values[tid] = [] 210 self.file_checksum_values[tid] = []
209 recipename = dataCache.pkg_fn[fn] 211 recipename = dataCache.pkg_fn[fn]
@@ -216,7 +218,6 @@ class SignatureGeneratorBasic(SignatureGenerator):
216 continue 218 continue
217 if dep not in self.taskhash: 219 if dep not in self.taskhash:
218 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep) 220 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
219 data = data + self.get_unihash(dep)
220 self.runtaskdeps[tid].append(dep) 221 self.runtaskdeps[tid].append(dep)
221 222
222 if task in dataCache.file_checksums[fn]: 223 if task in dataCache.file_checksums[fn]:
@@ -226,27 +227,41 @@ class SignatureGeneratorBasic(SignatureGenerator):
226 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) 227 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
227 for (f,cs) in checksums: 228 for (f,cs) in checksums:
228 self.file_checksum_values[tid].append((f,cs)) 229 self.file_checksum_values[tid].append((f,cs))
229 if cs:
230 data = data + cs
231 230
232 taskdep = dataCache.task_deps[fn] 231 taskdep = dataCache.task_deps[fn]
233 if 'nostamp' in taskdep and task in taskdep['nostamp']: 232 if 'nostamp' in taskdep and task in taskdep['nostamp']:
234 # Nostamp tasks need an implicit taint so that they force any dependent tasks to run 233 # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
235 if tid in self.taints and self.taints[tid].startswith("nostamp:"): 234 if tid in self.taints and self.taints[tid].startswith("nostamp:"):
236 # Don't reset taint value upon every call 235 # Don't reset taint value upon every call
237 data = data + self.taints[tid][8:] 236 pass
238 else: 237 else:
239 import uuid 238 import uuid
240 taint = str(uuid.uuid4()) 239 taint = str(uuid.uuid4())
241 data = data + taint
242 self.taints[tid] = "nostamp:" + taint 240 self.taints[tid] = "nostamp:" + taint
243 241
244 taint = self.read_taint(fn, task, dataCache.stamp[fn]) 242 taint = self.read_taint(fn, task, dataCache.stamp[fn])
245 if taint: 243 if taint:
246 data = data + taint
247 self.taints[tid] = taint 244 self.taints[tid] = taint
248 logger.warning("%s is tainted from a forced run" % tid) 245 logger.warning("%s is tainted from a forced run" % tid)
249 246
247 return
248
249 def get_taskhash(self, tid, deps, dataCache):
250
251 data = self.basehash[tid]
252 for dep in self.runtaskdeps[tid]:
253 data = data + self.get_unihash(dep)
254
255 for (f, cs) in self.file_checksum_values[tid]:
256 if cs:
257 data = data + cs
258
259 if tid in self.taints:
260 if self.taints[tid].startswith("nostamp:"):
261 data = data + self.taints[tid][8:]
262 else:
263 data = data + self.taints[tid]
264
250 h = hashlib.sha256(data.encode("utf-8")).hexdigest() 265 h = hashlib.sha256(data.encode("utf-8")).hexdigest()
251 self.taskhash[tid] = h 266 self.taskhash[tid] = h
252 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) 267 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])