summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-12-14 18:17:01 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-02-08 16:01:33 +0000
commit4d4f1ee5f601b79c3776fcddae7dd63d5ab157d3 (patch)
treebc48785a5d67259f60a752e96677de53cafe0283
parentcedfbac46670a3371bbddaf8736eb8e8555e2ce8 (diff)
downloadpoky-4d4f1ee5f601b79c3776fcddae7dd63d5ab157d3.tar.gz
bitbake: siggen: Cache unihash values to avoid cache lookup
Add unihash cache of values to speed up cache lookup. This avoids the overhead of the disk based check functions. (Bitbake rev: 5c9cc45b60904a1c355db9bf9c4495f1b25aca37) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/siggen.py22
1 files changed, 17 insertions, 5 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 209a342883..96807c46cd 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -40,6 +40,7 @@ class SignatureGenerator(object):
40 def __init__(self, data): 40 def __init__(self, data):
41 self.basehash = {} 41 self.basehash = {}
42 self.taskhash = {} 42 self.taskhash = {}
43 self.unihash = {}
43 self.runtaskdeps = {} 44 self.runtaskdeps = {}
44 self.file_checksum_values = {} 45 self.file_checksum_values = {}
45 self.taints = {} 46 self.taints = {}
@@ -80,19 +81,19 @@ class SignatureGenerator(object):
80 return 81 return
81 82
82 def get_taskdata(self): 83 def get_taskdata(self):
83 return (self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks) 84 return (self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks)
84 85
85 def set_taskdata(self, data): 86 def set_taskdata(self, data):
86 self.runtaskdeps, self.taskhash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks = data 87 self.runtaskdeps, self.taskhash, self.unihash, self.file_checksum_values, self.taints, self.basehash, self.unitaskhashes, self.tidtopn, self.setscenetasks = data
87 88
88 def reset(self, data): 89 def reset(self, data):
89 self.__init__(data) 90 self.__init__(data)
90 91
91 def get_taskhashes(self): 92 def get_taskhashes(self):
92 return self.taskhash, self.unitaskhashes, self.tidtopn 93 return self.taskhash, self.unihash, self.unitaskhashes, self.tidtopn
93 94
94 def set_taskhashes(self, hashes): 95 def set_taskhashes(self, hashes):
95 self.taskhash, self.unitaskhashes, self.tidtopn = hashes 96 self.taskhash, self.unihash, self.unitaskhashes, self.tidtopn = hashes
96 97
97 def save_unitaskhashes(self): 98 def save_unitaskhashes(self):
98 return 99 return
@@ -108,6 +109,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
108 def __init__(self, data): 109 def __init__(self, data):
109 self.basehash = {} 110 self.basehash = {}
110 self.taskhash = {} 111 self.taskhash = {}
112 self.unihash = {}
111 self.taskdeps = {} 113 self.taskdeps = {}
112 self.runtaskdeps = {} 114 self.runtaskdeps = {}
113 self.file_checksum_values = {} 115 self.file_checksum_values = {}
@@ -256,7 +258,13 @@ class SignatureGeneratorBasic(SignatureGenerator):
256 258
257 data = self.basehash[tid] 259 data = self.basehash[tid]
258 for dep in self.runtaskdeps[tid]: 260 for dep in self.runtaskdeps[tid]:
259 data = data + self.get_unihash(dep) 261 if dep in self.unihash:
262 if self.unihash[dep] is None:
263 data = data + self.taskhash[dep]
264 else:
265 data = data + self.unihash[dep]
266 else:
267 data = data + self.get_unihash(dep)
260 268
261 for (f, cs) in self.file_checksum_values[tid]: 269 for (f, cs) in self.file_checksum_values[tid]:
262 if cs: 270 if cs:
@@ -427,6 +435,7 @@ class SignatureGeneratorUniHashMixIn(object):
427 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid) 435 (mc, fn, taskname, taskfn) = bb.runqueue.split_tid_mcfn(tid)
428 key = mc + ":" + self.tidtopn[tid] + ":" + taskname 436 key = mc + ":" + self.tidtopn[tid] + ":" + taskname
429 self.unitaskhashes[key] = (self.taskhash[tid], unihash) 437 self.unitaskhashes[key] = (self.taskhash[tid], unihash)
438 self.unihash[tid] = unihash
430 439
431 def _get_unihash(self, tid, checkkey=None): 440 def _get_unihash(self, tid, checkkey=None):
432 if tid not in self.tidtopn: 441 if tid not in self.tidtopn:
@@ -447,12 +456,14 @@ class SignatureGeneratorUniHashMixIn(object):
447 456
448 # If its not a setscene task we can return 457 # If its not a setscene task we can return
449 if self.setscenetasks and tid not in self.setscenetasks: 458 if self.setscenetasks and tid not in self.setscenetasks:
459 self.unihash[tid] = None
450 return taskhash 460 return taskhash
451 461
452 # TODO: This cache can grow unbounded. It probably only needs to keep 462 # TODO: This cache can grow unbounded. It probably only needs to keep
453 # for each task 463 # for each task
454 unihash = self._get_unihash(tid) 464 unihash = self._get_unihash(tid)
455 if unihash is not None: 465 if unihash is not None:
466 self.unihash[tid] = unihash
456 return unihash 467 return unihash
457 468
458 # In the absence of being able to discover a unique hash from the 469 # In the absence of being able to discover a unique hash from the
@@ -487,6 +498,7 @@ class SignatureGeneratorUniHashMixIn(object):
487 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) 498 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
488 499
489 self.set_unihash(tid, unihash) 500 self.set_unihash(tid, unihash)
501 self.unihash[tid] = unihash
490 return unihash 502 return unihash
491 503
492 def report_unihash(self, path, task, d): 504 def report_unihash(self, path, task, d):