diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-12-11 16:37:24 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-12-17 08:52:28 +0000 |
commit | 878de40a501089048cffdc867b2ae1ad8a2ec9e5 (patch) | |
tree | ed48deb9e5f2d056d0b0c6df8d0109e0bbd40a97 /bitbake | |
parent | 2946c56b233370ac4b151558079f2fc676157bad (diff) | |
download | poky-878de40a501089048cffdc867b2ae1ad8a2ec9e5.tar.gz |
bitbake: siggen: Add dummy dataCaches from task context/datastore
One of the challenges in maintaining the code is that it sometimes uses
a datacaches structure and sometimes a datastore. Rather than continue
the current dual API madness, have the worker contexts create a dummy
datacaches structure with the entries we need. Whilst this does need to
be kept in sync with the real structure, that doesn't change and this
allows the code to be simplified.
With this new approach, we can unify the stamps dependency code again.
(Bitbake rev: c6d325fc9b53e9d588ab273ee3c2a99a70fba42c)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rwxr-xr-x | bitbake/bin/bitbake-worker | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/cooker.py | 2 | ||||
-rw-r--r-- | bitbake/lib/bb/siggen.py | 77 |
3 files changed, 43 insertions, 37 deletions
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker index 3cacdb0cec..ed266f0ac2 100755 --- a/bitbake/bin/bitbake-worker +++ b/bitbake/bin/bitbake-worker | |||
@@ -266,6 +266,7 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask): | |||
266 | the_data = databuilder.parseRecipe(fn, appends) | 266 | the_data = databuilder.parseRecipe(fn, appends) |
267 | the_data.setVar('BB_TASKHASH', taskhash) | 267 | the_data.setVar('BB_TASKHASH', taskhash) |
268 | the_data.setVar('BB_UNIHASH', unihash) | 268 | the_data.setVar('BB_UNIHASH', unihash) |
269 | bb.parse.siggen.setup_datacache_from_datastore(fn, the_data) | ||
269 | 270 | ||
270 | bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", ""))) | 271 | bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", ""))) |
271 | 272 | ||
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 728ef09675..ac7ac20c04 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -1448,6 +1448,8 @@ class BBCooker: | |||
1448 | self.recipecaches[mc].rundeps[fn] = defaultdict(list) | 1448 | self.recipecaches[mc].rundeps[fn] = defaultdict(list) |
1449 | self.recipecaches[mc].runrecs[fn] = defaultdict(list) | 1449 | self.recipecaches[mc].runrecs[fn] = defaultdict(list) |
1450 | 1450 | ||
1451 | bb.parse.siggen.setup_datacache(self.recipecaches) | ||
1452 | |||
1451 | # Invalidate task for target if force mode active | 1453 | # Invalidate task for target if force mode active |
1452 | if self.configuration.force: | 1454 | if self.configuration.force: |
1453 | logger.verbose("Invalidate task %s, %s", task, fn) | 1455 | logger.verbose("Invalidate task %s, %s", task, fn) |
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 6b73843c67..a63e37d22f 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py | |||
@@ -14,6 +14,7 @@ import bb.data | |||
14 | import difflib | 14 | import difflib |
15 | import simplediff | 15 | import simplediff |
16 | import json | 16 | import json |
17 | import types | ||
17 | import bb.compress.zstd | 18 | import bb.compress.zstd |
18 | from bb.checksum import FileChecksumCache | 19 | from bb.checksum import FileChecksumCache |
19 | from bb import runqueue | 20 | from bb import runqueue |
@@ -73,6 +74,24 @@ class SignatureGenerator(object): | |||
73 | def setup_datacache(self, datacaches): | 74 | def setup_datacache(self, datacaches): |
74 | self.datacaches = datacaches | 75 | self.datacaches = datacaches |
75 | 76 | ||
77 | def setup_datacache_from_datastore(self, mcfn, d): | ||
78 | # In task context we have no cache so setup internal data structures | ||
79 | # from the fully parsed data store provided | ||
80 | |||
81 | mc = d.getVar("__BBMULTICONFIG", False) or "" | ||
82 | tasks = d.getVar('__BBTASKS', False) | ||
83 | |||
84 | self.datacaches = {} | ||
85 | self.datacaches[mc] = types.SimpleNamespace() | ||
86 | setattr(self.datacaches[mc], "stamp", {}) | ||
87 | self.datacaches[mc].stamp[mcfn] = d.getVar('STAMP') | ||
88 | setattr(self.datacaches[mc], "stamp_extrainfo", {}) | ||
89 | self.datacaches[mc].stamp_extrainfo[mcfn] = {} | ||
90 | for t in tasks: | ||
91 | flag = d.getVarFlag(t, "stamp-extra-info") | ||
92 | if flag: | ||
93 | self.datacaches[mc].stamp_extrainfo[mcfn][t] = flag | ||
94 | |||
76 | def get_unihash(self, tid): | 95 | def get_unihash(self, tid): |
77 | return self.taskhash[tid] | 96 | return self.taskhash[tid] |
78 | 97 | ||
@@ -138,12 +157,9 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
138 | self.basehash = {} | 157 | self.basehash = {} |
139 | self.taskhash = {} | 158 | self.taskhash = {} |
140 | self.unihash = {} | 159 | self.unihash = {} |
141 | self.taskdeps = {} | ||
142 | self.runtaskdeps = {} | 160 | self.runtaskdeps = {} |
143 | self.file_checksum_values = {} | 161 | self.file_checksum_values = {} |
144 | self.taints = {} | 162 | self.taints = {} |
145 | self.gendeps = {} | ||
146 | self.lookupcache = {} | ||
147 | self.setscenetasks = set() | 163 | self.setscenetasks = set() |
148 | self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split()) | 164 | self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split()) |
149 | self.taskhash_ignore_tasks = None | 165 | self.taskhash_ignore_tasks = None |
@@ -186,11 +202,7 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
186 | bb.error("%s -Sprintdiff\n" % cmd) | 202 | bb.error("%s -Sprintdiff\n" % cmd) |
187 | self.basehash[tid] = basehash[tid] | 203 | self.basehash[tid] = basehash[tid] |
188 | 204 | ||
189 | self.taskdeps[fn] = taskdeps | 205 | return taskdeps, gendeps, lookupcache |
190 | self.gendeps[fn] = gendeps | ||
191 | self.lookupcache[fn] = lookupcache | ||
192 | |||
193 | return taskdeps | ||
194 | 206 | ||
195 | def set_setscene_tasks(self, setscene_tasks): | 207 | def set_setscene_tasks(self, setscene_tasks): |
196 | self.setscenetasks = set(setscene_tasks) | 208 | self.setscenetasks = set(setscene_tasks) |
@@ -202,7 +214,7 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
202 | fn = bb.cache.realfn2virtual(fn, variant, mc) | 214 | fn = bb.cache.realfn2virtual(fn, variant, mc) |
203 | 215 | ||
204 | try: | 216 | try: |
205 | taskdeps = self._build_data(fn, d) | 217 | taskdeps, gendeps, lookupcache = self._build_data(fn, d) |
206 | except bb.parse.SkipRecipe: | 218 | except bb.parse.SkipRecipe: |
207 | raise | 219 | raise |
208 | except: | 220 | except: |
@@ -218,18 +230,20 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
218 | basehashes[task] = self.basehash[fn + ":" + task] | 230 | basehashes[task] = self.basehash[fn + ":" + task] |
219 | 231 | ||
220 | d.setVar("__siggen_basehashes", basehashes) | 232 | d.setVar("__siggen_basehashes", basehashes) |
221 | d.setVar("__siggen_gendeps", self.gendeps[fn]) | 233 | d.setVar("__siggen_gendeps", gendeps) |
222 | d.setVar("__siggen_varvals", self.lookupcache[fn]) | 234 | d.setVar("__siggen_varvals", lookupcache) |
223 | d.setVar("__siggen_taskdeps", self.taskdeps[fn]) | 235 | d.setVar("__siggen_taskdeps", taskdeps) |
224 | 236 | ||
237 | def setup_datacache_from_datastore(self, mcfn, d): | ||
238 | super().setup_datacache_from_datastore(mcfn, d) | ||
225 | 239 | ||
226 | def postparsing_clean_cache(self): | 240 | mc = bb.runqueue.mc_from_tid(mcfn) |
227 | # | 241 | setattr(self.datacaches[mc], "siggen_varvals", {}) |
228 | # After parsing we can remove some things from memory to reduce our memory footprint | 242 | self.datacaches[mc].siggen_varvals[mcfn] = d.getVar("__siggen_varvals") |
229 | # | 243 | setattr(self.datacaches[mc], "siggen_taskdeps", {}) |
230 | self.gendeps = {} | 244 | self.datacaches[mc].siggen_taskdeps[mcfn] = d.getVar("__siggen_taskdeps") |
231 | self.lookupcache = {} | 245 | setattr(self.datacaches[mc], "siggen_gendeps", {}) |
232 | self.taskdeps = {} | 246 | self.datacaches[mc].siggen_gendeps[mcfn] = d.getVar("__siggen_gendeps") |
233 | 247 | ||
234 | def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): | 248 | def rundep_check(self, fn, recipename, task, dep, depname, dataCaches): |
235 | # Return True if we should keep the dependency, False to drop it | 249 | # Return True if we should keep the dependency, False to drop it |
@@ -353,27 +367,16 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
353 | data['task'] = task | 367 | data['task'] = task |
354 | data['basehash_ignore_vars'] = self.basehash_ignore_vars | 368 | data['basehash_ignore_vars'] = self.basehash_ignore_vars |
355 | data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks | 369 | data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks |
356 | if hasattr(self, "datacaches"): | 370 | data['taskdeps'] = self.datacaches[mc].siggen_taskdeps[fn][task] |
357 | data['taskdeps'] = self.datacaches[mc].siggen_taskdeps[fn][task] | ||
358 | else: | ||
359 | data['taskdeps'] = self.taskdeps[fn][task] | ||
360 | data['basehash'] = self.basehash[tid] | 371 | data['basehash'] = self.basehash[tid] |
361 | data['gendeps'] = {} | 372 | data['gendeps'] = {} |
362 | data['varvals'] = {} | 373 | data['varvals'] = {} |
363 | if hasattr(self, "datacaches"): | 374 | data['varvals'][task] = self.datacaches[mc].siggen_varvals[fn][task] |
364 | data['varvals'][task] = self.datacaches[mc].siggen_varvals[fn][task] | 375 | for dep in self.datacaches[mc].siggen_taskdeps[fn][task]: |
365 | for dep in self.datacaches[mc].siggen_taskdeps[fn][task]: | 376 | if dep in self.basehash_ignore_vars: |
366 | if dep in self.basehash_ignore_vars: | 377 | continue |
367 | continue | 378 | data['gendeps'][dep] = self.datacaches[mc].siggen_gendeps[fn][dep] |
368 | data['gendeps'][dep] = self.datacaches[mc].siggen_gendeps[fn][dep] | 379 | data['varvals'][dep] = self.datacaches[mc].siggen_varvals[fn][dep] |
369 | data['varvals'][dep] = self.datacaches[mc].siggen_varvals[fn][dep] | ||
370 | else: | ||
371 | data['varvals'][task] = self.lookupcache[fn][task] | ||
372 | for dep in self.taskdeps[fn][task]: | ||
373 | if dep in self.basehash_ignore_vars: | ||
374 | continue | ||
375 | data['gendeps'][dep] = self.gendeps[fn][dep] | ||
376 | data['varvals'][dep] = self.lookupcache[fn][dep] | ||
377 | 380 | ||
378 | if runtime and tid in self.taskhash: | 381 | if runtime and tid in self.taskhash: |
379 | data['runtaskdeps'] = self.runtaskdeps[tid] | 382 | data['runtaskdeps'] = self.runtaskdeps[tid] |