diff options
Diffstat (limited to 'bitbake/lib')
-rwxr-xr-x | bitbake/lib/bb/main.py | 5 | ||||
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 26 | ||||
-rw-r--r-- | bitbake/lib/bb/siggen.py | 39 |
3 files changed, 37 insertions, 33 deletions
diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py index f1ea7859da..ed3e1ede61 100755 --- a/bitbake/lib/bb/main.py +++ b/bitbake/lib/bb/main.py | |||
@@ -395,6 +395,11 @@ def setup_bitbake(configParams, extrafeatures=None): | |||
395 | # In status only mode there are no logs and no UI | 395 | # In status only mode there are no logs and no UI |
396 | logger.addHandler(handler) | 396 | logger.addHandler(handler) |
397 | 397 | ||
398 | if configParams.dump_signatures: | ||
399 | if extrafeatures is None: | ||
400 | extrafeatures = [] | ||
401 | extrafeatures.append(bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO) | ||
402 | |||
398 | if configParams.server_only: | 403 | if configParams.server_only: |
399 | featureset = [] | 404 | featureset = [] |
400 | ui_module = None | 405 | ui_module = None |
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 991aa94f67..61cb9f4c95 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -1608,28 +1608,28 @@ class RunQueue: | |||
1608 | else: | 1608 | else: |
1609 | self.rqexe.finish() | 1609 | self.rqexe.finish() |
1610 | 1610 | ||
1611 | def rq_dump_sigfn(self, fn, options): | 1611 | def _rq_dump_sigtid(self, tids): |
1612 | mc = bb.runqueue.mc_from_tid(fn) | 1612 | for tid in tids: |
1613 | the_data = self.cooker.databuilder.parseRecipe(fn, self.cooker.collections[mc].get_file_appends(fn)) | 1613 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
1614 | siggen = bb.parse.siggen | 1614 | dataCaches = self.rqdata.dataCaches |
1615 | dataCaches = self.rqdata.dataCaches | 1615 | bb.parse.siggen.dump_sigtask(taskfn, taskname, dataCaches[mc].stamp[taskfn], True) |
1616 | siggen.dump_sigfn(fn, dataCaches, options) | ||
1617 | 1616 | ||
1618 | def dump_signatures(self, options): | 1617 | def dump_signatures(self, options): |
1619 | fns = set() | 1618 | if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset: |
1620 | bb.note("Reparsing files to collect dependency data") | 1619 | bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled") |
1621 | 1620 | ||
1622 | for tid in self.rqdata.runtaskentries: | 1621 | bb.note("Writing task signature files") |
1623 | fn = fn_from_tid(tid) | ||
1624 | fns.add(fn) | ||
1625 | 1622 | ||
1626 | max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1) | 1623 | max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1) |
1624 | def chunkify(l, n): | ||
1625 | return [l[i::n] for i in range(n)] | ||
1626 | tids = chunkify(list(self.rqdata.runtaskentries), max_process) | ||
1627 | # We cannot use the real multiprocessing.Pool easily due to some local data | 1627 | # We cannot use the real multiprocessing.Pool easily due to some local data |
1628 | # that can't be pickled. This is a cheap multi-process solution. | 1628 | # that can't be pickled. This is a cheap multi-process solution. |
1629 | launched = [] | 1629 | launched = [] |
1630 | while fns: | 1630 | while tids: |
1631 | if len(launched) < max_process: | 1631 | if len(launched) < max_process: |
1632 | p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options)) | 1632 | p = Process(target=self._rq_dump_sigtid, args=(tids.pop(), )) |
1633 | p.start() | 1633 | p.start() |
1634 | launched.append(p) | 1634 | launched.append(p) |
1635 | for q in launched: | 1635 | for q in launched: |
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index e57f1ffe69..6b73843c67 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py | |||
@@ -335,8 +335,8 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
335 | self.unihash_cache.copyfile(targetdir) | 335 | self.unihash_cache.copyfile(targetdir) |
336 | 336 | ||
337 | def dump_sigtask(self, fn, task, stampbase, runtime): | 337 | def dump_sigtask(self, fn, task, stampbase, runtime): |
338 | |||
339 | tid = fn + ":" + task | 338 | tid = fn + ":" + task |
339 | mc = bb.runqueue.mc_from_tid(fn) | ||
340 | referencestamp = stampbase | 340 | referencestamp = stampbase |
341 | if isinstance(runtime, str) and runtime.startswith("customfile"): | 341 | if isinstance(runtime, str) and runtime.startswith("customfile"): |
342 | sigfile = stampbase | 342 | sigfile = stampbase |
@@ -353,16 +353,27 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
353 | data['task'] = task | 353 | data['task'] = task |
354 | data['basehash_ignore_vars'] = self.basehash_ignore_vars | 354 | data['basehash_ignore_vars'] = self.basehash_ignore_vars |
355 | data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks | 355 | data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks |
356 | data['taskdeps'] = self.taskdeps[fn][task] | 356 | if hasattr(self, "datacaches"): |
357 | data['taskdeps'] = self.datacaches[mc].siggen_taskdeps[fn][task] | ||
358 | else: | ||
359 | data['taskdeps'] = self.taskdeps[fn][task] | ||
357 | data['basehash'] = self.basehash[tid] | 360 | data['basehash'] = self.basehash[tid] |
358 | data['gendeps'] = {} | 361 | data['gendeps'] = {} |
359 | data['varvals'] = {} | 362 | data['varvals'] = {} |
360 | data['varvals'][task] = self.lookupcache[fn][task] | 363 | if hasattr(self, "datacaches"): |
361 | for dep in self.taskdeps[fn][task]: | 364 | data['varvals'][task] = self.datacaches[mc].siggen_varvals[fn][task] |
362 | if dep in self.basehash_ignore_vars: | 365 | for dep in self.datacaches[mc].siggen_taskdeps[fn][task]: |
363 | continue | 366 | if dep in self.basehash_ignore_vars: |
364 | data['gendeps'][dep] = self.gendeps[fn][dep] | 367 | continue |
365 | data['varvals'][dep] = self.lookupcache[fn][dep] | 368 | data['gendeps'][dep] = self.datacaches[mc].siggen_gendeps[fn][dep] |
369 | data['varvals'][dep] = self.datacaches[mc].siggen_varvals[fn][dep] | ||
370 | else: | ||
371 | data['varvals'][task] = self.lookupcache[fn][task] | ||
372 | for dep in self.taskdeps[fn][task]: | ||
373 | if dep in self.basehash_ignore_vars: | ||
374 | continue | ||
375 | data['gendeps'][dep] = self.gendeps[fn][dep] | ||
376 | data['varvals'][dep] = self.lookupcache[fn][dep] | ||
366 | 377 | ||
367 | if runtime and tid in self.taskhash: | 378 | if runtime and tid in self.taskhash: |
368 | data['runtaskdeps'] = self.runtaskdeps[tid] | 379 | data['runtaskdeps'] = self.runtaskdeps[tid] |
@@ -409,18 +420,6 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
409 | pass | 420 | pass |
410 | raise err | 421 | raise err |
411 | 422 | ||
412 | def dump_sigfn(self, fn, dataCaches, options): | ||
413 | if fn in self.taskdeps: | ||
414 | for task in self.taskdeps[fn]: | ||
415 | tid = fn + ":" + task | ||
416 | mc = bb.runqueue.mc_from_tid(tid) | ||
417 | if tid not in self.taskhash: | ||
418 | continue | ||
419 | if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]: | ||
420 | bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % tid) | ||
421 | bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid])) | ||
422 | self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True) | ||
423 | |||
424 | class SignatureGeneratorBasicHash(SignatureGeneratorBasic): | 423 | class SignatureGeneratorBasicHash(SignatureGeneratorBasic): |
425 | name = "basichash" | 424 | name = "basichash" |
426 | 425 | ||