diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-12-09 15:56:19 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-12-17 08:52:28 +0000 |
commit | 2946c56b233370ac4b151558079f2fc676157bad (patch) | |
tree | 231e573d1a8a9edd3327c3ea8c533f28df8234b9 /bitbake/lib/bb/runqueue.py | |
parent | 4754b1021ebdc8272b324bb8f2ffff03c8719233 (diff) | |
download | poky-2946c56b233370ac4b151558079f2fc676157bad.tar.gz |
bitbake: bitbake: siggen/runqueue: Switch to using RECIPE_SIGGEN_INFO feature for signature dumping
Now that we have cache support for the taskdep/gendep/lookupcache data,
we can switch to use that cooker feature and skip the secondary reparse to
write the sig files. This does make the initial parse longer but means the
secondary one isn't needed.
At present parsing with the larger cache isn't optimal but we have plans
in place which will make this faster than the current reparse code being
removed here.
(Bitbake rev: 5951b5b56449855bc2a30146af65eb287a35fcef)
(Bitbake rev: 1252e5bce51ae912ecff9dcc354a371786ff2c72)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 26 |
1 files changed, 13 insertions, 13 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 991aa94f67..61cb9f4c95 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -1608,28 +1608,28 @@ class RunQueue: | |||
1608 | else: | 1608 | else: |
1609 | self.rqexe.finish() | 1609 | self.rqexe.finish() |
1610 | 1610 | ||
1611 | def rq_dump_sigfn(self, fn, options): | 1611 | def _rq_dump_sigtid(self, tids): |
1612 | mc = bb.runqueue.mc_from_tid(fn) | 1612 | for tid in tids: |
1613 | the_data = self.cooker.databuilder.parseRecipe(fn, self.cooker.collections[mc].get_file_appends(fn)) | 1613 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) |
1614 | siggen = bb.parse.siggen | 1614 | dataCaches = self.rqdata.dataCaches |
1615 | dataCaches = self.rqdata.dataCaches | 1615 | bb.parse.siggen.dump_sigtask(taskfn, taskname, dataCaches[mc].stamp[taskfn], True) |
1616 | siggen.dump_sigfn(fn, dataCaches, options) | ||
1617 | 1616 | ||
1618 | def dump_signatures(self, options): | 1617 | def dump_signatures(self, options): |
1619 | fns = set() | 1618 | if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset: |
1620 | bb.note("Reparsing files to collect dependency data") | 1619 | bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled") |
1621 | 1620 | ||
1622 | for tid in self.rqdata.runtaskentries: | 1621 | bb.note("Writing task signature files") |
1623 | fn = fn_from_tid(tid) | ||
1624 | fns.add(fn) | ||
1625 | 1622 | ||
1626 | max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1) | 1623 | max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1) |
1624 | def chunkify(l, n): | ||
1625 | return [l[i::n] for i in range(n)] | ||
1626 | tids = chunkify(list(self.rqdata.runtaskentries), max_process) | ||
1627 | # We cannot use the real multiprocessing.Pool easily due to some local data | 1627 | # We cannot use the real multiprocessing.Pool easily due to some local data |
1628 | # that can't be pickled. This is a cheap multi-process solution. | 1628 | # that can't be pickled. This is a cheap multi-process solution. |
1629 | launched = [] | 1629 | launched = [] |
1630 | while fns: | 1630 | while tids: |
1631 | if len(launched) < max_process: | 1631 | if len(launched) < max_process: |
1632 | p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options)) | 1632 | p = Process(target=self._rq_dump_sigtid, args=(tids.pop(), )) |
1633 | p.start() | 1633 | p.start() |
1634 | launched.append(p) | 1634 | launched.append(p) |
1635 | for q in launched: | 1635 | for q in launched: |