summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/runqueue.py32
-rw-r--r--bitbake/lib/bb/siggen.py4
2 files changed, 29 insertions, 7 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 48c6a79ffb..d42eb81664 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -36,6 +36,7 @@ from bb import msg, data, event
36from bb import monitordisk 36from bb import monitordisk
37import subprocess 37import subprocess
38import pickle 38import pickle
39from multiprocessing import Process
39 40
40bblogger = logging.getLogger("BitBake") 41bblogger = logging.getLogger("BitBake")
41logger = logging.getLogger("BitBake.RunQueue") 42logger = logging.getLogger("BitBake.RunQueue")
@@ -1303,15 +1304,36 @@ class RunQueue:
1303 else: 1304 else:
1304 self.rqexe.finish() 1305 self.rqexe.finish()
1305 1306
1307 def rq_dump_sigfn(self, fn, options):
1308 bb_cache = bb.cache.NoCache(self.cooker.databuilder)
1309 the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
1310 siggen = bb.parse.siggen
1311 dataCaches = self.rqdata.dataCaches
1312 siggen.dump_sigfn(fn, dataCaches, options)
1313
1306 def dump_signatures(self, options): 1314 def dump_signatures(self, options):
1307 done = set() 1315 fns = set()
1308 bb.note("Reparsing files to collect dependency data") 1316 bb.note("Reparsing files to collect dependency data")
1309 bb_cache = bb.cache.NoCache(self.cooker.databuilder) 1317
1310 for tid in self.rqdata.runtaskentries: 1318 for tid in self.rqdata.runtaskentries:
1311 fn = fn_from_tid(tid) 1319 fn = fn_from_tid(tid)
1312 if fn not in done: 1320 fns.add(fn)
1313 the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) 1321
1314 done.add(fn) 1322 max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
1323 # We cannot use the real multiprocessing.Pool easily due to some local data
1324 # that can't be pickled. This is a cheap multi-process solution.
1325 launched = []
1326 while fns:
1327 if len(launched) < max_process:
1328 p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options))
1329 p.start()
1330 launched.append(p)
1331 for q in launched:
1332 # The finished processes are joined when calling is_alive()
1333 if not q.is_alive():
1334 launched.remove(q)
1335 for p in launched:
1336 p.join()
1315 1337
1316 bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options) 1338 bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options)
1317 1339
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 4226c80c85..3ceeef1ccc 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -307,8 +307,8 @@ class SignatureGeneratorBasic(SignatureGenerator):
307 pass 307 pass
308 raise err 308 raise err
309 309
310 def dump_sigs(self, dataCaches, options): 310 def dump_sigfn(self, fn, dataCaches, options):
311 for fn in self.taskdeps: 311 if fn in self.taskdeps:
312 for task in self.taskdeps[fn]: 312 for task in self.taskdeps[fn]:
313 tid = fn + ":" + task 313 tid = fn + ":" + task
314 (mc, _, _) = bb.runqueue.split_tid(tid) 314 (mc, _, _) = bb.runqueue.split_tid(tid)