summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/runqueue.py
diff options
context:
space:
mode:
authorJianxun Zhang <jianxun.zhang@linux.intel.com>2016-12-21 12:27:37 -0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-01-17 11:29:28 +0000
commit7eb02e837ed253a2ae69023595c25660b4d56b7c (patch)
tree3eb6a36ac22a10787419b7cb369dfc4ba3bf2e55 /bitbake/lib/bb/runqueue.py
parent62591d97221714678a3a02c5c42fffe9e538f5f9 (diff)
downloadpoky-7eb02e837ed253a2ae69023595c25660b4d56b7c.tar.gz
bitbake: use multiple processes to dump signatures.
This change significantly shortens the time on reparsing stage of '-S' option. Each file is reparsed and then dumped within a dedicated process. The maximum number of the running processes is not greater than the value of BB_NUMBER_PARSE_THREADS if it is set. The dump_sigs() in class SignatureGeneratorBasic is _replaced_ by a new dump_sigfn() interface, so calls from the outside and subclasses are dispatched to the implementation in the base class of SignatureGeneratorBasic. Fixes [YOCTO #10352] (Bitbake rev: 99d3703edd77a21770b366c6ad65a3c0f5183493) Signed-off-by: Jianxun Zhang <jianxun.zhang@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r--bitbake/lib/bb/runqueue.py32
1 files changed, 27 insertions, 5 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 48c6a79ffb..d42eb81664 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -36,6 +36,7 @@ from bb import msg, data, event
36from bb import monitordisk 36from bb import monitordisk
37import subprocess 37import subprocess
38import pickle 38import pickle
39from multiprocessing import Process
39 40
40bblogger = logging.getLogger("BitBake") 41bblogger = logging.getLogger("BitBake")
41logger = logging.getLogger("BitBake.RunQueue") 42logger = logging.getLogger("BitBake.RunQueue")
@@ -1303,15 +1304,36 @@ class RunQueue:
1303 else: 1304 else:
1304 self.rqexe.finish() 1305 self.rqexe.finish()
1305 1306
1307 def rq_dump_sigfn(self, fn, options):
1308 bb_cache = bb.cache.NoCache(self.cooker.databuilder)
1309 the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
1310 siggen = bb.parse.siggen
1311 dataCaches = self.rqdata.dataCaches
1312 siggen.dump_sigfn(fn, dataCaches, options)
1313
1306 def dump_signatures(self, options): 1314 def dump_signatures(self, options):
1307 done = set() 1315 fns = set()
1308 bb.note("Reparsing files to collect dependency data") 1316 bb.note("Reparsing files to collect dependency data")
1309 bb_cache = bb.cache.NoCache(self.cooker.databuilder) 1317
1310 for tid in self.rqdata.runtaskentries: 1318 for tid in self.rqdata.runtaskentries:
1311 fn = fn_from_tid(tid) 1319 fn = fn_from_tid(tid)
1312 if fn not in done: 1320 fns.add(fn)
1313 the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) 1321
1314 done.add(fn) 1322 max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
1323 # We cannot use the real multiprocessing.Pool easily due to some local data
1324 # that can't be pickled. This is a cheap multi-process solution.
1325 launched = []
1326 while fns:
1327 if len(launched) < max_process:
1328 p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options))
1329 p.start()
1330 launched.append(p)
1331 for q in launched:
1332 # The finished processes are joined when calling is_alive()
1333 if not q.is_alive():
1334 launched.remove(q)
1335 for p in launched:
1336 p.join()
1315 1337
1316 bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options) 1338 bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options)
1317 1339