summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-12-28 16:40:33 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-08 11:16:03 +0000
commitd4036c6fcf20fbae52293343d0a1ee38e5f0c5ba (patch)
treec62d03f9acce36b3bec55b113ee1bf5df5d7c48b /bitbake/lib
parent77c3b26533b3ac28189e48833c393d1e7e615811 (diff)
downloadpoky-d4036c6fcf20fbae52293343d0a1ee38e5f0c5ba.tar.gz
bitbake: cooker: Split recipes to parse amongst threads ahead of time
We have two choices, split the recipes amongst the parsing threads in blocks ahead of time, or have a queue which parsers pull from when idle. The optimum approach depends on how similar the pieces are. For the single recipe reparse case, there is currently a significant wait for the feeder thread to start (around 0.25s in a 2s command). Its possible splitting into blocks in advance may be unluckly for some other workloads but experimentally it seems to work better overall for me at least. (Bitbake rev: ae79868861568d673a70472e85a4bde9e2d84a8f) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/cooker.py17
1 files changed, 6 insertions, 11 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index d1d2868d6f..e6b8d880ae 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1941,11 +1941,8 @@ class Parser(multiprocessing.Process):
1941 result = pending.pop() 1941 result = pending.pop()
1942 else: 1942 else:
1943 try: 1943 try:
1944 job = self.jobs.get(timeout=0.25) 1944 job = self.jobs.pop()
1945 except queue.Empty: 1945 except IndexError:
1946 continue
1947
1948 if job is None:
1949 break 1946 break
1950 result = self.parse(*job) 1947 result = self.parse(*job)
1951 1948
@@ -2032,12 +2029,12 @@ class CookerParser(object):
2032 self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) 2029 self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
2033 self.result_queue = multiprocessing.Queue() 2030 self.result_queue = multiprocessing.Queue()
2034 2031
2035 self.jobs = multiprocessing.Queue() 2032 def chunkify(lst,n):
2036 for j in self.willparse: 2033 return [lst[i::n] for i in range(n)]
2037 self.jobs.put(j) 2034 self.jobs = chunkify(self.willparse, self.num_processes)
2038 2035
2039 for i in range(0, self.num_processes): 2036 for i in range(0, self.num_processes):
2040 parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) 2037 parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
2041 parser.start() 2038 parser.start()
2042 self.process_names.append(parser.name) 2039 self.process_names.append(parser.name)
2043 self.processes.append(parser) 2040 self.processes.append(parser)
@@ -2065,8 +2062,6 @@ class CookerParser(object):
2065 for process in self.processes: 2062 for process in self.processes:
2066 self.parser_quit.put(None) 2063 self.parser_quit.put(None)
2067 2064
2068 self.jobs.cancel_join_thread()
2069
2070 for process in self.processes: 2065 for process in self.processes:
2071 if force: 2066 if force:
2072 process.join(.1) 2067 process.join(.1)