summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-12-28 10:23:05 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-01-08 11:16:03 +0000
commit77c3b26533b3ac28189e48833c393d1e7e615811 (patch)
treea0ddea5fbf74f08ad94ea318003ff2424cdacdf3 /bitbake/lib/bb
parent94b37d134dc45a51b92a6e0cddbdca4b309c23c5 (diff)
downloadpoky-77c3b26533b3ac28189e48833c393d1e7e615811.tar.gz
bitbake: cooker: Remove Feeder() since its no longer needed with moderm multiprocessing
There used to be many bugs in multiprocessing and we implemented our own feeder process to avoid them. Now that we have python 3.x, these are fixed and just using the standard Queue mechanism appears to work fine. We can therefore drop the unneeded code and simplify. (Bitbake rev: b2d39fc37fcf3c81a562ec1ef4f8b4c1a493fc57) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/cooker.py42
1 files changed, 5 insertions, 37 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index db52964c3a..d1d2868d6f 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1895,35 +1895,6 @@ class ParsingFailure(Exception):
1895 self.recipe = recipe 1895 self.recipe = recipe
1896 Exception.__init__(self, realexception, recipe) 1896 Exception.__init__(self, realexception, recipe)
1897 1897
1898class Feeder(multiprocessing.Process):
1899 def __init__(self, jobs, to_parsers, quit):
1900 self.quit = quit
1901 self.jobs = jobs
1902 self.to_parsers = to_parsers
1903 multiprocessing.Process.__init__(self)
1904
1905 def run(self):
1906 while True:
1907 try:
1908 quit = self.quit.get_nowait()
1909 except queue.Empty:
1910 pass
1911 else:
1912 if quit == 'cancel':
1913 self.to_parsers.cancel_join_thread()
1914 break
1915
1916 try:
1917 job = self.jobs.pop()
1918 except IndexError:
1919 break
1920
1921 try:
1922 self.to_parsers.put(job, timeout=0.5)
1923 except queue.Full:
1924 self.jobs.insert(0, job)
1925 continue
1926
1927class Parser(multiprocessing.Process): 1898class Parser(multiprocessing.Process):
1928 def __init__(self, jobs, results, quit, init, profile): 1899 def __init__(self, jobs, results, quit, init, profile):
1929 self.jobs = jobs 1900 self.jobs = jobs
@@ -2058,12 +2029,13 @@ class CookerParser(object):
2058 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) 2029 multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
2059 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) 2030 multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
2060 2031
2061 self.feeder_quit = multiprocessing.Queue(maxsize=1)
2062 self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) 2032 self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
2063 self.jobs = multiprocessing.Queue(maxsize=self.num_processes)
2064 self.result_queue = multiprocessing.Queue() 2033 self.result_queue = multiprocessing.Queue()
2065 self.feeder = Feeder(self.willparse, self.jobs, self.feeder_quit) 2034
2066 self.feeder.start() 2035 self.jobs = multiprocessing.Queue()
2036 for j in self.willparse:
2037 self.jobs.put(j)
2038
2067 for i in range(0, self.num_processes): 2039 for i in range(0, self.num_processes):
2068 parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) 2040 parser = Parser(self.jobs, self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
2069 parser.start() 2041 parser.start()
@@ -2086,12 +2058,9 @@ class CookerParser(object):
2086 self.total) 2058 self.total)
2087 2059
2088 bb.event.fire(event, self.cfgdata) 2060 bb.event.fire(event, self.cfgdata)
2089 self.feeder_quit.put(None)
2090 for process in self.processes: 2061 for process in self.processes:
2091 self.parser_quit.put(None) 2062 self.parser_quit.put(None)
2092 else: 2063 else:
2093 self.feeder_quit.put('cancel')
2094
2095 self.parser_quit.cancel_join_thread() 2064 self.parser_quit.cancel_join_thread()
2096 for process in self.processes: 2065 for process in self.processes:
2097 self.parser_quit.put(None) 2066 self.parser_quit.put(None)
@@ -2104,7 +2073,6 @@ class CookerParser(object):
2104 process.terminate() 2073 process.terminate()
2105 else: 2074 else:
2106 process.join() 2075 process.join()
2107 self.feeder.join()
2108 2076
2109 sync = threading.Thread(target=self.bb_cache.sync) 2077 sync = threading.Thread(target=self.bb_cache.sync)
2110 sync.start() 2078 sync.start()