diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-01-04 17:47:44 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-01-06 17:40:01 +0000 |
commit | 9043034da53658ab2db67da28699e8a01bfe5206 (patch) | |
tree | bbf955cc3a0b180a9b849b35beb8b2fc5f17dfe6 /bitbake/lib/bb/cooker.py | |
parent | 69abe10566f71f3e579e35e9c39e7ef1e4dcac47 (diff) | |
download | poky-9043034da53658ab2db67da28699e8a01bfe5206.tar.gz |
bitbake: cooker: Rework the parsing results submission
The loop submitting the parser results was not efficient on many core systems
as may of the parsers could block for a long time waiting to send the results.
While a result was pending, the code wouldn't parse further jobs.
Change the code to parse further jobs even if the results can't be submitted
and lower the result submission timeout to keep the parsers busy.
(Bitbake rev: 9ece4a2e406509bd89dbce8dac80a02e600b0ecf)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r-- | bitbake/lib/bb/cooker.py | 26 |
1 files changed, 15 insertions, 11 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 738849d189..1d347ddc52 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -2107,25 +2107,29 @@ class Parser(multiprocessing.Process): | |||
2107 | multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) | 2107 | multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) |
2108 | 2108 | ||
2109 | pending = [] | 2109 | pending = [] |
2110 | havejobs = True | ||
2110 | try: | 2111 | try: |
2111 | while True: | 2112 | while havejobs or pending: |
2112 | if self.quit.is_set(): | 2113 | if self.quit.is_set(): |
2113 | break | 2114 | break |
2114 | 2115 | ||
2115 | if pending: | 2116 | job = None |
2116 | result = pending.pop() | 2117 | try: |
2117 | else: | 2118 | job = self.jobs.pop() |
2118 | try: | 2119 | except IndexError: |
2119 | job = self.jobs.pop() | 2120 | havejobs = False |
2120 | except IndexError: | 2121 | if job: |
2121 | break | ||
2122 | result = self.parse(*job) | 2122 | result = self.parse(*job) |
2123 | # Clear the siggen cache after parsing to control memory usage, its huge | 2123 | # Clear the siggen cache after parsing to control memory usage, its huge |
2124 | bb.parse.siggen.postparsing_clean_cache() | 2124 | bb.parse.siggen.postparsing_clean_cache() |
2125 | try: | ||
2126 | self.results.put(result, timeout=0.25) | ||
2127 | except queue.Full: | ||
2128 | pending.append(result) | 2125 | pending.append(result) |
2126 | |||
2127 | if pending: | ||
2128 | try: | ||
2129 | result = pending.pop() | ||
2130 | self.results.put(result, timeout=0.05) | ||
2131 | except queue.Full: | ||
2132 | pending.append(result) | ||
2129 | finally: | 2133 | finally: |
2130 | self.results.close() | 2134 | self.results.close() |
2131 | self.results.join_thread() | 2135 | self.results.join_thread() |