summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2024-11-29 12:51:23 +0000
committerSteve Sakoman <steve@sakoman.com>2024-12-09 06:25:53 -0800
commite619153f24884928bd0b45a9bec06a1a0120b137 (patch)
tree8f04f24d5165a9a2a02c5c45f2b59ed356ba11c6
parent7e081bd98fdc5435e850d1df79a5e0f1e30293d0 (diff)
downloadpoky-e619153f24884928bd0b45a9bec06a1a0120b137.tar.gz
bitbake: runqueue: Fix performance of multiconfigs with large overlap
There have been complaints about the performance of large multiconfig builds for a while. The key missing data point was that the builds needed to have large overlaps in sstate objects. This can be simulated by building the same things with just different TMPDIRs. In runqueue/bitbake terms this equates to large numbers of deferred tasks. The issue is that the expensive checks in the setscene loop were hit every time through runqueue's execute function before the check on deferred tasks. This leads to task execution starvation as that only happens once per iteration. Move the skip check earlier in the function which speeds things up enormously and should improve performance of such builds for users. (Bitbake rev: 5392a2bf6d82626e11eec5578eb4de53ba09f1fe) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> (cherry picked from commit 9c6c506757f2b3e28c8b20513b45da6b4659c95f) Signed-off-by: Steve Sakoman <steve@sakoman.com>
-rw-r--r--bitbake/lib/bb/runqueue.py6
1 files changed, 4 insertions, 2 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 3462ed4457..9f3abff85f 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -2195,6 +2195,9 @@ class RunQueueExecute:
2195 # Find the next setscene to run 2195 # Find the next setscene to run
2196 for nexttask in self.sorted_setscene_tids: 2196 for nexttask in self.sorted_setscene_tids:
2197 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred: 2197 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred:
2198 if nexttask in self.sq_deferred and self.sq_deferred[nexttask] not in self.runq_complete:
2199 # Skip deferred tasks quickly before the 'expensive' tests below - this is key to performant multiconfig builds
2200 continue
2198 if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \ 2201 if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \
2199 nexttask not in self.sq_needed_harddeps and \ 2202 nexttask not in self.sq_needed_harddeps and \
2200 self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \ 2203 self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \
@@ -2224,8 +2227,7 @@ class RunQueueExecute:
2224 if t in self.runq_running and t not in self.runq_complete: 2227 if t in self.runq_running and t not in self.runq_complete:
2225 continue 2228 continue
2226 if nexttask in self.sq_deferred: 2229 if nexttask in self.sq_deferred:
2227 if self.sq_deferred[nexttask] not in self.runq_complete: 2230 # Deferred tasks that were still deferred were skipped above so we now need to process
2228 continue
2229 logger.debug("Task %s no longer deferred" % nexttask) 2231 logger.debug("Task %s no longer deferred" % nexttask)
2230 del self.sq_deferred[nexttask] 2232 del self.sq_deferred[nexttask]
2231 valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False) 2233 valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False)