bitbake: runqueue: Improve setcene performance when encoutering many 'hard' dependencies

"bitbake world -n --setscene-only" shows poor performance as the numbers of tasks
increases. Analysys shows this is due to the "deferred" hard dependencies being
repeatedly processed which doesn't allow much forward progress in overall task
execution.

To avoid this, mark when it has been done and don't reprocess until dependencies
are updated. We have to be careful as we've seen bugs where these dependencies
aren't processed at the right time. They also have to be reproceseed during task
migration/rehashing so the code has to "self heal" the data structures.

(Bitbake rev: e5609bac06c17dabcf6286b47b1a3f19f5a1160f)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 11 insertions, 1 deletions

diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 95cab5132f..e86ccd8c61 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -1840,6 +1840,7 @@ class RunQueueExecute:
         self.failed_tids = []
         self.sq_deferred = {}
         self.sq_needed_harddeps = set()
+        self.sq_harddep_deferred = set()
 
         self.stampcache = {}
 
@@ -2163,7 +2164,7 @@ class RunQueueExecute:
         if not self.sqdone and self.can_start_task():
             # Find the next setscene to run
             for nexttask in self.sorted_setscene_tids:
-                if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
+                if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred:
                     if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \
                             nexttask not in self.sq_needed_harddeps and \
                             self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \
@@ -2184,6 +2185,7 @@ class RunQueueExecute:
                                 self.sq_buildable.add(dep)
                                 self.sq_needed_harddeps.add(dep)
                                 updated = True
+                        self.sq_harddep_deferred.add(nexttask)
                         if updated:
                             return True
                         continue
@@ -2678,6 +2680,7 @@ class RunQueueExecute:
         if changed:
             self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
             self.sq_needed_harddeps = set()
+            self.sq_harddep_deferred = set()
             self.holdoff_need_update = True
 
     def scenequeue_updatecounters(self, task, fail=False):
@@ -2712,6 +2715,13 @@ class RunQueueExecute:
                         new.add(dep)
             next = new
 
+        # If this task was one which other setscene tasks have a hard dependency upon, we need
+        # to walk through the hard dependencies and allow execution of those which have completed dependencies.
+        if task in self.sqdata.sq_harddeps:
+            for dep in self.sq_harddep_deferred.copy():
+                if self.sqdata.sq_harddeps_rev[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                    self.sq_harddep_deferred.remove(dep)
+
         self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
         self.holdoff_need_update = True