1 files changed, 195 insertions, 124 deletions

diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index bc7e18175d..63d4edd892 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -10,10 +10,12 @@ Handles preparation and execution of a queue of tasks
 #
 
 import copy
+import enum
 import os
 import sys
 import stat
 import errno
+import itertools
 import logging
 import re
 import bb
@@ -124,14 +126,16 @@ class RunQueueStats:
     def updateActiveSetscene(self, active):
         self.setscene_active = active
 
-# These values indicate the next step due to be run in the
-# runQueue state machine
-runQueuePrepare = 2
-runQueueSceneInit = 3
-runQueueRunning = 6
-runQueueFailed = 7
-runQueueCleanUp = 8
-runQueueComplete = 9
+
+# Indicates the next step due to run in the runQueue state machine
+class RunQueueState(enum.Enum):
+    PREPARE = 0
+    SCENE_INIT = 1
+    DUMP_SIGS = 2
+    RUNNING = 3
+    FAILED = 4
+    CLEAN_UP = 5
+    COMPLETE = 6
 
 class RunQueueScheduler(object):
     """
@@ -475,7 +479,6 @@ class RunQueueData:
         self.runtaskentries = {}
 
     def runq_depends_names(self, ids):
-        import re
         ret = []
         for id in ids:
             nam = os.path.basename(id)
@@ -677,7 +680,7 @@ class RunQueueData:
 
         self.init_progress_reporter.start()
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Step A - Work out a list of tasks to run
         #
@@ -728,6 +731,8 @@ class RunQueueData:
                 if mc == frommc:
                     fn = taskData[mcdep].build_targets[pn][0]
                     newdep = '%s:%s' % (fn,deptask)
+                    if newdep not in taskData[mcdep].taskentries:
+                        bb.fatal("Task mcdepends on non-existent task %s" % (newdep))
                     taskData[mc].taskentries[tid].tdepends.append(newdep)
 
         for mc in taskData:
@@ -826,7 +831,7 @@ class RunQueueData:
         #self.dump_data()
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Resolve recursive 'recrdeptask' dependencies (Part B)
         #
@@ -923,7 +928,7 @@ class RunQueueData:
             self.runtaskentries[tid].depends.difference_update(recursivetasksselfref)
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         #self.dump_data()
 
@@ -1005,7 +1010,7 @@ class RunQueueData:
                 mark_active(tid, 1)
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Step C - Prune all inactive tasks
         #
@@ -1052,7 +1057,7 @@ class RunQueueData:
                 bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets)))
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Handle runonly
         if self.cooker.configuration.runonly:
@@ -1093,7 +1098,7 @@ class RunQueueData:
         logger.verbose("Assign Weightings")
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Generate a list of reverse dependencies to ease future calculations
         for tid in self.runtaskentries:
@@ -1101,7 +1106,7 @@ class RunQueueData:
                 self.runtaskentries[dep].revdeps.add(tid)
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Identify tasks at the end of dependency chains
         # Error on circular dependency loops (length two)
@@ -1118,14 +1123,14 @@ class RunQueueData:
         logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Calculate task weights
         # Check of higher length circular dependencies
         self.runq_weight = self.calculate_task_weights(endpoints)
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Sanity Check - Check for multiple tasks building the same provider
         for mc in self.dataCaches:
@@ -1226,7 +1231,7 @@ class RunQueueData:
 
         self.init_progress_reporter.next_stage()
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Iterate over the task list looking for tasks with a 'setscene' function
         self.runq_setscene_tids = set()
@@ -1239,7 +1244,7 @@ class RunQueueData:
                 self.runq_setscene_tids.add(tid)
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Invalidate task if force mode active
         if self.cooker.configuration.force:
@@ -1256,7 +1261,7 @@ class RunQueueData:
                     invalidate_task(fn + ":" + st, True)
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         # Create and print to the logs a virtual/xxxx -> PN (fn) table
         for mc in taskData:
@@ -1269,31 +1274,45 @@ class RunQueueData:
                 bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCaches[mc])
 
         self.init_progress_reporter.next_stage()
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
         bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
 
+        starttime = time.time()
+        lasttime = starttime
+
         # Iterate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(self.runtaskentries)
         while todeal:
+            ready = set()
             for tid in todeal.copy():
                 if not (self.runtaskentries[tid].depends - dealtwith):
-                    dealtwith.add(tid)
-                    todeal.remove(tid)
-                    self.prepare_task_hash(tid)
-                bb.event.check_for_interrupts(self.cooker.data)
+                    self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
+                    # get_taskhash for a given tid *must* be called before get_unihash* below
+                    self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
+                    ready.add(tid)
+            unihashes = bb.parse.siggen.get_unihashes(ready)
+            for tid in ready:
+                dealtwith.add(tid)
+                todeal.remove(tid)
+                self.runtaskentries[tid].unihash = unihashes[tid]
+
+            bb.event.check_for_interrupts()
+
+            if time.time() > (lasttime + 30):
+                lasttime = time.time()
+                hashequiv_logger.verbose("Initial setup loop progress: %s of %s in %s" % (len(todeal), len(self.runtaskentries), lasttime - starttime))
+
+        endtime = time.time()
+        if (endtime-starttime > 60):
+            hashequiv_logger.verbose("Initial setup loop took: %s" % (endtime-starttime))
 
         bb.parse.siggen.writeout_file_checksum_cache()
 
         #self.dump_data()
         return len(self.runtaskentries)
 
-    def prepare_task_hash(self, tid):
-        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
-        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
-        self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
-
     def dump_data(self):
         """
         Dump some debug information on the internal data structures
@@ -1320,13 +1339,13 @@ class RunQueue:
         self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION") or None
         self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID") or None
 
-        self.state = runQueuePrepare
+        self.state = RunQueueState.PREPARE
 
         # For disk space monitor
         # Invoked at regular time intervals via the bitbake heartbeat event
         # while the build is running. We generate a unique name for the handler
         # here, just in case that there ever is more than one RunQueue instance,
-        # start the handler when reaching runQueueSceneInit, and stop it when
+        # start the handler when reaching RunQueueState.SCENE_INIT, and stop it when
         # done with the build.
         self.dm = monitordisk.diskMonitor(cfgData)
         self.dm_event_handler_name = '_bb_diskmonitor_' + str(id(self))
@@ -1538,9 +1557,9 @@ class RunQueue:
         """
 
         retval = True
-        bb.event.check_for_interrupts(self.cooker.data)
+        bb.event.check_for_interrupts()
 
-        if self.state is runQueuePrepare:
+        if self.state == RunQueueState.PREPARE:
             # NOTE: if you add, remove or significantly refactor the stages of this
             # process then you should recalculate the weightings here. This is quite
             # easy to do - just change the next line temporarily to pass debug=True as
@@ -1551,12 +1570,12 @@ class RunQueue:
                                                             "Initialising tasks",
                                                             [43, 967, 4, 3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
             if self.rqdata.prepare() == 0:
-                self.state = runQueueComplete
+                self.state = RunQueueState.COMPLETE
             else:
-                self.state = runQueueSceneInit
+                self.state = RunQueueState.SCENE_INIT
                 bb.parse.siggen.save_unitaskhashes()
 
-        if self.state is runQueueSceneInit:
+        if self.state == RunQueueState.SCENE_INIT:
             self.rqdata.init_progress_reporter.next_stage()
 
             # we are ready to run,  emit dependency info to any UI or class which
@@ -1567,24 +1586,29 @@ class RunQueue:
 
             if not self.dm_event_handler_registered:
                  res = bb.event.register(self.dm_event_handler_name,
-                                         lambda x, y: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
+                                         lambda x, y: self.dm.check(self) if self.state in [RunQueueState.RUNNING, RunQueueState.CLEAN_UP] else False,
                                          ('bb.event.HeartbeatEvent',), data=self.cfgData)
                  self.dm_event_handler_registered = True
 
             self.rqdata.init_progress_reporter.next_stage()
             self.rqexe = RunQueueExecute(self)
 
-            dump = self.cooker.configuration.dump_signatures
-            if dump:
+            dumpsigs = self.cooker.configuration.dump_signatures
+            if dumpsigs:
                 self.rqdata.init_progress_reporter.finish()
-                if 'printdiff' in dump:
-                    invalidtasks = self.print_diffscenetasks()
-                self.dump_signatures(dump)
-                if 'printdiff' in dump:
-                    self.write_diffscenetasks(invalidtasks)
-                self.state = runQueueComplete
-
-        if self.state is runQueueSceneInit:
+                if 'printdiff' in dumpsigs:
+                    self.invalidtasks_dump = self.print_diffscenetasks()
+                self.state = RunQueueState.DUMP_SIGS
+
+        if self.state == RunQueueState.DUMP_SIGS:
+            dumpsigs = self.cooker.configuration.dump_signatures
+            retval = self.dump_signatures(dumpsigs)
+            if retval is False:
+                if 'printdiff' in dumpsigs:
+                    self.write_diffscenetasks(self.invalidtasks_dump)
+                self.state = RunQueueState.COMPLETE
+
+        if self.state == RunQueueState.SCENE_INIT:
             self.start_worker(self.rqexe)
             self.rqdata.init_progress_reporter.finish()
 
@@ -1597,15 +1621,15 @@ class RunQueue:
                     self.rqexe.tasks_notcovered.add(tid)
                 self.rqexe.sqdone = True
             logger.info('Executing Tasks')
-            self.state = runQueueRunning
+            self.state = RunQueueState.RUNNING
 
-        if self.state is runQueueRunning:
+        if self.state == RunQueueState.RUNNING:
             retval = self.rqexe.execute()
 
-        if self.state is runQueueCleanUp:
+        if self.state == RunQueueState.CLEAN_UP:
             retval = self.rqexe.finish()
 
-        build_done = self.state is runQueueComplete or self.state is runQueueFailed
+        build_done = self.state in [RunQueueState.COMPLETE, RunQueueState.FAILED]
 
         if build_done and self.dm_event_handler_registered:
             bb.event.remove(self.dm_event_handler_name, None, data=self.cfgData)
@@ -1621,10 +1645,10 @@ class RunQueue:
                     # Let's avoid the word "failed" if nothing actually did
                     logger.info("Tasks Summary: Attempted %d tasks of which %d didn't need to be rerun and all succeeded.", self.rqexe.stats.completed, self.rqexe.stats.skipped)
 
-        if self.state is runQueueFailed:
+        if self.state == RunQueueState.FAILED:
             raise bb.runqueue.TaskFailure(self.rqexe.failed_tids)
 
-        if self.state is runQueueComplete:
+        if self.state == RunQueueState.COMPLETE:
             # All done
             return False
 
@@ -1644,7 +1668,7 @@ class RunQueue:
                 self.teardown_workers()
             except:
                 pass
-            self.state = runQueueComplete
+            self.state = RunQueueState.COMPLETE
             raise
         except Exception as err:
             logger.exception("An uncaught exception occurred in runqueue")
@@ -1652,12 +1676,12 @@ class RunQueue:
                 self.teardown_workers()
             except:
                 pass
-            self.state = runQueueComplete
+            self.state = RunQueueState.COMPLETE
             raise
 
     def finish_runqueue(self, now = False):
         if not self.rqexe:
-            self.state = runQueueComplete
+            self.state = RunQueueState.COMPLETE
             return
 
         if now:
@@ -1672,33 +1696,42 @@ class RunQueue:
             bb.parse.siggen.dump_sigtask(taskfn, taskname, dataCaches[mc].stamp[taskfn], True)
 
     def dump_signatures(self, options):
-        if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset:
-            bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled")
-
-        bb.note("Writing task signature files")
-
-        max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
-        def chunkify(l, n):
-            return [l[i::n] for i in range(n)]
-        tids = chunkify(list(self.rqdata.runtaskentries), max_process)
-        # We cannot use the real multiprocessing.Pool easily due to some local data
-        # that can't be pickled. This is a cheap multi-process solution.
-        launched = []
-        while tids:
-            if len(launched) < max_process:
-                p = Process(target=self._rq_dump_sigtid, args=(tids.pop(), ))
+        if not hasattr(self, "dumpsigs_launched"):
+            if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset:
+                bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled")
+
+            bb.note("Writing task signature files")
+
+            max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
+            def chunkify(l, n):
+                return [l[i::n] for i in range(n)]
+            dumpsigs_tids = chunkify(list(self.rqdata.runtaskentries), max_process)
+
+            # We cannot use the real multiprocessing.Pool easily due to some local data
+            # that can't be pickled. This is a cheap multi-process solution.
+            self.dumpsigs_launched = []
+
+            for tids in dumpsigs_tids:
+                p = Process(target=self._rq_dump_sigtid, args=(tids, ))
                 p.start()
-                launched.append(p)
-            for q in launched:
-                # The finished processes are joined when calling is_alive()
-                if not q.is_alive():
-                    launched.remove(q)
-        for p in launched:
+                self.dumpsigs_launched.append(p)
+
+            return 1.0
+
+        for q in self.dumpsigs_launched:
+            # The finished processes are joined when calling is_alive()
+            if not q.is_alive():
+                self.dumpsigs_launched.remove(q)
+
+        if self.dumpsigs_launched:
+            return 1.0
+
+        for p in self.dumpsigs_launched:
                 p.join()
 
         bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options)
 
-        return
+        return False
 
     def print_diffscenetasks(self):
         def get_root_invalid_tasks(task, taskdepends, valid, noexec, visited_invalid):
@@ -1971,14 +2004,14 @@ class RunQueueExecute:
                 pass
 
         if self.failed_tids:
-            self.rq.state = runQueueFailed
+            self.rq.state = RunQueueState.FAILED
             return
 
-        self.rq.state = runQueueComplete
+        self.rq.state = RunQueueState.COMPLETE
         return
 
     def finish(self):
-        self.rq.state = runQueueCleanUp
+        self.rq.state = RunQueueState.CLEAN_UP
 
         active = self.stats.active + len(self.sq_live)
         if active > 0:
@@ -1987,10 +2020,10 @@ class RunQueueExecute:
             return self.rq.active_fds()
 
         if self.failed_tids:
-            self.rq.state = runQueueFailed
+            self.rq.state = RunQueueState.FAILED
             return True
 
-        self.rq.state = runQueueComplete
+        self.rq.state = RunQueueState.COMPLETE
         return True
 
     # Used by setscene only
@@ -2109,7 +2142,7 @@ class RunQueueExecute:
         bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=("".join(fakeroot_log) or None)), self.cfgData)
 
         if self.rqdata.taskData[''].halt:
-            self.rq.state = runQueueCleanUp
+            self.rq.state = RunQueueState.CLEAN_UP
 
     def task_skip(self, task, reason):
         self.runq_running.add(task)
@@ -2175,12 +2208,20 @@ class RunQueueExecute:
         if not hasattr(self, "sorted_setscene_tids"):
             # Don't want to sort this set every execution
             self.sorted_setscene_tids = sorted(self.rqdata.runq_setscene_tids)
+            # Resume looping where we left off when we returned to feed the mainloop
+            self.setscene_tids_generator = itertools.cycle(self.rqdata.runq_setscene_tids)
 
         task = None
         if not self.sqdone and self.can_start_task():
-            # Find the next setscene to run
-            for nexttask in self.sorted_setscene_tids:
+            loopcount = 0
+            # Find the next setscene to run, exit the loop when we've processed all tids or found something to execute
+            while loopcount < len(self.rqdata.runq_setscene_tids):
+                loopcount += 1
+                nexttask = next(self.setscene_tids_generator)
                 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred:
+                    if nexttask in self.sq_deferred and self.sq_deferred[nexttask] not in self.runq_complete:
+                        # Skip deferred tasks quickly before the 'expensive' tests below - this is key to performant multiconfig builds
+                        continue
                     if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \
                             nexttask not in self.sq_needed_harddeps and \
                             self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \
@@ -2210,8 +2251,7 @@ class RunQueueExecute:
                         if t in self.runq_running and t not in self.runq_complete:
                             continue
                     if nexttask in self.sq_deferred:
-                        if self.sq_deferred[nexttask] not in self.runq_complete:
-                            continue
+                        # Deferred tasks that were still deferred were skipped above so we now need to process
                         logger.debug("Task %s no longer deferred" % nexttask)
                         del self.sq_deferred[nexttask]
                         valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False)
@@ -2296,17 +2336,17 @@ class RunQueueExecute:
 
             err = self.summarise_scenequeue_errors()
             if err:
-                self.rq.state = runQueueFailed
+                self.rq.state = RunQueueState.FAILED
                 return True
 
             if self.cooker.configuration.setsceneonly:
-                self.rq.state = runQueueComplete
+                self.rq.state = RunQueueState.COMPLETE
                 return True
             self.sqdone = True
 
             if self.stats.total == 0:
                 # nothing to do
-                self.rq.state = runQueueComplete
+                self.rq.state = RunQueueState.COMPLETE
                 return True
 
         if self.cooker.configuration.setsceneonly:
@@ -2373,7 +2413,7 @@ class RunQueueExecute:
                         self.rq.start_fakeworker(self, mc)
                     except OSError as exc:
                         logger.critical("Failed to spawn fakeroot worker to run %s: %s" % (task, str(exc)))
-                        self.rq.state = runQueueFailed
+                        self.rq.state = RunQueueState.FAILED
                         self.stats.taskFailed()
                         return True
                 RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, runtask, "runtask")
@@ -2401,7 +2441,7 @@ class RunQueueExecute:
             return True
 
         if self.failed_tids:
-            self.rq.state = runQueueFailed
+            self.rq.state = RunQueueState.FAILED
             return True
 
         # Sanity Checks
@@ -2418,9 +2458,9 @@ class RunQueueExecute:
                 err = True
 
         if err:
-            self.rq.state = runQueueFailed
+            self.rq.state = RunQueueState.FAILED
         else:
-            self.rq.state = runQueueComplete
+            self.rq.state = RunQueueState.COMPLETE
 
         return True
 
@@ -2438,14 +2478,17 @@ class RunQueueExecute:
         taskdepdata_cache = {}
         for task in self.rqdata.runtaskentries:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
-            pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-            deps = self.rqdata.runtaskentries[task].depends
-            provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-            taskhash = self.rqdata.runtaskentries[task].hash
-            unihash = self.rqdata.runtaskentries[task].unihash
-            deps = self.filtermcdeps(task, mc, deps)
-            hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn]
-            taskdepdata_cache[task] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn]
+            taskdepdata_cache[task] = bb.TaskData(
+                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn],
+                taskname = taskname,
+                fn = fn,
+                deps = self.filtermcdeps(task, mc, self.rqdata.runtaskentries[task].depends),
+                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn],
+                taskhash = self.rqdata.runtaskentries[task].hash,
+                unihash = self.rqdata.runtaskentries[task].unihash,
+                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn],
+                taskhash_deps = self.rqdata.runtaskentries[task].taskhash_deps,
+            )
 
         self.taskdepdata_cache = taskdepdata_cache
 
@@ -2460,9 +2503,11 @@ class RunQueueExecute:
         while next:
             additional = []
             for revdep in next:
-                self.taskdepdata_cache[revdep][6] = self.rqdata.runtaskentries[revdep].unihash
+                self.taskdepdata_cache[revdep] = self.taskdepdata_cache[revdep]._replace(
+                    unihash=self.rqdata.runtaskentries[revdep].unihash
+                )
                 taskdepdata[revdep] = self.taskdepdata_cache[revdep]
-                for revdep2 in self.taskdepdata_cache[revdep][3]:
+                for revdep2 in self.taskdepdata_cache[revdep].deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
             next = additional
@@ -2531,9 +2576,6 @@ class RunQueueExecute:
                     self.rqdata.runtaskentries[hashtid].unihash = unihash
                     bb.parse.siggen.set_unihash(hashtid, unihash)
                     toprocess.add(hashtid)
-                if torehash:
-                    # Need to save after set_unihash above
-                    bb.parse.siggen.save_unitaskhashes()
 
         # Work out all tasks which depend upon these
         total = set()
@@ -2556,17 +2598,28 @@ class RunQueueExecute:
             elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
                 next.add(p)
 
+        starttime = time.time()
+        lasttime = starttime
+
         # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
         while next:
             current = next.copy()
             next = set()
+            ready = {}
             for tid in current:
                 if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
                     continue
+                # get_taskhash for a given tid *must* be called before get_unihash* below
+                ready[tid] = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
+
+            unihashes = bb.parse.siggen.get_unihashes(ready.keys())
+
+            for tid in ready:
                 orighash = self.rqdata.runtaskentries[tid].hash
-                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
+                newhash = ready[tid]
                 origuni = self.rqdata.runtaskentries[tid].unihash
-                newuni = bb.parse.siggen.get_unihash(tid)
+                newuni = unihashes[tid]
+
                 # FIXME, need to check it can come from sstate at all for determinism?
                 remapped = False
                 if newuni == origuni:
@@ -2587,6 +2640,15 @@ class RunQueueExecute:
                 next |= self.rqdata.runtaskentries[tid].revdeps
                 total.remove(tid)
                 next.intersection_update(total)
+                bb.event.check_for_interrupts()
+
+                if time.time() > (lasttime + 30):
+                    lasttime = time.time()
+                    hashequiv_logger.verbose("Rehash loop slow progress: %s in %s" % (len(total), lasttime - starttime))
+
+        endtime = time.time()
+        if (endtime-starttime > 60):
+            hashequiv_logger.verbose("Rehash loop took more than 60s: %s" % (endtime-starttime))
 
         if changed:
             for mc in self.rq.worker:
@@ -2628,7 +2690,7 @@ class RunQueueExecute:
                 if dep in self.runq_complete and dep not in self.runq_tasksrun:
                     bb.error("Task %s marked as completed but now needing to rerun? Halting build." % dep)
                     self.failed_tids.append(tid)
-                    self.rq.state = runQueueCleanUp
+                    self.rq.state = RunQueueState.CLEAN_UP
                     return
 
                 if dep not in self.runq_complete:
@@ -2712,8 +2774,12 @@ class RunQueueExecute:
                 logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
                 self.sq_task_failoutright(dep)
                 continue
+
+        # For performance, only compute allcovered once if needed
+        if self.sqdata.sq_deps[task]:
+            allcovered = self.scenequeue_covered | self.scenequeue_notcovered
         for dep in sorted(self.sqdata.sq_deps[task]):
-            if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+            if self.sqdata.sq_revdeps[dep].issubset(allcovered):
                 if dep not in self.sq_buildable:
                     self.sq_buildable.add(dep)
 
@@ -2759,7 +2825,7 @@ class RunQueueExecute:
             pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
             if not check_setscene_enforce_ignore_tasks(pn, taskname, self.rqdata.setscene_ignore_tasks):
                 logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
-                self.rq.state = runQueueCleanUp
+                self.rq.state = RunQueueState.CLEAN_UP
 
     def sq_task_complete(self, task):
         bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
@@ -2806,13 +2872,19 @@ class RunQueueExecute:
             additional = []
             for revdep in next:
                 (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
-                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
                 deps = getsetscenedeps(revdep)
-                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskhash = self.rqdata.runtaskentries[revdep].hash
-                unihash = self.rqdata.runtaskentries[revdep].unihash
-                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn]
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn]
+
+                taskdepdata[revdep] = bb.TaskData(
+                    pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn],
+                    taskname = taskname,
+                    fn = fn,
+                    deps = deps,
+                    provides = self.rqdata.dataCaches[mc].fn_provides[taskfn],
+                    taskhash = self.rqdata.runtaskentries[revdep].hash,
+                    unihash = self.rqdata.runtaskentries[revdep].unihash,
+                    hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn],
+                    taskhash_deps = self.rqdata.runtaskentries[revdep].taskhash_deps,
+                )
                 for revdep2 in deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
@@ -2964,14 +3036,13 @@ def build_scenequeue_data(sqdata, rqdata, sqrq):
     rqdata.init_progress_reporter.next_stage(len(rqdata.runtaskentries))
 
     # Sanity check all dependencies could be changed to setscene task references
-    for taskcounter, tid in enumerate(rqdata.runtaskentries):
+    for tid in rqdata.runtaskentries:
         if tid in rqdata.runq_setscene_tids:
             pass
         elif sq_revdeps_squash[tid]:
             bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, halting. Please report this problem.")
         else:
             del sq_revdeps_squash[tid]
-        rqdata.init_progress_reporter.update(taskcounter)
 
     rqdata.init_progress_reporter.next_stage()
 
@@ -3261,7 +3332,7 @@ class runQueuePipe():
 
         start = len(self.queue)
         try:
-            self.queue.extend(self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512 * 1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise