2 files changed, 55 insertions, 133 deletions

diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index d9a67a3167..02160ef4d7 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -73,7 +73,7 @@ def build_tid(mc, fn, taskname):
 def pending_hash_index(tid, rqdata):
     (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
     pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
-    h = rqdata.runtaskentries[tid].hash
+    h = rqdata.runtaskentries[tid].unihash
     return pn + ":" + "taskname" + h
 
 class RunQueueStats:
@@ -2299,9 +2299,6 @@ class RunQueueExecute:
         for tid in changed:
             if tid not in self.rqdata.runq_setscene_tids:
                 continue
-            valid = self.rq.validate_hashes(set([tid]), self.cooker.data, None, False)
-            if not valid:
-                continue
             if tid in self.runq_running:
                 continue
             if tid not in self.pending_migrations:
@@ -2358,6 +2355,7 @@ class RunQueueExecute:
 
             logger.info("Setscene task %s now valid and being rerun" % tid)
             self.sqdone = False
+            update_scenequeue_data([tid], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
 
         if changed:
             self.holdoff_need_update = True
@@ -2674,64 +2672,77 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
 
     rqdata.init_progress_reporter.next_stage()
 
-    multiconfigs = set()
+    sqdata.multiconfigs = set()
     for tid in sqdata.sq_revdeps:
-        multiconfigs.add(mc_from_tid(tid))
+        sqdata.multiconfigs.add(mc_from_tid(tid))
         if len(sqdata.sq_revdeps[tid]) == 0:
             sqrq.sq_buildable.add(tid)
 
     rqdata.init_progress_reporter.finish()
 
-    if rq.hashvalidate:
-        noexec = []
-        stamppresent = []
-        tocheck = set()
+    sqdata.noexec = set()
+    sqdata.stamppresent = set()
+    sqdata.valid = set()
 
-        for tid in sorted(sqdata.sq_revdeps):
-            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+    update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq)
 
-            taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
+def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq):
 
-            if 'noexec' in taskdep and taskname in taskdep['noexec']:
-                noexec.append(tid)
-                sqrq.sq_task_skip(tid)
-                bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
-                continue
+    tocheck = set()
 
-            if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
-                logger.debug(2, 'Setscene stamp current for task %s', tid)
-                stamppresent.append(tid)
-                sqrq.sq_task_skip(tid)
-                continue
+    for tid in sorted(tids):
+        if tid in sqdata.stamppresent:
+            sqdata.stamppresent.remove(tid)
+        if tid in sqdata.valid:
+            sqdata.valid.remove(tid)
 
-            if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
-                logger.debug(2, 'Normal stamp current for task %s', tid)
-                stamppresent.append(tid)
-                sqrq.sq_task_skip(tid)
-                continue
+        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
 
-            tocheck.add(tid)
+        taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
 
-        valid = rq.validate_hashes(tocheck, cooker.data, len(stamppresent), False)
+        if 'noexec' in taskdep and taskname in taskdep['noexec']:
+            sqdata.noexec.add(tid)
+            sqrq.sq_task_skip(tid)
+            bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
+            continue
+
+        if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
+            logger.debug(2, 'Setscene stamp current for task %s', tid)
+            sqdata.stamppresent.add(tid)
+            sqrq.sq_task_skip(tid)
+            continue
 
-        valid_new = stamppresent
-        for v in valid:
-            valid_new.append(v)
+        if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
+            logger.debug(2, 'Normal stamp current for task %s', tid)
+            sqdata.stamppresent.add(tid)
+            sqrq.sq_task_skip(tid)
+            continue
 
-        hashes = {}
-        for mc in sorted(multiconfigs):
-          for tid in sorted(sqdata.sq_revdeps):
+        tocheck.add(tid)
+
+    sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False)
+
+    sqdata.hashes = {}
+    for mc in sorted(sqdata.multiconfigs):
+        for tid in sorted(sqdata.sq_revdeps):
             if mc_from_tid(tid) != mc:
                 continue
-            if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered:
-                sqdata.outrightfail.add(tid)
+            if tid in sqdata.stamppresent:
+                continue
+            if tid in sqdata.valid:
+                continue
+            if tid in sqdata.noexec:
+                continue
+            if tid in sqrq.scenequeue_notcovered:
+                continue
+            sqdata.outrightfail.add(tid)
 
-                h = pending_hash_index(tid, rqdata)
-                if h not in hashes:
-                    hashes[h] = tid
-                else:
-                    sqrq.sq_deferred[tid] = hashes[h]
-                    bb.warn("Deferring %s after %s" % (tid, hashes[h]))
+            h = pending_hash_index(tid, rqdata)
+            if h not in sqdata.hashes:
+                sqdata.hashes[h] = tid
+            else:
+                sqrq.sq_deferred[tid] = sqdata.hashes[h]
+                bb.warn("Deferring %s after %s" % (tid, sqdata.hashes[h]))
 
 
 class TaskFailure(Exception):
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py
index cb4d526f13..01b992c47c 100644
--- a/bitbake/lib/bb/tests/runqueue.py
+++ b/bitbake/lib/bb/tests/runqueue.py
@@ -312,92 +312,3 @@ class RunQueueTests(unittest.TestCase):
                 else:
                     self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i)
 
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
-    def test_hashserv_partial_match(self):
-        # e1:do_package matches initial built but not second hash value
-        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
-            extraenv = {
-                "BB_HASHSERVE" : "auto",
-                "BB_SIGNATURE_HANDLER" : "TestEquivHash"
-            }
-            cmd = ["bitbake", "a1", "b1"]
-            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
-                             'populate_sysroot_setscene', 'package_qa_setscene']
-            sstatevalid = ""
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
-            self.assertEqual(set(tasks), set(expected))
-            with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
-               f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
-            with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
-               f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
-            cmd = ["bitbake", "e1"]
-            sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d"
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
-                        'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
-                        'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
-                        'e1:package_setscene'] + ['e1:' + x for x in self.alltasks]
-            expected.remove('e1:package')
-            self.assertEqual(set(tasks), set(expected))
-
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
-    def test_hashserv_partial_match2(self):
-        # e1:do_package + e1:do_populate_sysroot matches initial built but not second hash value
-        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
-            extraenv = {
-                "BB_HASHSERVE" : "auto",
-                "BB_SIGNATURE_HANDLER" : "TestEquivHash"
-            }
-            cmd = ["bitbake", "a1", "b1"]
-            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
-                             'populate_sysroot_setscene', 'package_qa_setscene']
-            sstatevalid = ""
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
-            self.assertEqual(set(tasks), set(expected))
-            with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
-               f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
-            with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
-               f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
-            cmd = ["bitbake", "e1"]
-            sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d e1:do_populate_sysroot:ef7dc0e2dd55d0534e75cba50731ff42f949818b6f29a65d72bc05856e56711d"
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
-                        'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
-                        'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
-                        'e1:package_setscene', 'e1:populate_sysroot_setscene', 'e1:build', 'e1:package_qa', 'e1:package_write_rpm', 'e1:package_write_ipk', 'e1:packagedata']
-            self.assertEqual(set(tasks), set(expected))
-
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
-    def test_hashserv_partial_match3(self):
-        # e1:do_package is valid for a1 but not after b1
-        # In former buggy code, this triggered e1:do_fetch, then e1:do_populate_sysroot to run
-        # with none of the intermediate tasks which is a serious bug
-        with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
-            extraenv = {
-                "BB_HASHSERVE" : "auto",
-                "BB_SIGNATURE_HANDLER" : "TestEquivHash"
-            }
-            cmd = ["bitbake", "a1", "b1"]
-            setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
-                             'populate_sysroot_setscene', 'package_qa_setscene']
-            sstatevalid = ""
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
-            expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
-            self.assertEqual(set(tasks), set(expected))
-            with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
-               f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
-            with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
-               f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
-            cmd = ["bitbake", "e1", "-DD"]
-            sstatevalid = "e1:do_package:af056eae12a733a6a8c4f4da8c6757e588e13565852c94e2aad4d953a3989c13 e1:do_package:a3677703db82b22d28d57c1820a47851dd780104580863f5bd32e66e003a779d"
-            tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="e1:fetch b1:install")
-            expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
-                        'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
-                        'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
-                        'e1:package_setscene']  + ['e1:' + x for x in self.alltasks]
-            expected.remove('e1:package')
-            self.assertEqual(set(tasks), set(expected))
-
-