summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-19 15:19:39 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-09-30 17:23:35 +0100
commitd0b7471ba89e6c7c1c00019bfe6c6c0aea94dd5a (patch)
treeb028f021428dc07c70bb97dade3c52190b11ca1d /bitbake/lib/bb
parentb83d5c15adcc2d106194d85dd1b0496678c675e1 (diff)
downloadpoky-d0b7471ba89e6c7c1c00019bfe6c6c0aea94dd5a.tar.gz
bitbake: runqueue: Fix task migration problems
Tasks were not migrating consistently, particularly: * if a task was rehashed which had already run * if a task which was valid became invalid due to a rehash We need to always run the migration code for rehashed tasks and then reprocess them for hash validity. This means rearranging the code. It also means several tests are no longer correct and can't be written correctly to work on all possible workflows so those are removed. (Bitbake rev: 8443989ee41e9b162972935513e437b5c66ea74d) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/runqueue.py99
-rw-r--r--bitbake/lib/bb/tests/runqueue.py89
2 files changed, 55 insertions, 133 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index d9a67a3167..02160ef4d7 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -73,7 +73,7 @@ def build_tid(mc, fn, taskname):
73def pending_hash_index(tid, rqdata): 73def pending_hash_index(tid, rqdata):
74 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) 74 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
75 pn = rqdata.dataCaches[mc].pkg_fn[taskfn] 75 pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
76 h = rqdata.runtaskentries[tid].hash 76 h = rqdata.runtaskentries[tid].unihash
77 return pn + ":" + "taskname" + h 77 return pn + ":" + "taskname" + h
78 78
79class RunQueueStats: 79class RunQueueStats:
@@ -2299,9 +2299,6 @@ class RunQueueExecute:
2299 for tid in changed: 2299 for tid in changed:
2300 if tid not in self.rqdata.runq_setscene_tids: 2300 if tid not in self.rqdata.runq_setscene_tids:
2301 continue 2301 continue
2302 valid = self.rq.validate_hashes(set([tid]), self.cooker.data, None, False)
2303 if not valid:
2304 continue
2305 if tid in self.runq_running: 2302 if tid in self.runq_running:
2306 continue 2303 continue
2307 if tid not in self.pending_migrations: 2304 if tid not in self.pending_migrations:
@@ -2358,6 +2355,7 @@ class RunQueueExecute:
2358 2355
2359 logger.info("Setscene task %s now valid and being rerun" % tid) 2356 logger.info("Setscene task %s now valid and being rerun" % tid)
2360 self.sqdone = False 2357 self.sqdone = False
2358 update_scenequeue_data([tid], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
2361 2359
2362 if changed: 2360 if changed:
2363 self.holdoff_need_update = True 2361 self.holdoff_need_update = True
@@ -2674,64 +2672,77 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
2674 2672
2675 rqdata.init_progress_reporter.next_stage() 2673 rqdata.init_progress_reporter.next_stage()
2676 2674
2677 multiconfigs = set() 2675 sqdata.multiconfigs = set()
2678 for tid in sqdata.sq_revdeps: 2676 for tid in sqdata.sq_revdeps:
2679 multiconfigs.add(mc_from_tid(tid)) 2677 sqdata.multiconfigs.add(mc_from_tid(tid))
2680 if len(sqdata.sq_revdeps[tid]) == 0: 2678 if len(sqdata.sq_revdeps[tid]) == 0:
2681 sqrq.sq_buildable.add(tid) 2679 sqrq.sq_buildable.add(tid)
2682 2680
2683 rqdata.init_progress_reporter.finish() 2681 rqdata.init_progress_reporter.finish()
2684 2682
2685 if rq.hashvalidate: 2683 sqdata.noexec = set()
2686 noexec = [] 2684 sqdata.stamppresent = set()
2687 stamppresent = [] 2685 sqdata.valid = set()
2688 tocheck = set()
2689 2686
2690 for tid in sorted(sqdata.sq_revdeps): 2687 update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq)
2691 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
2692 2688
2693 taskdep = rqdata.dataCaches[mc].task_deps[taskfn] 2689def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq):
2694 2690
2695 if 'noexec' in taskdep and taskname in taskdep['noexec']: 2691 tocheck = set()
2696 noexec.append(tid)
2697 sqrq.sq_task_skip(tid)
2698 bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
2699 continue
2700 2692
2701 if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache): 2693 for tid in sorted(tids):
2702 logger.debug(2, 'Setscene stamp current for task %s', tid) 2694 if tid in sqdata.stamppresent:
2703 stamppresent.append(tid) 2695 sqdata.stamppresent.remove(tid)
2704 sqrq.sq_task_skip(tid) 2696 if tid in sqdata.valid:
2705 continue 2697 sqdata.valid.remove(tid)
2706 2698
2707 if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache): 2699 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
2708 logger.debug(2, 'Normal stamp current for task %s', tid)
2709 stamppresent.append(tid)
2710 sqrq.sq_task_skip(tid)
2711 continue
2712 2700
2713 tocheck.add(tid) 2701 taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
2714 2702
2715 valid = rq.validate_hashes(tocheck, cooker.data, len(stamppresent), False) 2703 if 'noexec' in taskdep and taskname in taskdep['noexec']:
2704 sqdata.noexec.add(tid)
2705 sqrq.sq_task_skip(tid)
2706 bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
2707 continue
2708
2709 if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
2710 logger.debug(2, 'Setscene stamp current for task %s', tid)
2711 sqdata.stamppresent.add(tid)
2712 sqrq.sq_task_skip(tid)
2713 continue
2716 2714
2717 valid_new = stamppresent 2715 if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
2718 for v in valid: 2716 logger.debug(2, 'Normal stamp current for task %s', tid)
2719 valid_new.append(v) 2717 sqdata.stamppresent.add(tid)
2718 sqrq.sq_task_skip(tid)
2719 continue
2720 2720
2721 hashes = {} 2721 tocheck.add(tid)
2722 for mc in sorted(multiconfigs): 2722
2723 for tid in sorted(sqdata.sq_revdeps): 2723 sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False)
2724
2725 sqdata.hashes = {}
2726 for mc in sorted(sqdata.multiconfigs):
2727 for tid in sorted(sqdata.sq_revdeps):
2724 if mc_from_tid(tid) != mc: 2728 if mc_from_tid(tid) != mc:
2725 continue 2729 continue
2726 if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered: 2730 if tid in sqdata.stamppresent:
2727 sqdata.outrightfail.add(tid) 2731 continue
2732 if tid in sqdata.valid:
2733 continue
2734 if tid in sqdata.noexec:
2735 continue
2736 if tid in sqrq.scenequeue_notcovered:
2737 continue
2738 sqdata.outrightfail.add(tid)
2728 2739
2729 h = pending_hash_index(tid, rqdata) 2740 h = pending_hash_index(tid, rqdata)
2730 if h not in hashes: 2741 if h not in sqdata.hashes:
2731 hashes[h] = tid 2742 sqdata.hashes[h] = tid
2732 else: 2743 else:
2733 sqrq.sq_deferred[tid] = hashes[h] 2744 sqrq.sq_deferred[tid] = sqdata.hashes[h]
2734 bb.warn("Deferring %s after %s" % (tid, hashes[h])) 2745 bb.warn("Deferring %s after %s" % (tid, sqdata.hashes[h]))
2735 2746
2736 2747
2737class TaskFailure(Exception): 2748class TaskFailure(Exception):
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py
index cb4d526f13..01b992c47c 100644
--- a/bitbake/lib/bb/tests/runqueue.py
+++ b/bitbake/lib/bb/tests/runqueue.py
@@ -312,92 +312,3 @@ class RunQueueTests(unittest.TestCase):
312 else: 312 else:
313 self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i) 313 self.assertEqual(tasks.count(i), 1, "%s not in task list once" % i)
314 314
315 @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
316 def test_hashserv_partial_match(self):
317 # e1:do_package matches initial built but not second hash value
318 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
319 extraenv = {
320 "BB_HASHSERVE" : "auto",
321 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
322 }
323 cmd = ["bitbake", "a1", "b1"]
324 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
325 'populate_sysroot_setscene', 'package_qa_setscene']
326 sstatevalid = ""
327 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
328 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
329 self.assertEqual(set(tasks), set(expected))
330 with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
331 f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
332 with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
333 f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
334 cmd = ["bitbake", "e1"]
335 sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d"
336 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
337 expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
338 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
339 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
340 'e1:package_setscene'] + ['e1:' + x for x in self.alltasks]
341 expected.remove('e1:package')
342 self.assertEqual(set(tasks), set(expected))
343
344 @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
345 def test_hashserv_partial_match2(self):
346 # e1:do_package + e1:do_populate_sysroot matches initial built but not second hash value
347 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
348 extraenv = {
349 "BB_HASHSERVE" : "auto",
350 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
351 }
352 cmd = ["bitbake", "a1", "b1"]
353 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
354 'populate_sysroot_setscene', 'package_qa_setscene']
355 sstatevalid = ""
356 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
357 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
358 self.assertEqual(set(tasks), set(expected))
359 with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
360 f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
361 with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
362 f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
363 cmd = ["bitbake", "e1"]
364 sstatevalid = "e1:do_package:685e69a026b2f029483fdefe6a11e1e06641dd2a0f6f86e27b9b550f8f21229d e1:do_populate_sysroot:ef7dc0e2dd55d0534e75cba50731ff42f949818b6f29a65d72bc05856e56711d"
365 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
366 expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
367 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
368 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
369 'e1:package_setscene', 'e1:populate_sysroot_setscene', 'e1:build', 'e1:package_qa', 'e1:package_write_rpm', 'e1:package_write_ipk', 'e1:packagedata']
370 self.assertEqual(set(tasks), set(expected))
371
372 @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
373 def test_hashserv_partial_match3(self):
374 # e1:do_package is valid for a1 but not after b1
375 # In former buggy code, this triggered e1:do_fetch, then e1:do_populate_sysroot to run
376 # with none of the intermediate tasks which is a serious bug
377 with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
378 extraenv = {
379 "BB_HASHSERVE" : "auto",
380 "BB_SIGNATURE_HANDLER" : "TestEquivHash"
381 }
382 cmd = ["bitbake", "a1", "b1"]
383 setscenetasks = ['package_write_ipk_setscene', 'package_write_rpm_setscene', 'packagedata_setscene',
384 'populate_sysroot_setscene', 'package_qa_setscene']
385 sstatevalid = ""
386 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True)
387 expected = ['a1:' + x for x in self.alltasks] + ['b1:' + x for x in self.alltasks]
388 self.assertEqual(set(tasks), set(expected))
389 with open(tempdir + "/stamps/a1.do_install.taint", "w") as f:
390 f.write("d460a29e-903f-4b76-a96b-3bcc22a65994")
391 with open(tempdir + "/stamps/b1.do_install.taint", "w") as f:
392 f.write("ed36d46a-2977-458a-b3de-eef885bc1817")
393 cmd = ["bitbake", "e1", "-DD"]
394 sstatevalid = "e1:do_package:af056eae12a733a6a8c4f4da8c6757e588e13565852c94e2aad4d953a3989c13 e1:do_package:a3677703db82b22d28d57c1820a47851dd780104580863f5bd32e66e003a779d"
395 tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv, cleanup=True, slowtasks="e1:fetch b1:install")
396 expected = ['a1:package', 'a1:install', 'b1:package', 'b1:install', 'a1:populate_sysroot', 'b1:populate_sysroot',
397 'a1:package_write_ipk_setscene', 'b1:packagedata_setscene', 'b1:package_write_rpm_setscene',
398 'a1:package_write_rpm_setscene', 'b1:package_write_ipk_setscene', 'a1:packagedata_setscene',
399 'e1:package_setscene'] + ['e1:' + x for x in self.alltasks]
400 expected.remove('e1:package')
401 self.assertEqual(set(tasks), set(expected))
402
403