summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/runqueue.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-19 15:19:39 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-09-30 17:23:35 +0100
commitd0b7471ba89e6c7c1c00019bfe6c6c0aea94dd5a (patch)
treeb028f021428dc07c70bb97dade3c52190b11ca1d /bitbake/lib/bb/runqueue.py
parentb83d5c15adcc2d106194d85dd1b0496678c675e1 (diff)
downloadpoky-d0b7471ba89e6c7c1c00019bfe6c6c0aea94dd5a.tar.gz
bitbake: runqueue: Fix task migration problems
Tasks were not migrating consistently, particularly: * if a task was rehashed which had already run * if a task which was valid became invalid due to a rehash We need to always run the migration code for rehashed tasks and then reprocess them for hash validity. This means rearranging the code. It also means several tests are no longer correct and can't be written correctly to work on all possible workflows so those are removed. (Bitbake rev: 8443989ee41e9b162972935513e437b5c66ea74d) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r--bitbake/lib/bb/runqueue.py99
1 files changed, 55 insertions, 44 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index d9a67a3167..02160ef4d7 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -73,7 +73,7 @@ def build_tid(mc, fn, taskname):
73def pending_hash_index(tid, rqdata): 73def pending_hash_index(tid, rqdata):
74 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) 74 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
75 pn = rqdata.dataCaches[mc].pkg_fn[taskfn] 75 pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
76 h = rqdata.runtaskentries[tid].hash 76 h = rqdata.runtaskentries[tid].unihash
77 return pn + ":" + "taskname" + h 77 return pn + ":" + "taskname" + h
78 78
79class RunQueueStats: 79class RunQueueStats:
@@ -2299,9 +2299,6 @@ class RunQueueExecute:
2299 for tid in changed: 2299 for tid in changed:
2300 if tid not in self.rqdata.runq_setscene_tids: 2300 if tid not in self.rqdata.runq_setscene_tids:
2301 continue 2301 continue
2302 valid = self.rq.validate_hashes(set([tid]), self.cooker.data, None, False)
2303 if not valid:
2304 continue
2305 if tid in self.runq_running: 2302 if tid in self.runq_running:
2306 continue 2303 continue
2307 if tid not in self.pending_migrations: 2304 if tid not in self.pending_migrations:
@@ -2358,6 +2355,7 @@ class RunQueueExecute:
2358 2355
2359 logger.info("Setscene task %s now valid and being rerun" % tid) 2356 logger.info("Setscene task %s now valid and being rerun" % tid)
2360 self.sqdone = False 2357 self.sqdone = False
2358 update_scenequeue_data([tid], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
2361 2359
2362 if changed: 2360 if changed:
2363 self.holdoff_need_update = True 2361 self.holdoff_need_update = True
@@ -2674,64 +2672,77 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
2674 2672
2675 rqdata.init_progress_reporter.next_stage() 2673 rqdata.init_progress_reporter.next_stage()
2676 2674
2677 multiconfigs = set() 2675 sqdata.multiconfigs = set()
2678 for tid in sqdata.sq_revdeps: 2676 for tid in sqdata.sq_revdeps:
2679 multiconfigs.add(mc_from_tid(tid)) 2677 sqdata.multiconfigs.add(mc_from_tid(tid))
2680 if len(sqdata.sq_revdeps[tid]) == 0: 2678 if len(sqdata.sq_revdeps[tid]) == 0:
2681 sqrq.sq_buildable.add(tid) 2679 sqrq.sq_buildable.add(tid)
2682 2680
2683 rqdata.init_progress_reporter.finish() 2681 rqdata.init_progress_reporter.finish()
2684 2682
2685 if rq.hashvalidate: 2683 sqdata.noexec = set()
2686 noexec = [] 2684 sqdata.stamppresent = set()
2687 stamppresent = [] 2685 sqdata.valid = set()
2688 tocheck = set()
2689 2686
2690 for tid in sorted(sqdata.sq_revdeps): 2687 update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq)
2691 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
2692 2688
2693 taskdep = rqdata.dataCaches[mc].task_deps[taskfn] 2689def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq):
2694 2690
2695 if 'noexec' in taskdep and taskname in taskdep['noexec']: 2691 tocheck = set()
2696 noexec.append(tid)
2697 sqrq.sq_task_skip(tid)
2698 bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
2699 continue
2700 2692
2701 if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache): 2693 for tid in sorted(tids):
2702 logger.debug(2, 'Setscene stamp current for task %s', tid) 2694 if tid in sqdata.stamppresent:
2703 stamppresent.append(tid) 2695 sqdata.stamppresent.remove(tid)
2704 sqrq.sq_task_skip(tid) 2696 if tid in sqdata.valid:
2705 continue 2697 sqdata.valid.remove(tid)
2706 2698
2707 if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache): 2699 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
2708 logger.debug(2, 'Normal stamp current for task %s', tid)
2709 stamppresent.append(tid)
2710 sqrq.sq_task_skip(tid)
2711 continue
2712 2700
2713 tocheck.add(tid) 2701 taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
2714 2702
2715 valid = rq.validate_hashes(tocheck, cooker.data, len(stamppresent), False) 2703 if 'noexec' in taskdep and taskname in taskdep['noexec']:
2704 sqdata.noexec.add(tid)
2705 sqrq.sq_task_skip(tid)
2706 bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
2707 continue
2708
2709 if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
2710 logger.debug(2, 'Setscene stamp current for task %s', tid)
2711 sqdata.stamppresent.add(tid)
2712 sqrq.sq_task_skip(tid)
2713 continue
2716 2714
2717 valid_new = stamppresent 2715 if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
2718 for v in valid: 2716 logger.debug(2, 'Normal stamp current for task %s', tid)
2719 valid_new.append(v) 2717 sqdata.stamppresent.add(tid)
2718 sqrq.sq_task_skip(tid)
2719 continue
2720 2720
2721 hashes = {} 2721 tocheck.add(tid)
2722 for mc in sorted(multiconfigs): 2722
2723 for tid in sorted(sqdata.sq_revdeps): 2723 sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False)
2724
2725 sqdata.hashes = {}
2726 for mc in sorted(sqdata.multiconfigs):
2727 for tid in sorted(sqdata.sq_revdeps):
2724 if mc_from_tid(tid) != mc: 2728 if mc_from_tid(tid) != mc:
2725 continue 2729 continue
2726 if tid not in valid_new and tid not in noexec and tid not in sqrq.scenequeue_notcovered: 2730 if tid in sqdata.stamppresent:
2727 sqdata.outrightfail.add(tid) 2731 continue
2732 if tid in sqdata.valid:
2733 continue
2734 if tid in sqdata.noexec:
2735 continue
2736 if tid in sqrq.scenequeue_notcovered:
2737 continue
2738 sqdata.outrightfail.add(tid)
2728 2739
2729 h = pending_hash_index(tid, rqdata) 2740 h = pending_hash_index(tid, rqdata)
2730 if h not in hashes: 2741 if h not in sqdata.hashes:
2731 hashes[h] = tid 2742 sqdata.hashes[h] = tid
2732 else: 2743 else:
2733 sqrq.sq_deferred[tid] = hashes[h] 2744 sqrq.sq_deferred[tid] = sqdata.hashes[h]
2734 bb.warn("Deferring %s after %s" % (tid, hashes[h])) 2745 bb.warn("Deferring %s after %s" % (tid, sqdata.hashes[h]))
2735 2746
2736 2747
2737class TaskFailure(Exception): 2748class TaskFailure(Exception):