diff options
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 102 |
1 files changed, 72 insertions, 30 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index bc7e18175d..3462ed4457 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -1273,27 +1273,41 @@ class RunQueueData: | |||
1273 | 1273 | ||
1274 | bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids) | 1274 | bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids) |
1275 | 1275 | ||
1276 | starttime = time.time() | ||
1277 | lasttime = starttime | ||
1278 | |||
1276 | # Iterate over the task list and call into the siggen code | 1279 | # Iterate over the task list and call into the siggen code |
1277 | dealtwith = set() | 1280 | dealtwith = set() |
1278 | todeal = set(self.runtaskentries) | 1281 | todeal = set(self.runtaskentries) |
1279 | while todeal: | 1282 | while todeal: |
1283 | ready = set() | ||
1280 | for tid in todeal.copy(): | 1284 | for tid in todeal.copy(): |
1281 | if not (self.runtaskentries[tid].depends - dealtwith): | 1285 | if not (self.runtaskentries[tid].depends - dealtwith): |
1282 | dealtwith.add(tid) | 1286 | self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) |
1283 | todeal.remove(tid) | 1287 | # get_taskhash for a given tid *must* be called before get_unihash* below |
1284 | self.prepare_task_hash(tid) | 1288 | self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) |
1285 | bb.event.check_for_interrupts(self.cooker.data) | 1289 | ready.add(tid) |
1290 | unihashes = bb.parse.siggen.get_unihashes(ready) | ||
1291 | for tid in ready: | ||
1292 | dealtwith.add(tid) | ||
1293 | todeal.remove(tid) | ||
1294 | self.runtaskentries[tid].unihash = unihashes[tid] | ||
1295 | |||
1296 | bb.event.check_for_interrupts(self.cooker.data) | ||
1297 | |||
1298 | if time.time() > (lasttime + 30): | ||
1299 | lasttime = time.time() | ||
1300 | hashequiv_logger.verbose("Initial setup loop progress: %s of %s in %s" % (len(todeal), len(self.runtaskentries), lasttime - starttime)) | ||
1301 | |||
1302 | endtime = time.time() | ||
1303 | if (endtime-starttime > 60): | ||
1304 | hashequiv_logger.verbose("Initial setup loop took: %s" % (endtime-starttime)) | ||
1286 | 1305 | ||
1287 | bb.parse.siggen.writeout_file_checksum_cache() | 1306 | bb.parse.siggen.writeout_file_checksum_cache() |
1288 | 1307 | ||
1289 | #self.dump_data() | 1308 | #self.dump_data() |
1290 | return len(self.runtaskentries) | 1309 | return len(self.runtaskentries) |
1291 | 1310 | ||
1292 | def prepare_task_hash(self, tid): | ||
1293 | bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) | ||
1294 | self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches) | ||
1295 | self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid) | ||
1296 | |||
1297 | def dump_data(self): | 1311 | def dump_data(self): |
1298 | """ | 1312 | """ |
1299 | Dump some debug information on the internal data structures | 1313 | Dump some debug information on the internal data structures |
@@ -2438,14 +2452,17 @@ class RunQueueExecute: | |||
2438 | taskdepdata_cache = {} | 2452 | taskdepdata_cache = {} |
2439 | for task in self.rqdata.runtaskentries: | 2453 | for task in self.rqdata.runtaskentries: |
2440 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) | 2454 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) |
2441 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | 2455 | taskdepdata_cache[task] = bb.TaskData( |
2442 | deps = self.rqdata.runtaskentries[task].depends | 2456 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn], |
2443 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] | 2457 | taskname = taskname, |
2444 | taskhash = self.rqdata.runtaskentries[task].hash | 2458 | fn = fn, |
2445 | unihash = self.rqdata.runtaskentries[task].unihash | 2459 | deps = self.filtermcdeps(task, mc, self.rqdata.runtaskentries[task].depends), |
2446 | deps = self.filtermcdeps(task, mc, deps) | 2460 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn], |
2447 | hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn] | 2461 | taskhash = self.rqdata.runtaskentries[task].hash, |
2448 | taskdepdata_cache[task] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn] | 2462 | unihash = self.rqdata.runtaskentries[task].unihash, |
2463 | hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn], | ||
2464 | taskhash_deps = self.rqdata.runtaskentries[task].taskhash_deps, | ||
2465 | ) | ||
2449 | 2466 | ||
2450 | self.taskdepdata_cache = taskdepdata_cache | 2467 | self.taskdepdata_cache = taskdepdata_cache |
2451 | 2468 | ||
@@ -2460,9 +2477,11 @@ class RunQueueExecute: | |||
2460 | while next: | 2477 | while next: |
2461 | additional = [] | 2478 | additional = [] |
2462 | for revdep in next: | 2479 | for revdep in next: |
2463 | self.taskdepdata_cache[revdep][6] = self.rqdata.runtaskentries[revdep].unihash | 2480 | self.taskdepdata_cache[revdep] = self.taskdepdata_cache[revdep]._replace( |
2481 | unihash=self.rqdata.runtaskentries[revdep].unihash | ||
2482 | ) | ||
2464 | taskdepdata[revdep] = self.taskdepdata_cache[revdep] | 2483 | taskdepdata[revdep] = self.taskdepdata_cache[revdep] |
2465 | for revdep2 in self.taskdepdata_cache[revdep][3]: | 2484 | for revdep2 in self.taskdepdata_cache[revdep].deps: |
2466 | if revdep2 not in taskdepdata: | 2485 | if revdep2 not in taskdepdata: |
2467 | additional.append(revdep2) | 2486 | additional.append(revdep2) |
2468 | next = additional | 2487 | next = additional |
@@ -2531,9 +2550,6 @@ class RunQueueExecute: | |||
2531 | self.rqdata.runtaskentries[hashtid].unihash = unihash | 2550 | self.rqdata.runtaskentries[hashtid].unihash = unihash |
2532 | bb.parse.siggen.set_unihash(hashtid, unihash) | 2551 | bb.parse.siggen.set_unihash(hashtid, unihash) |
2533 | toprocess.add(hashtid) | 2552 | toprocess.add(hashtid) |
2534 | if torehash: | ||
2535 | # Need to save after set_unihash above | ||
2536 | bb.parse.siggen.save_unitaskhashes() | ||
2537 | 2553 | ||
2538 | # Work out all tasks which depend upon these | 2554 | # Work out all tasks which depend upon these |
2539 | total = set() | 2555 | total = set() |
@@ -2556,17 +2572,28 @@ class RunQueueExecute: | |||
2556 | elif self.rqdata.runtaskentries[p].depends.isdisjoint(total): | 2572 | elif self.rqdata.runtaskentries[p].depends.isdisjoint(total): |
2557 | next.add(p) | 2573 | next.add(p) |
2558 | 2574 | ||
2575 | starttime = time.time() | ||
2576 | lasttime = starttime | ||
2577 | |||
2559 | # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled | 2578 | # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled |
2560 | while next: | 2579 | while next: |
2561 | current = next.copy() | 2580 | current = next.copy() |
2562 | next = set() | 2581 | next = set() |
2582 | ready = {} | ||
2563 | for tid in current: | 2583 | for tid in current: |
2564 | if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): | 2584 | if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): |
2565 | continue | 2585 | continue |
2586 | # get_taskhash for a given tid *must* be called before get_unihash* below | ||
2587 | ready[tid] = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches) | ||
2588 | |||
2589 | unihashes = bb.parse.siggen.get_unihashes(ready.keys()) | ||
2590 | |||
2591 | for tid in ready: | ||
2566 | orighash = self.rqdata.runtaskentries[tid].hash | 2592 | orighash = self.rqdata.runtaskentries[tid].hash |
2567 | newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches) | 2593 | newhash = ready[tid] |
2568 | origuni = self.rqdata.runtaskentries[tid].unihash | 2594 | origuni = self.rqdata.runtaskentries[tid].unihash |
2569 | newuni = bb.parse.siggen.get_unihash(tid) | 2595 | newuni = unihashes[tid] |
2596 | |||
2570 | # FIXME, need to check it can come from sstate at all for determinism? | 2597 | # FIXME, need to check it can come from sstate at all for determinism? |
2571 | remapped = False | 2598 | remapped = False |
2572 | if newuni == origuni: | 2599 | if newuni == origuni: |
@@ -2587,6 +2614,15 @@ class RunQueueExecute: | |||
2587 | next |= self.rqdata.runtaskentries[tid].revdeps | 2614 | next |= self.rqdata.runtaskentries[tid].revdeps |
2588 | total.remove(tid) | 2615 | total.remove(tid) |
2589 | next.intersection_update(total) | 2616 | next.intersection_update(total) |
2617 | bb.event.check_for_interrupts(self.cooker.data) | ||
2618 | |||
2619 | if time.time() > (lasttime + 30): | ||
2620 | lasttime = time.time() | ||
2621 | hashequiv_logger.verbose("Rehash loop slow progress: %s in %s" % (len(total), lasttime - starttime)) | ||
2622 | |||
2623 | endtime = time.time() | ||
2624 | if (endtime-starttime > 60): | ||
2625 | hashequiv_logger.verbose("Rehash loop took more than 60s: %s" % (endtime-starttime)) | ||
2590 | 2626 | ||
2591 | if changed: | 2627 | if changed: |
2592 | for mc in self.rq.worker: | 2628 | for mc in self.rq.worker: |
@@ -2806,13 +2842,19 @@ class RunQueueExecute: | |||
2806 | additional = [] | 2842 | additional = [] |
2807 | for revdep in next: | 2843 | for revdep in next: |
2808 | (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep) | 2844 | (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep) |
2809 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
2810 | deps = getsetscenedeps(revdep) | 2845 | deps = getsetscenedeps(revdep) |
2811 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] | 2846 | |
2812 | taskhash = self.rqdata.runtaskentries[revdep].hash | 2847 | taskdepdata[revdep] = bb.TaskData( |
2813 | unihash = self.rqdata.runtaskentries[revdep].unihash | 2848 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn], |
2814 | hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn] | 2849 | taskname = taskname, |
2815 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash, hashfn] | 2850 | fn = fn, |
2851 | deps = deps, | ||
2852 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn], | ||
2853 | taskhash = self.rqdata.runtaskentries[revdep].hash, | ||
2854 | unihash = self.rqdata.runtaskentries[revdep].unihash, | ||
2855 | hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn], | ||
2856 | taskhash_deps = self.rqdata.runtaskentries[revdep].taskhash_deps, | ||
2857 | ) | ||
2816 | for revdep2 in deps: | 2858 | for revdep2 in deps: |
2817 | if revdep2 not in taskdepdata: | 2859 | if revdep2 not in taskdepdata: |
2818 | additional.append(revdep2) | 2860 | additional.append(revdep2) |