From eb9809741c4ad33457eea29a90bdb16d4f4995c3 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 13 Dec 2019 16:06:47 +0000 Subject: bitbake: runqueue: Rework process_possible_migrations() to improve performance The looping over multiple changed hashes causes many calls to get_taskhash and get_unihash which are potentially slow and then overwritten. Instead, batch up all the tasks which have changed unihashes and then do one big loop over the changed tasks rather than each in turn. This makes worlds of difference to the performance graphs and should speed up build where many tasks are being rehashed. (Bitbake rev: c9c68d898985cf0bec6fc95f54c151cc50255cac) Signed-off-by: Richard Purdie --- bitbake/lib/bb/runqueue.py | 103 ++++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 47 deletions(-) (limited to 'bitbake/lib/bb/runqueue.py') diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index cb499a1cba..a45b27ce51 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py @@ -2248,6 +2248,7 @@ class RunQueueExecute: def process_possible_migrations(self): changed = set() + toprocess = set() for tid, unihash in self.updated_taskhash_queue.copy(): if tid in self.runq_running and tid not in self.runq_complete: continue @@ -2258,53 +2259,61 @@ class RunQueueExecute: logger.info("Task %s unihash changed to %s" % (tid, unihash)) self.rqdata.runtaskentries[tid].unihash = unihash bb.parse.siggen.set_unihash(tid, unihash) - - # Work out all tasks which depend on this one - total = set() - next = set(self.rqdata.runtaskentries[tid].revdeps) - while next: - current = next.copy() - total = total |next - next = set() - for ntid in current: - next |= self.rqdata.runtaskentries[ntid].revdeps - next.difference_update(total) - - # Now iterate those tasks in dependency order to regenerate their taskhash/unihash - done = set() - next = set(self.rqdata.runtaskentries[tid].revdeps) - while next: - current = next.copy() - next = set() - for tid in current: - if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): - continue - procdep = [] - for dep in self.rqdata.runtaskentries[tid].depends: - procdep.append(dep) - orighash = self.rqdata.runtaskentries[tid].hash - newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)]) - origuni = self.rqdata.runtaskentries[tid].unihash - newuni = bb.parse.siggen.get_unihash(tid) - # FIXME, need to check it can come from sstate at all for determinism? - remapped = False - if newuni == origuni: - # Nothing to do, we match, skip code below - remapped = True - elif tid in self.scenequeue_covered or tid in self.sq_live: - # Already ran this setscene task or it running. Report the new taskhash - remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches) - logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid)) - - if not remapped: - logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni)) - self.rqdata.runtaskentries[tid].hash = newhash - self.rqdata.runtaskentries[tid].unihash = newuni - changed.add(tid) - - next |= self.rqdata.runtaskentries[tid].revdeps - total.remove(tid) - next.intersection_update(total) + toprocess.add(tid) + + # Work out all tasks which depend upon these + total = set() + for p in toprocess: + next = set(self.rqdata.runtaskentries[p].revdeps) + while next: + current = next.copy() + total = total | next + next = set() + for ntid in current: + next |= self.rqdata.runtaskentries[ntid].revdeps + next.difference_update(total) + + # Now iterate those tasks in dependency order to regenerate their taskhash/unihash + next = set() + for p in total: + if len(self.rqdata.runtaskentries[p].depends) == 0: + next.add(p) + elif self.rqdata.runtaskentries[p].depends.isdisjoint(total): + next.add(p) + + # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled + while next: + current = next.copy() + next = set() + for tid in current: + if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): + continue + procdep = [] + for dep in self.rqdata.runtaskentries[tid].depends: + procdep.append(dep) + orighash = self.rqdata.runtaskentries[tid].hash + newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)]) + origuni = self.rqdata.runtaskentries[tid].unihash + newuni = bb.parse.siggen.get_unihash(tid) + # FIXME, need to check it can come from sstate at all for determinism? + remapped = False + if newuni == origuni: + # Nothing to do, we match, skip code below + remapped = True + elif tid in self.scenequeue_covered or tid in self.sq_live: + # Already ran this setscene task or it running. Report the new taskhash + remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches) + logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid)) + + if not remapped: + #logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni)) + self.rqdata.runtaskentries[tid].hash = newhash + self.rqdata.runtaskentries[tid].unihash = newuni + changed.add(tid) + + next |= self.rqdata.runtaskentries[tid].revdeps + total.remove(tid) + next.intersection_update(total) if changed: for mc in self.rq.worker: -- cgit v1.2.3-54-g00ecf