summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-12-13 16:06:47 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-12-16 23:27:14 +0000
commiteb9809741c4ad33457eea29a90bdb16d4f4995c3 (patch)
tree6c5c2f9018d4e91ffdc0ead0cca1461846f72c37 /bitbake
parente7f8c836f679f0d16dfa7390d58523ff473bb654 (diff)
downloadpoky-eb9809741c4ad33457eea29a90bdb16d4f4995c3.tar.gz
bitbake: runqueue: Rework process_possible_migrations() to improve performance
The looping over multiple changed hashes causes many calls to get_taskhash and get_unihash which are potentially slow and then overwritten. Instead, batch up all the tasks which have changed unihashes and then do one big loop over the changed tasks rather than each in turn. This makes worlds of difference to the performance graphs and should speed up build where many tasks are being rehashed. (Bitbake rev: c9c68d898985cf0bec6fc95f54c151cc50255cac) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/runqueue.py103
1 files changed, 56 insertions, 47 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index cb499a1cba..a45b27ce51 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -2248,6 +2248,7 @@ class RunQueueExecute:
2248 def process_possible_migrations(self): 2248 def process_possible_migrations(self):
2249 2249
2250 changed = set() 2250 changed = set()
2251 toprocess = set()
2251 for tid, unihash in self.updated_taskhash_queue.copy(): 2252 for tid, unihash in self.updated_taskhash_queue.copy():
2252 if tid in self.runq_running and tid not in self.runq_complete: 2253 if tid in self.runq_running and tid not in self.runq_complete:
2253 continue 2254 continue
@@ -2258,53 +2259,61 @@ class RunQueueExecute:
2258 logger.info("Task %s unihash changed to %s" % (tid, unihash)) 2259 logger.info("Task %s unihash changed to %s" % (tid, unihash))
2259 self.rqdata.runtaskentries[tid].unihash = unihash 2260 self.rqdata.runtaskentries[tid].unihash = unihash
2260 bb.parse.siggen.set_unihash(tid, unihash) 2261 bb.parse.siggen.set_unihash(tid, unihash)
2261 2262 toprocess.add(tid)
2262 # Work out all tasks which depend on this one 2263
2263 total = set() 2264 # Work out all tasks which depend upon these
2264 next = set(self.rqdata.runtaskentries[tid].revdeps) 2265 total = set()
2265 while next: 2266 for p in toprocess:
2266 current = next.copy() 2267 next = set(self.rqdata.runtaskentries[p].revdeps)
2267 total = total |next 2268 while next:
2268 next = set() 2269 current = next.copy()
2269 for ntid in current: 2270 total = total | next
2270 next |= self.rqdata.runtaskentries[ntid].revdeps 2271 next = set()
2271 next.difference_update(total) 2272 for ntid in current:
2272 2273 next |= self.rqdata.runtaskentries[ntid].revdeps
2273 # Now iterate those tasks in dependency order to regenerate their taskhash/unihash 2274 next.difference_update(total)
2274 done = set() 2275
2275 next = set(self.rqdata.runtaskentries[tid].revdeps) 2276 # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
2276 while next: 2277 next = set()
2277 current = next.copy() 2278 for p in total:
2278 next = set() 2279 if len(self.rqdata.runtaskentries[p].depends) == 0:
2279 for tid in current: 2280 next.add(p)
2280 if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total): 2281 elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
2281 continue 2282 next.add(p)
2282 procdep = [] 2283
2283 for dep in self.rqdata.runtaskentries[tid].depends: 2284 # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
2284 procdep.append(dep) 2285 while next:
2285 orighash = self.rqdata.runtaskentries[tid].hash 2286 current = next.copy()
2286 newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)]) 2287 next = set()
2287 origuni = self.rqdata.runtaskentries[tid].unihash 2288 for tid in current:
2288 newuni = bb.parse.siggen.get_unihash(tid) 2289 if not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
2289 # FIXME, need to check it can come from sstate at all for determinism? 2290 continue
2290 remapped = False 2291 procdep = []
2291 if newuni == origuni: 2292 for dep in self.rqdata.runtaskentries[tid].depends:
2292 # Nothing to do, we match, skip code below 2293 procdep.append(dep)
2293 remapped = True 2294 orighash = self.rqdata.runtaskentries[tid].hash
2294 elif tid in self.scenequeue_covered or tid in self.sq_live: 2295 newhash = bb.parse.siggen.get_taskhash(tid, procdep, self.rqdata.dataCaches[mc_from_tid(tid)])
2295 # Already ran this setscene task or it running. Report the new taskhash 2296 origuni = self.rqdata.runtaskentries[tid].unihash
2296 remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches) 2297 newuni = bb.parse.siggen.get_unihash(tid)
2297 logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid)) 2298 # FIXME, need to check it can come from sstate at all for determinism?
2298 2299 remapped = False
2299 if not remapped: 2300 if newuni == origuni:
2300 logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni)) 2301 # Nothing to do, we match, skip code below
2301 self.rqdata.runtaskentries[tid].hash = newhash 2302 remapped = True
2302 self.rqdata.runtaskentries[tid].unihash = newuni 2303 elif tid in self.scenequeue_covered or tid in self.sq_live:
2303 changed.add(tid) 2304 # Already ran this setscene task or it running. Report the new taskhash
2304 2305 remapped = bb.parse.siggen.report_unihash_equiv(tid, newhash, origuni, newuni, self.rqdata.dataCaches)
2305 next |= self.rqdata.runtaskentries[tid].revdeps 2306 logger.info("Already covered setscene for %s so ignoring rehash (remap)" % (tid))
2306 total.remove(tid) 2307
2307 next.intersection_update(total) 2308 if not remapped:
2309 #logger.debug(1, "Task %s hash changes: %s->%s %s->%s" % (tid, orighash, newhash, origuni, newuni))
2310 self.rqdata.runtaskentries[tid].hash = newhash
2311 self.rqdata.runtaskentries[tid].unihash = newuni
2312 changed.add(tid)
2313
2314 next |= self.rqdata.runtaskentries[tid].revdeps
2315 total.remove(tid)
2316 next.intersection_update(total)
2308 2317
2309 if changed: 2318 if changed:
2310 for mc in self.rq.worker: 2319 for mc in self.rq.worker: