From af04a52bf7d7ae94c187e85028a80dee6d7853f0 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Sun, 12 Jun 2016 23:55:48 +0100 Subject: bitbake: taskdata/runqueue: Rewrite without use of ID indirection I'm not sure what possesed me when I wrote this code originally but its indirection of everyting to use numeric IDs and position dependent lists is horrific. Given the way python internals work, its completely and utterly pointless from performance perspective. It also makes the code hard to understand and debug since any numeric ID has to be translated into something human readable. The hard part is that the IDs are infectous and spread from taskdata into runqueue and even partly into cooker for the dependency graph processing. The only real way to deal with this is to convert everything to use a more sane data structure. This patch: * Uses ":" as the ID for tasks rather than a number * Changes to dict() based structures rather than position dependent lists * Drops the build name, runtime name and filename ID indexes On the most part there shouldn't be user visible changes. Sadly we did leak datastructures to the setscene verify function which has to be rewritten. To handle this, the variable name used to specifiy the version changes from BB_SETSCENE_VERIFY_FUNCTION to BB_SETSCENE_VERIFY_FUNCTION2 allowing multiple versions of bitbake to work with suitably written metadata. Anyone with custom schedulers may also need to change them. I believe the benefits in code readability and easier debugging far outweigh those issues though. It also means we have a saner codebase to add multiconfig support on top of. During development, I did have some of the original code coexisting with the new data stores to allow comparision of the data and check it was working correcty, particuarly for taskdata. I have also compared task-depends.dot files before and after the change. There should be no functionality changes in this patch, its purely a data structure change and that is visible in the patch. (Bitbake rev: 2c88afb60da54e58f555411a7bd7b006b0c29306) Signed-off-by: Richard Purdie --- bitbake/lib/bb/cooker.py | 72 +++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 43 deletions(-) (limited to 'bitbake/lib/bb/cooker.py') diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index ce818f3e59..907565401d 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py @@ -623,9 +623,7 @@ class BBCooker: taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) - targetid = taskdata.getbuild_id(pkgs_to_build[0]) - fnid = taskdata.build_targets[targetid][0] - fn = taskdata.fn_index[fnid] + fn = taskdata.build_targets[pkgs_to_build[0]][0] else: envdata = self.data @@ -714,7 +712,7 @@ class BBCooker: def buildDependTree(self, rq, taskdata): - seen_fnids = [] + seen_fns = [] depend_tree = {} depend_tree["depends"] = {} depend_tree["tdepends"] = {} @@ -732,10 +730,9 @@ class BBCooker: version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] depend_tree['providermap'][name] = (pn, version) - for task in range(len(rq.rqdata.runq_fnid)): - taskname = rq.rqdata.runq_task[task] - fnid = rq.rqdata.runq_fnid[task] - fn = taskdata.fn_index[fnid] + for tid in rq.rqdata.runtaskentries: + taskname = bb.runqueue.taskname_from_tid(tid) + fn = bb.runqueue.fn_from_tid(tid) pn = self.recipecache.pkg_fn[fn] version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] if pn not in depend_tree["pn"]: @@ -756,24 +753,24 @@ class BBCooker: depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn] - for dep in rq.rqdata.runq_depends[task]: - depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]] + for dep in rq.rqdata.runtaskentries[tid].depends: + depfn = bb.runqueue.fn_from_tid(dep) deppn = self.recipecache.pkg_fn[depfn] - dotname = "%s.%s" % (pn, rq.rqdata.runq_task[task]) + dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(dep)) if not dotname in depend_tree["tdepends"]: depend_tree["tdepends"][dotname] = [] - depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.rqdata.runq_task[dep])) - if fnid not in seen_fnids: - seen_fnids.append(fnid) + depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep))) + if fn not in seen_fns: + seen_fns.append(fn) packages = [] depend_tree["depends"][pn] = [] - for dep in taskdata.depids[fnid]: - depend_tree["depends"][pn].append(taskdata.build_names_index[dep]) + for dep in taskdata.depids[fn]: + depend_tree["depends"][pn].append(dep) depend_tree["rdepends-pn"][pn] = [] - for rdep in taskdata.rdepids[fnid]: - depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) + for rdep in taskdata.rdepids[fn]: + depend_tree["rdepends-pn"][pn].append(rdep) rdepends = self.recipecache.rundeps[fn] for package in rdepends: @@ -805,12 +802,8 @@ class BBCooker: Create a dependency tree of pkgs_to_build, returning the data. """ _, taskdata = self.prepareTreeData(pkgs_to_build, task) - tasks_fnid = [] - if len(taskdata.tasks_name) != 0: - for task in range(len(taskdata.tasks_name)): - tasks_fnid.append(taskdata.tasks_fnid[task]) - seen_fnids = [] + seen_fns = [] depend_tree = {} depend_tree["depends"] = {} depend_tree["pn"] = {} @@ -825,9 +818,8 @@ class BBCooker: cachefields = getattr(cache_class, 'cachefields', []) extra_info = extra_info + cachefields - for task in range(len(tasks_fnid)): - fnid = tasks_fnid[task] - fn = taskdata.fn_index[fnid] + for tid in taskdata.taskentries: + fn = bb.runqueue.fn_from_tid(tid) pn = self.recipecache.pkg_fn[fn] if pn not in depend_tree["pn"]: @@ -843,33 +835,27 @@ class BBCooker: for ei in extra_info: depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn] - if fnid not in seen_fnids: - seen_fnids.append(fnid) + if fn not in seen_fns: + seen_fns.append(fn) depend_tree["depends"][pn] = [] - for dep in taskdata.depids[fnid]: - item = taskdata.build_names_index[dep] + for item in taskdata.depids[fn]: pn_provider = "" - targetid = taskdata.getbuild_id(item) - if targetid in taskdata.build_targets and taskdata.build_targets[targetid]: - id = taskdata.build_targets[targetid][0] - fn_provider = taskdata.fn_index[id] + if dep in taskdata.build_targets and taskdata.build_targets[dep]: + fn_provider = taskdata.build_targets[dep][0] pn_provider = self.recipecache.pkg_fn[fn_provider] else: pn_provider = item depend_tree["depends"][pn].append(pn_provider) depend_tree["rdepends-pn"][pn] = [] - for rdep in taskdata.rdepids[fnid]: - item = taskdata.run_names_index[rdep] + for rdep in taskdata.rdepids[fn]: pn_rprovider = "" - targetid = taskdata.getrun_id(item) - if targetid in taskdata.run_targets and taskdata.run_targets[targetid]: - id = taskdata.run_targets[targetid][0] - fn_rprovider = taskdata.fn_index[id] + if rdep in taskdata.run_targets and taskdata.run_targets[rdep]: + fn_rprovider = taskdata.run_targets[rdep][0] pn_rprovider = self.recipecache.pkg_fn[fn_rprovider] else: - pn_rprovider = item + pn_rprovider = rdep depend_tree["rdepends-pn"][pn].append(pn_rprovider) depend_tree["rdepends-pkg"].update(rdepends) @@ -1350,7 +1336,7 @@ class BBCooker: return False if not retval: - bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures, interrupted), self.expanded_data) + bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.expanded_data) self.command.finishAsyncCommand(msg) return False if retval is True: @@ -1386,7 +1372,7 @@ class BBCooker: return False if not retval: - bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures, interrupted), self.data) + bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.data) self.command.finishAsyncCommand(msg) return False if retval is True: -- cgit v1.2.3-54-g00ecf