bitbake: taskdata/runqueue: Rewrite without use of ID indirection

I'm not sure what possesed me when I wrote this code originally but its
indirection of everyting to use numeric IDs and position dependent lists
is horrific. Given the way python internals work, its completely and
utterly pointless from  performance perspective. It also makes the code
hard to understand and debug since any numeric ID has to be translated
into something human readable.

The hard part is that the IDs are infectous and spread from taskdata
into runqueue and even partly into cooker for the dependency graph
processing. The only real way to deal with this is to convert everything
to use a more sane data structure.

This patch:
* Uses "<fn>:<taskname>" as the ID for tasks rather than a number
* Changes to dict() based structures rather than position dependent lists
* Drops the build name, runtime name and filename ID indexes

On the most part there shouldn't be user visible changes. Sadly we did
leak datastructures to the setscene verify function which has to be
rewritten. To handle this, the variable name used to specifiy the version
changes from BB_SETSCENE_VERIFY_FUNCTION to BB_SETSCENE_VERIFY_FUNCTION2
allowing multiple versions of bitbake to work with suitably written
metadata. Anyone with custom schedulers may also need to change them.

I believe the benefits in code readability and easier debugging far
outweigh those issues though. It also means we have a saner codebase
to add multiconfig support on top of.

During development, I did have some of the original code coexisting with
the new data stores to allow comparision of the data and check it was
working correcty, particuarly for taskdata. I have also compared
task-depends.dot files before and after the change. There should be no
functionality changes in this patch, its purely a data structure change
and that is visible in the patch.

(Bitbake rev: 2c88afb60da54e58f555411a7bd7b006b0c29306)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 152 insertions, 260 deletions

diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
index 65628c86f2..d8bdbcabf9 100644
--- a/bitbake/lib/bb/taskdata.py
+++ b/bitbake/lib/bb/taskdata.py
@@ -37,27 +37,24 @@ def re_match_strings(target, strings):
     return any(name == target or re.match(name, target)
                for name in strings)
 
+class TaskEntry:
+    def __init__(self):
+        self.tdepends = []
+        self.idepends = []
+        self.irdepends = []
+
 class TaskData:
     """
     BitBake Task Data implementation
     """
     def __init__(self, abort = True, tryaltconfigs = False, skiplist = None, allowincomplete = False):
-        self.build_names_index = []
-        self.run_names_index = []
-        self.fn_index = []
-
         self.build_targets = {}
         self.run_targets = {}
 
         self.external_targets = []
 
-        self.tasks_fnid = []
-        self.tasks_name = []
-        self.tasks_tdepends = []
-        self.tasks_idepends = []
-        self.tasks_irdepends = []
-        # Cache to speed up task ID lookups
-        self.tasks_lookup = {}
+        self.seenfns = []
+        self.taskentries = {}
 
         self.depids = {}
         self.rdepids = {}
@@ -66,7 +63,7 @@ class TaskData:
 
         self.failed_deps = []
         self.failed_rdeps = []
-        self.failed_fnids = []
+        self.failed_fns = []
 
         self.abort = abort
         self.tryaltconfigs = tryaltconfigs
@@ -74,88 +71,6 @@ class TaskData:
 
         self.skiplist = skiplist
 
-    def getbuild_id(self, name):
-        """
-        Return an ID number for the build target name.
-        If it doesn't exist, create one.
-        """
-        if not name in self.build_names_index:
-            self.build_names_index.append(name)
-            return len(self.build_names_index) - 1
-
-        return self.build_names_index.index(name)
-
-    def getrun_id(self, name):
-        """
-        Return an ID number for the run target name.
-        If it doesn't exist, create one.
-        """
-        if not name in self.run_names_index:
-            self.run_names_index.append(name)
-            return len(self.run_names_index) - 1
-
-        return self.run_names_index.index(name)
-
-    def getfn_id(self, name):
-        """
-        Return an ID number for the filename.
-        If it doesn't exist, create one.
-        """
-        if not name in self.fn_index:
-            self.fn_index.append(name)
-            return len(self.fn_index) - 1
-
-        return self.fn_index.index(name)
-
-    def gettask_ids(self, fnid):
-        """
-        Return an array of the ID numbers matching a given fnid.
-        """
-        ids = []
-        if fnid in self.tasks_lookup:
-            for task in self.tasks_lookup[fnid]:
-                ids.append(self.tasks_lookup[fnid][task])
-        return ids
-
-    def gettask_id_fromfnid(self, fnid, task):
-        """
-        Return an ID number for the task matching fnid and task.
-        """
-        if fnid in self.tasks_lookup:
-            if task in self.tasks_lookup[fnid]:
-                return self.tasks_lookup[fnid][task]
-
-        return None
-
-    def gettask_id(self, fn, task, create = True):
-        """
-        Return an ID number for the task matching fn and task.
-        If it doesn't exist, create one by default.
-        Optionally return None instead.
-        """
-        fnid = self.getfn_id(fn)
-
-        if fnid in self.tasks_lookup:
-            if task in self.tasks_lookup[fnid]:
-                return self.tasks_lookup[fnid][task]
-
-        if not create:
-            return None
-
-        self.tasks_name.append(task)
-        self.tasks_fnid.append(fnid)
-        self.tasks_tdepends.append([])
-        self.tasks_idepends.append([])
-        self.tasks_irdepends.append([])
-
-        listid = len(self.tasks_name) - 1
-
-        if fnid not in self.tasks_lookup:
-            self.tasks_lookup[fnid] = {}
-        self.tasks_lookup[fnid][task] = listid
-
-        return listid
-
     def add_tasks(self, fn, dataCache):
         """
         Add tasks for a given fn to the database
@@ -163,29 +78,31 @@ class TaskData:
 
         task_deps = dataCache.task_deps[fn]
 
-        fnid = self.getfn_id(fn)
-
-        if fnid in self.failed_fnids:
+        if fn in self.failed_fns:
             bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
 
         # Check if we've already seen this fn
-        if fnid in self.tasks_fnid:
+        if fn in self.seenfns:
             return
 
+        self.seenfns.append(fn)
+
         self.add_extra_deps(fn, dataCache)
 
         for task in task_deps['tasks']:
 
+            tid = "%s:%s" % (fn, task)
+            self.taskentries[tid] = TaskEntry()
+
             # Work out task dependencies
             parentids = []
             for dep in task_deps['parents'][task]:
                 if dep not in task_deps['tasks']:
                     bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep))
                     continue
-                parentid = self.gettask_id(fn, dep)
+                parentid = "%s:%s" % (fn, dep)
                 parentids.append(parentid)
-            taskid = self.gettask_id(fn, task)
-            self.tasks_tdepends[taskid].extend(parentids)
+            self.taskentries[tid].tdepends.extend(parentids)
 
             # Touch all intertask dependencies
             if 'depends' in task_deps and task in task_deps['depends']:
@@ -194,29 +111,30 @@ class TaskData:
                     if dep:
                         if ":" not in dep:
                             bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep))
-                        ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1]))
-                self.tasks_idepends[taskid].extend(ids)
+                        ids.append(((dep.split(":")[0]), dep.split(":")[1]))
+                        self.seen_build_target(dep.split(":")[0])
+                self.taskentries[tid].idepends.extend(ids)
             if 'rdepends' in task_deps and task in task_deps['rdepends']:
                 ids = []
                 for dep in task_deps['rdepends'][task].split():
                     if dep:
                         if ":" not in dep:
                             bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep))
-                        ids.append(((self.getrun_id(dep.split(":")[0])), dep.split(":")[1]))
-                self.tasks_irdepends[taskid].extend(ids)
-
+                        ids.append(((dep.split(":")[0]), dep.split(":")[1]))
+                        self.seen_run_target(dep.split(":")[0])
+                self.taskentries[tid].irdepends.extend(ids)
 
         # Work out build dependencies
-        if not fnid in self.depids:
-            dependids = {}
+        if not fn in self.depids:
+            dependids = set()
             for depend in dataCache.deps[fn]:
-                dependids[self.getbuild_id(depend)] = None
-            self.depids[fnid] = dependids.keys()
+                dependids.add(depend)
+            self.depids[fn] = list(dependids)
             logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
 
         # Work out runtime dependencies
-        if not fnid in self.rdepids:
-            rdependids = {}
+        if not fn in self.rdepids:
+            rdependids = set()
             rdepends = dataCache.rundeps[fn]
             rrecs = dataCache.runrecs[fn]
             rdependlist = []
@@ -224,24 +142,26 @@ class TaskData:
             for package in rdepends:
                 for rdepend in rdepends[package]:
                     rdependlist.append(rdepend)
-                    rdependids[self.getrun_id(rdepend)] = None
+                    rdependids.add(rdepend)
             for package in rrecs:
                 for rdepend in rrecs[package]:
                     rreclist.append(rdepend)
-                    rdependids[self.getrun_id(rdepend)] = None
+                    rdependids.add(rdepend)
             if rdependlist:
                 logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
             if rreclist:
                 logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
-            self.rdepids[fnid] = rdependids.keys()
+            self.rdepids[fn] = list(rdependids)
 
-        for dep in self.depids[fnid]:
+        for dep in self.depids[fn]:
+            self.seen_build_target(dep)
             if dep in self.failed_deps:
-                self.fail_fnid(fnid)
+                self.fail_fn(fn)
                 return
-        for dep in self.rdepids[fnid]:
+        for dep in self.rdepids[fn]:
+            self.seen_run_target(dep)
             if dep in self.failed_rdeps:
-                self.fail_fnid(fnid)
+                self.fail_fn(fn)
                 return
 
     def add_extra_deps(self, fn, dataCache):
@@ -263,9 +183,7 @@ class TaskData:
         """
         Have we a build target matching this name?
         """
-        targetid = self.getbuild_id(target)
-
-        if targetid in self.build_targets:
+        if target in self.build_targets and self.build_targets[target]:
             return True
         return False
 
@@ -273,50 +191,54 @@ class TaskData:
         """
         Have we a runtime target matching this name?
         """
-        targetid = self.getrun_id(target)
-
-        if targetid in self.run_targets:
+        if target in self.run_targets and self.run_targets[target]:
             return True
         return False
 
+    def seen_build_target(self, name):
+        """
+        Maintain a list of build targets
+        """
+        if name not in self.build_targets:
+            self.build_targets[name] = []
+
     def add_build_target(self, fn, item):
         """
         Add a build target.
         If already present, append the provider fn to the list
         """
-        targetid = self.getbuild_id(item)
-        fnid = self.getfn_id(fn)
-
-        if targetid in self.build_targets:
-            if fnid in self.build_targets[targetid]:
+        if item in self.build_targets:
+            if fn in self.build_targets[item]:
                 return
-            self.build_targets[targetid].append(fnid)
+            self.build_targets[item].append(fn)
             return
-        self.build_targets[targetid] = [fnid]
+        self.build_targets[item] = [fn]
+
+    def seen_run_target(self, name):
+        """
+        Maintain a list of runtime build targets
+        """
+        if name not in self.run_targets:
+            self.run_targets[name] = []
 
     def add_runtime_target(self, fn, item):
         """
         Add a runtime target.
         If already present, append the provider fn to the list
         """
-        targetid = self.getrun_id(item)
-        fnid = self.getfn_id(fn)
-
-        if targetid in self.run_targets:
-            if fnid in self.run_targets[targetid]:
+        if item in self.run_targets:
+            if fn in self.run_targets[item]:
                 return
-            self.run_targets[targetid].append(fnid)
+            self.run_targets[item].append(fn)
             return
-        self.run_targets[targetid] = [fnid]
+        self.run_targets[item] = [fn]
 
-    def mark_external_target(self, item):
+    def mark_external_target(self, target):
         """
         Mark a build target as being externally requested
         """
-        targetid = self.getbuild_id(item)
-
-        if targetid not in self.external_targets:
-            self.external_targets.append(targetid)
+        if target not in self.external_targets:
+            self.external_targets.append(target)
 
     def get_unresolved_build_targets(self, dataCache):
         """
@@ -324,12 +246,12 @@ class TaskData:
         are unknown.
         """
         unresolved = []
-        for target in self.build_names_index:
+        for target in self.build_targets:
             if re_match_strings(target, dataCache.ignored_dependencies):
                 continue
-            if self.build_names_index.index(target) in self.failed_deps:
+            if target in self.failed_deps:
                 continue
-            if not self.have_build_target(target):
+            if not self.build_targets[target]:
                 unresolved.append(target)
         return unresolved
 
@@ -339,12 +261,12 @@ class TaskData:
         are unknown.
         """
         unresolved = []
-        for target in self.run_names_index:
+        for target in self.run_targets:
             if re_match_strings(target, dataCache.ignored_dependencies):
                 continue
-            if self.run_names_index.index(target) in self.failed_rdeps:
+            if target in self.failed_rdeps:
                 continue
-            if not self.have_runtime_target(target):
+            if not self.run_targets[target]:
                 unresolved.append(target)
         return unresolved
 
@@ -352,50 +274,26 @@ class TaskData:
         """
         Return a list of providers of item
         """
-        targetid = self.getbuild_id(item)
+        return self.build_targets[item]
 
-        return self.build_targets[targetid]
-
-    def get_dependees(self, itemid):
+    def get_dependees(self, item):
         """
         Return a list of targets which depend on item
         """
         dependees = []
-        for fnid in self.depids:
-            if itemid in self.depids[fnid]:
-                dependees.append(fnid)
-        return dependees
-
-    def get_dependees_str(self, item):
-        """
-        Return a list of targets which depend on item as a user readable string
-        """
-        itemid = self.getbuild_id(item)
-        dependees = []
-        for fnid in self.depids:
-            if itemid in self.depids[fnid]:
-                dependees.append(self.fn_index[fnid])
+        for fn in self.depids:
+            if item in self.depids[fn]:
+                dependees.append(fn)
         return dependees
 
-    def get_rdependees(self, itemid):
+    def get_rdependees(self, item):
         """
         Return a list of targets which depend on runtime item
         """
         dependees = []
-        for fnid in self.rdepids:
-            if itemid in self.rdepids[fnid]:
-                dependees.append(fnid)
-        return dependees
-
-    def get_rdependees_str(self, item):
-        """
-        Return a list of targets which depend on runtime item as a user readable string
-        """
-        itemid = self.getrun_id(item)
-        dependees = []
-        for fnid in self.rdepids:
-            if itemid in self.rdepids[fnid]:
-                dependees.append(self.fn_index[fnid])
+        for fn in self.rdepids:
+            if item in self.rdepids[fn]:
+                dependees.append(fn)
         return dependees
 
     def get_reasons(self, item, runtime=False):
@@ -431,7 +329,7 @@ class TaskData:
         except bb.providers.NoProvider:
             if self.abort:
                 raise
-            self.remove_buildtarget(self.getbuild_id(item))
+            self.remove_buildtarget(item)
 
         self.mark_external_target(item)
 
@@ -453,7 +351,7 @@ class TaskData:
                 new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
                 if new not in close_matches:
                     close_matches.append(new)
-            bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
+            bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
             raise bb.providers.NoProvider(item)
 
         if self.have_build_target(item):
@@ -462,10 +360,10 @@ class TaskData:
         all_p = dataCache.providers[item]
 
         eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
-        eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
+        eligible = [p for p in eligible if not p in self.failed_fns]
 
         if not eligible:
-            bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
+            bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
             raise bb.providers.NoProvider(item)
 
         if len(eligible) > 1 and foundUnique == False:
@@ -477,8 +375,7 @@ class TaskData:
             self.consider_msgs_cache.append(item)
 
         for fn in eligible:
-            fnid = self.getfn_id(fn)
-            if fnid in self.failed_fnids:
+            if fn in self.failed_fns:
                 continue
             logger.debug(2, "adding %s to satisfy %s", fn, item)
             self.add_build_target(fn, item)
@@ -502,14 +399,14 @@ class TaskData:
         all_p = bb.providers.getRuntimeProviders(dataCache, item)
 
         if not all_p:
-            bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=self.get_reasons(item, True)), cfgData)
+            bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
             raise bb.providers.NoRProvider(item)
 
         eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
-        eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
+        eligible = [p for p in eligible if not p in self.failed_fns]
 
         if not eligible:
-            bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
+            bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
             raise bb.providers.NoRProvider(item)
 
         if len(eligible) > 1 and numberPreferred == 0:
@@ -531,82 +428,80 @@ class TaskData:
 
         # run through the list until we find one that we can build
         for fn in eligible:
-            fnid = self.getfn_id(fn)
-            if fnid in self.failed_fnids:
+            if fn in self.failed_fns:
                 continue
             logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
             self.add_runtime_target(fn, item)
             self.add_tasks(fn, dataCache)
 
-    def fail_fnid(self, fnid, missing_list=None):
+    def fail_fn(self, fn, missing_list=None):
         """
         Mark a file as failed (unbuildable)
         Remove any references from build and runtime provider lists
 
         missing_list, A list of missing requirements for this target
         """
-        if fnid in self.failed_fnids:
+        if fn in self.failed_fns:
             return
         if not missing_list:
             missing_list = []
-        logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid])
-        self.failed_fnids.append(fnid)
+        logger.debug(1, "File '%s' is unbuildable, removing...", fn)
+        self.failed_fns.append(fn)
         for target in self.build_targets:
-            if fnid in self.build_targets[target]:
-                self.build_targets[target].remove(fnid)
+            if fn in self.build_targets[target]:
+                self.build_targets[target].remove(fn)
                 if len(self.build_targets[target]) == 0:
                     self.remove_buildtarget(target, missing_list)
         for target in self.run_targets:
-            if fnid in self.run_targets[target]:
-                self.run_targets[target].remove(fnid)
+            if fn in self.run_targets[target]:
+                self.run_targets[target].remove(fn)
                 if len(self.run_targets[target]) == 0:
                     self.remove_runtarget(target, missing_list)
 
-    def remove_buildtarget(self, targetid, missing_list=None):
+    def remove_buildtarget(self, target, missing_list=None):
         """
         Mark a build target as failed (unbuildable)
         Trigger removal of any files that have this as a dependency
         """
         if not missing_list:
-            missing_list = [self.build_names_index[targetid]]
+            missing_list = [target]
         else:
-            missing_list = [self.build_names_index[targetid]] + missing_list
-        logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.build_names_index[targetid], missing_list)
-        self.failed_deps.append(targetid)
-        dependees = self.get_dependees(targetid)
-        for fnid in dependees:
-            self.fail_fnid(fnid, missing_list)
-        for taskid in range(len(self.tasks_idepends)):
-            idepends = self.tasks_idepends[taskid]
-            for (idependid, idependtask) in idepends:
-                if idependid == targetid:
-                    self.fail_fnid(self.tasks_fnid[taskid], missing_list)
-
-        if self.abort and targetid in self.external_targets:
-            target = self.build_names_index[targetid]
+            missing_list = [target] + missing_list
+        logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
+        self.failed_deps.append(target)
+        dependees = self.get_dependees(target)
+        for fn in dependees:
+            self.fail_fn(fn, missing_list)
+        for tid in self.taskentries:
+            for (idepend, idependtask) in self.taskentries[tid].idepends:
+                if idepend == target:
+                    fn = tid.rsplit(":",1)[0]
+                    self.fail_fn(fn, missing_list)
+
+        if self.abort and target in self.external_targets:
             logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
             raise bb.providers.NoProvider(target)
 
-    def remove_runtarget(self, targetid, missing_list=None):
+    def remove_runtarget(self, target, missing_list=None):
         """
         Mark a run target as failed (unbuildable)
         Trigger removal of any files that have this as a dependency
         """
         if not missing_list:
-            missing_list = [self.run_names_index[targetid]]
+            missing_list = [target]
         else:
-            missing_list = [self.run_names_index[targetid]] + missing_list
-
-        logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.run_names_index[targetid], missing_list)
-        self.failed_rdeps.append(targetid)
-        dependees = self.get_rdependees(targetid)
-        for fnid in dependees:
-            self.fail_fnid(fnid, missing_list)
-        for taskid in range(len(self.tasks_irdepends)):
-            irdepends = self.tasks_irdepends[taskid]
-            for (idependid, idependtask) in irdepends:
-                if idependid == targetid:
-                    self.fail_fnid(self.tasks_fnid[taskid], missing_list)
+            missing_list = [target] + missing_list
+
+        logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
+        self.failed_rdeps.append(target)
+        dependees = self.get_rdependees(target)
+        for fn in dependees:
+            self.fail_fn(fn, missing_list)
+        for tid in self.taskentries:
+            for (idepend, idependtask) in self.taskentries[tid].irdepends:
+                if idepend == target:
+                    fn = tid.rsplit(":",1)[0]
+                    self.fail_fn(fn, missing_list)
 
     def add_unresolved(self, cfgData, dataCache):
         """
@@ -620,17 +515,16 @@ class TaskData:
                     self.add_provider_internal(cfgData, dataCache, target)
                     added = added + 1
                 except bb.providers.NoProvider:
-                    targetid = self.getbuild_id(target)
-                    if self.abort and targetid in self.external_targets and not self.allowincomplete:
+                    if self.abort and target in self.external_targets and not self.allowincomplete:
                         raise
                     if not self.allowincomplete:
-                        self.remove_buildtarget(targetid)
+                        self.remove_buildtarget(target)
             for target in self.get_unresolved_run_targets(dataCache):
                 try:
                     self.add_rprovider(cfgData, dataCache, target)
                     added = added + 1
                 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
-                    self.remove_runtarget(self.getrun_id(target))
+                    self.remove_runtarget(target)
             logger.debug(1, "Resolved " + str(added) + " extra dependencies")
             if added == 0:
                 break
@@ -638,13 +532,13 @@ class TaskData:
 
     def get_providermap(self, prefix=None):
         provmap = {}
-        for name in self.build_names_index:
+        for name in self.build_targets:
             if prefix and not name.startswith(prefix):
                 continue
             if self.have_build_target(name):
                 provider = self.get_provider(name)
                 if provider:
-                    provmap[name] = self.fn_index[provider[0]]
+                    provmap[name] = provider[0]
         return provmap
 
     def dump_data(self):
@@ -652,39 +546,37 @@ class TaskData:
         Dump some debug information on the internal data structures
         """
         logger.debug(3, "build_names:")
-        logger.debug(3, ", ".join(self.build_names_index))
+        logger.debug(3, ", ".join(self.build_targets))
 
         logger.debug(3, "run_names:")
-        logger.debug(3, ", ".join(self.run_names_index))
+        logger.debug(3, ", ".join(self.run_targets))
 
         logger.debug(3, "build_targets:")
-        for buildid in range(len(self.build_names_index)):
-            target = self.build_names_index[buildid]
+        for target in self.build_targets:
             targets = "None"
-            if buildid in self.build_targets:
-                targets = self.build_targets[buildid]
-            logger.debug(3, " (%s)%s: %s", buildid, target, targets)
+            if target in self.build_targets:
+                targets = self.build_targets[target]
+            logger.debug(3, " %s: %s", target, targets)
 
         logger.debug(3, "run_targets:")
-        for runid in range(len(self.run_names_index)):
-            target = self.run_names_index[runid]
+        for target in self.run_targets:
             targets = "None"
-            if runid in self.run_targets:
-                targets = self.run_targets[runid]
-            logger.debug(3, " (%s)%s: %s", runid, target, targets)
+            if target in self.run_targets:
+                targets = self.run_targets[target]
+            logger.debug(3, " %s: %s", target, targets)
 
         logger.debug(3, "tasks:")
-        for task in range(len(self.tasks_name)):
-            logger.debug(3, " (%s)%s - %s: %s",
-                       task,
-                       self.fn_index[self.tasks_fnid[task]],
-                       self.tasks_name[task],
-                       self.tasks_tdepends[task])
+        for tid in self.taskentries:
+            logger.debug(3, " %s: %s %s %s",
+                       tid,
+                       self.taskentries[tid].idepends,
+                       self.taskentries[tid].irdepends,
+                       self.taskentries[tid].tdepends)
 
         logger.debug(3, "dependency ids (per fn):")
-        for fnid in self.depids:
-            logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.depids[fnid])
+        for fn in self.depids:
+            logger.debug(3, " %s: %s", fn, self.depids[fn])
 
         logger.debug(3, "runtime dependency ids (per fn):")
-        for fnid in self.rdepids:
-            logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.rdepids[fnid])
+        for fn in self.rdepids:
+            logger.debug(3, " %s: %s", fn, self.rdepids[fn])