summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2016-06-12 23:55:48 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-06-15 08:35:07 +0100
commitaf04a52bf7d7ae94c187e85028a80dee6d7853f0 (patch)
treed0c65de172d7ded2e1ba28f09c271da97e10cbe9
parent29b4fb17966bfaec74b41cce533725c04ec1078f (diff)
downloadpoky-af04a52bf7d7ae94c187e85028a80dee6d7853f0.tar.gz
bitbake: taskdata/runqueue: Rewrite without use of ID indirection
I'm not sure what possesed me when I wrote this code originally but its indirection of everyting to use numeric IDs and position dependent lists is horrific. Given the way python internals work, its completely and utterly pointless from performance perspective. It also makes the code hard to understand and debug since any numeric ID has to be translated into something human readable. The hard part is that the IDs are infectous and spread from taskdata into runqueue and even partly into cooker for the dependency graph processing. The only real way to deal with this is to convert everything to use a more sane data structure. This patch: * Uses "<fn>:<taskname>" as the ID for tasks rather than a number * Changes to dict() based structures rather than position dependent lists * Drops the build name, runtime name and filename ID indexes On the most part there shouldn't be user visible changes. Sadly we did leak datastructures to the setscene verify function which has to be rewritten. To handle this, the variable name used to specifiy the version changes from BB_SETSCENE_VERIFY_FUNCTION to BB_SETSCENE_VERIFY_FUNCTION2 allowing multiple versions of bitbake to work with suitably written metadata. Anyone with custom schedulers may also need to change them. I believe the benefits in code readability and easier debugging far outweigh those issues though. It also means we have a saner codebase to add multiconfig support on top of. During development, I did have some of the original code coexisting with the new data stores to allow comparision of the data and check it was working correcty, particuarly for taskdata. I have also compared task-depends.dot files before and after the change. There should be no functionality changes in this patch, its purely a data structure change and that is visible in the patch. (Bitbake rev: 2c88afb60da54e58f555411a7bd7b006b0c29306) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cooker.py72
-rw-r--r--bitbake/lib/bb/runqueue.py958
-rw-r--r--bitbake/lib/bb/taskdata.py412
3 files changed, 639 insertions, 803 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index ce818f3e59..907565401d 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -623,9 +623,7 @@ class BBCooker:
623 623
624 taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) 624 taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
625 625
626 targetid = taskdata.getbuild_id(pkgs_to_build[0]) 626 fn = taskdata.build_targets[pkgs_to_build[0]][0]
627 fnid = taskdata.build_targets[targetid][0]
628 fn = taskdata.fn_index[fnid]
629 else: 627 else:
630 envdata = self.data 628 envdata = self.data
631 629
@@ -714,7 +712,7 @@ class BBCooker:
714 712
715 713
716 def buildDependTree(self, rq, taskdata): 714 def buildDependTree(self, rq, taskdata):
717 seen_fnids = [] 715 seen_fns = []
718 depend_tree = {} 716 depend_tree = {}
719 depend_tree["depends"] = {} 717 depend_tree["depends"] = {}
720 depend_tree["tdepends"] = {} 718 depend_tree["tdepends"] = {}
@@ -732,10 +730,9 @@ class BBCooker:
732 version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] 730 version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
733 depend_tree['providermap'][name] = (pn, version) 731 depend_tree['providermap'][name] = (pn, version)
734 732
735 for task in range(len(rq.rqdata.runq_fnid)): 733 for tid in rq.rqdata.runtaskentries:
736 taskname = rq.rqdata.runq_task[task] 734 taskname = bb.runqueue.taskname_from_tid(tid)
737 fnid = rq.rqdata.runq_fnid[task] 735 fn = bb.runqueue.fn_from_tid(tid)
738 fn = taskdata.fn_index[fnid]
739 pn = self.recipecache.pkg_fn[fn] 736 pn = self.recipecache.pkg_fn[fn]
740 version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn] 737 version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
741 if pn not in depend_tree["pn"]: 738 if pn not in depend_tree["pn"]:
@@ -756,24 +753,24 @@ class BBCooker:
756 depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn] 753 depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
757 754
758 755
759 for dep in rq.rqdata.runq_depends[task]: 756 for dep in rq.rqdata.runtaskentries[tid].depends:
760 depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]] 757 depfn = bb.runqueue.fn_from_tid(dep)
761 deppn = self.recipecache.pkg_fn[depfn] 758 deppn = self.recipecache.pkg_fn[depfn]
762 dotname = "%s.%s" % (pn, rq.rqdata.runq_task[task]) 759 dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(dep))
763 if not dotname in depend_tree["tdepends"]: 760 if not dotname in depend_tree["tdepends"]:
764 depend_tree["tdepends"][dotname] = [] 761 depend_tree["tdepends"][dotname] = []
765 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.rqdata.runq_task[dep])) 762 depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
766 if fnid not in seen_fnids: 763 if fn not in seen_fns:
767 seen_fnids.append(fnid) 764 seen_fns.append(fn)
768 packages = [] 765 packages = []
769 766
770 depend_tree["depends"][pn] = [] 767 depend_tree["depends"][pn] = []
771 for dep in taskdata.depids[fnid]: 768 for dep in taskdata.depids[fn]:
772 depend_tree["depends"][pn].append(taskdata.build_names_index[dep]) 769 depend_tree["depends"][pn].append(dep)
773 770
774 depend_tree["rdepends-pn"][pn] = [] 771 depend_tree["rdepends-pn"][pn] = []
775 for rdep in taskdata.rdepids[fnid]: 772 for rdep in taskdata.rdepids[fn]:
776 depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep]) 773 depend_tree["rdepends-pn"][pn].append(rdep)
777 774
778 rdepends = self.recipecache.rundeps[fn] 775 rdepends = self.recipecache.rundeps[fn]
779 for package in rdepends: 776 for package in rdepends:
@@ -805,12 +802,8 @@ class BBCooker:
805 Create a dependency tree of pkgs_to_build, returning the data. 802 Create a dependency tree of pkgs_to_build, returning the data.
806 """ 803 """
807 _, taskdata = self.prepareTreeData(pkgs_to_build, task) 804 _, taskdata = self.prepareTreeData(pkgs_to_build, task)
808 tasks_fnid = []
809 if len(taskdata.tasks_name) != 0:
810 for task in range(len(taskdata.tasks_name)):
811 tasks_fnid.append(taskdata.tasks_fnid[task])
812 805
813 seen_fnids = [] 806 seen_fns = []
814 depend_tree = {} 807 depend_tree = {}
815 depend_tree["depends"] = {} 808 depend_tree["depends"] = {}
816 depend_tree["pn"] = {} 809 depend_tree["pn"] = {}
@@ -825,9 +818,8 @@ class BBCooker:
825 cachefields = getattr(cache_class, 'cachefields', []) 818 cachefields = getattr(cache_class, 'cachefields', [])
826 extra_info = extra_info + cachefields 819 extra_info = extra_info + cachefields
827 820
828 for task in range(len(tasks_fnid)): 821 for tid in taskdata.taskentries:
829 fnid = tasks_fnid[task] 822 fn = bb.runqueue.fn_from_tid(tid)
830 fn = taskdata.fn_index[fnid]
831 pn = self.recipecache.pkg_fn[fn] 823 pn = self.recipecache.pkg_fn[fn]
832 824
833 if pn not in depend_tree["pn"]: 825 if pn not in depend_tree["pn"]:
@@ -843,33 +835,27 @@ class BBCooker:
843 for ei in extra_info: 835 for ei in extra_info:
844 depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn] 836 depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
845 837
846 if fnid not in seen_fnids: 838 if fn not in seen_fns:
847 seen_fnids.append(fnid) 839 seen_fns.append(fn)
848 840
849 depend_tree["depends"][pn] = [] 841 depend_tree["depends"][pn] = []
850 for dep in taskdata.depids[fnid]: 842 for item in taskdata.depids[fn]:
851 item = taskdata.build_names_index[dep]
852 pn_provider = "" 843 pn_provider = ""
853 targetid = taskdata.getbuild_id(item) 844 if dep in taskdata.build_targets and taskdata.build_targets[dep]:
854 if targetid in taskdata.build_targets and taskdata.build_targets[targetid]: 845 fn_provider = taskdata.build_targets[dep][0]
855 id = taskdata.build_targets[targetid][0]
856 fn_provider = taskdata.fn_index[id]
857 pn_provider = self.recipecache.pkg_fn[fn_provider] 846 pn_provider = self.recipecache.pkg_fn[fn_provider]
858 else: 847 else:
859 pn_provider = item 848 pn_provider = item
860 depend_tree["depends"][pn].append(pn_provider) 849 depend_tree["depends"][pn].append(pn_provider)
861 850
862 depend_tree["rdepends-pn"][pn] = [] 851 depend_tree["rdepends-pn"][pn] = []
863 for rdep in taskdata.rdepids[fnid]: 852 for rdep in taskdata.rdepids[fn]:
864 item = taskdata.run_names_index[rdep]
865 pn_rprovider = "" 853 pn_rprovider = ""
866 targetid = taskdata.getrun_id(item) 854 if rdep in taskdata.run_targets and taskdata.run_targets[rdep]:
867 if targetid in taskdata.run_targets and taskdata.run_targets[targetid]: 855 fn_rprovider = taskdata.run_targets[rdep][0]
868 id = taskdata.run_targets[targetid][0]
869 fn_rprovider = taskdata.fn_index[id]
870 pn_rprovider = self.recipecache.pkg_fn[fn_rprovider] 856 pn_rprovider = self.recipecache.pkg_fn[fn_rprovider]
871 else: 857 else:
872 pn_rprovider = item 858 pn_rprovider = rdep
873 depend_tree["rdepends-pn"][pn].append(pn_rprovider) 859 depend_tree["rdepends-pn"][pn].append(pn_rprovider)
874 860
875 depend_tree["rdepends-pkg"].update(rdepends) 861 depend_tree["rdepends-pkg"].update(rdepends)
@@ -1350,7 +1336,7 @@ class BBCooker:
1350 return False 1336 return False
1351 1337
1352 if not retval: 1338 if not retval:
1353 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures, interrupted), self.expanded_data) 1339 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.expanded_data)
1354 self.command.finishAsyncCommand(msg) 1340 self.command.finishAsyncCommand(msg)
1355 return False 1341 return False
1356 if retval is True: 1342 if retval is True:
@@ -1386,7 +1372,7 @@ class BBCooker:
1386 return False 1372 return False
1387 1373
1388 if not retval: 1374 if not retval:
1389 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets, failures, interrupted), self.data) 1375 bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.data)
1390 self.command.finishAsyncCommand(msg) 1376 self.command.finishAsyncCommand(msg)
1391 return False 1377 return False
1392 if retval is True: 1378 if retval is True:
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index de618ff1d8..b32e8e694d 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -42,6 +42,12 @@ logger = logging.getLogger("BitBake.RunQueue")
42 42
43__find_md5__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])' ) 43__find_md5__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])' )
44 44
45def fn_from_tid(tid):
46 return tid.rsplit(":", 1)[0]
47
48def taskname_from_tid(tid):
49 return tid.rsplit(":", 1)[1]
50
45class RunQueueStats: 51class RunQueueStats:
46 """ 52 """
47 Holds statistics on the tasks handled by the associated runQueue 53 Holds statistics on the tasks handled by the associated runQueue
@@ -97,19 +103,18 @@ class RunQueueScheduler(object):
97 """ 103 """
98 self.rq = runqueue 104 self.rq = runqueue
99 self.rqdata = rqdata 105 self.rqdata = rqdata
100 self.numTasks = len(self.rqdata.runq_fnid) 106 self.numTasks = len(self.rqdata.runtaskentries)
101 107
102 self.prio_map = [] 108 self.prio_map = [self.rqdata.runtaskentries.keys()]
103 self.prio_map.extend(range(self.numTasks))
104 109
105 self.buildable = [] 110 self.buildable = []
106 self.stamps = {} 111 self.stamps = {}
107 for taskid in range(self.numTasks): 112 for tid in self.rqdata.runtaskentries:
108 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]] 113 fn = fn_from_tid(tid)
109 taskname = self.rqdata.runq_task[taskid] 114 taskname = taskname_from_tid(tid)
110 self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) 115 self.stamps[tid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
111 if taskid in self.rq.runq_buildable: 116 if tid in self.rq.runq_buildable:
112 self.buildable.append(taskid) 117 self.buildable.append(tid)
113 118
114 self.rev_prio_map = None 119 self.rev_prio_map = None
115 120
@@ -121,26 +126,26 @@ class RunQueueScheduler(object):
121 if not self.buildable: 126 if not self.buildable:
122 return None 127 return None
123 if len(self.buildable) == 1: 128 if len(self.buildable) == 1:
124 taskid = self.buildable[0] 129 tid = self.buildable[0]
125 stamp = self.stamps[taskid] 130 stamp = self.stamps[tid]
126 if stamp not in self.rq.build_stamps.values(): 131 if stamp not in self.rq.build_stamps.values():
127 return taskid 132 return tid
128 133
129 if not self.rev_prio_map: 134 if not self.rev_prio_map:
130 self.rev_prio_map = list(range(self.numTasks)) 135 self.rev_prio_map = {}
131 for taskid in range(self.numTasks): 136 for tid in self.rqdata.runtaskentries:
132 self.rev_prio_map[self.prio_map[taskid]] = taskid 137 self.rev_prio_map[tid] = self.prio_map.index(tid)
133 138
134 best = None 139 best = None
135 bestprio = None 140 bestprio = None
136 for taskid in self.buildable: 141 for tid in self.buildable:
137 prio = self.rev_prio_map[taskid] 142 prio = self.rev_prio_map[tid]
138 if bestprio is None or bestprio > prio: 143 if bestprio is None or bestprio > prio:
139 stamp = self.stamps[taskid] 144 stamp = self.stamps[tid]
140 if stamp in self.rq.build_stamps.values(): 145 if stamp in self.rq.build_stamps.values():
141 continue 146 continue
142 bestprio = prio 147 bestprio = prio
143 best = taskid 148 best = tid
144 149
145 return best 150 return best
146 151
@@ -167,14 +172,17 @@ class RunQueueSchedulerSpeed(RunQueueScheduler):
167 """ 172 """
168 RunQueueScheduler.__init__(self, runqueue, rqdata) 173 RunQueueScheduler.__init__(self, runqueue, rqdata)
169 174
170 sortweight = sorted(copy.deepcopy(self.rqdata.runq_weight)) 175 weights = {}
171 copyweight = copy.deepcopy(self.rqdata.runq_weight) 176 for tid in self.rqdata.runtaskentries:
172 self.prio_map = [] 177 weight = self.rqdata.runtaskentries[tid].weight
178 if not weight in weights:
179 weights[weight] = []
180 weights[weight].append(tid)
173 181
174 for weight in sortweight: 182 self.prio_map = []
175 idx = copyweight.index(weight) 183 for weight in sorted(weights):
176 self.prio_map.append(idx) 184 for w in weights[weight]:
177 copyweight[idx] = -1 185 self.prio_map.append(w)
178 186
179 self.prio_map.reverse() 187 self.prio_map.reverse()
180 188
@@ -191,25 +199,33 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
191 def __init__(self, runqueue, rqdata): 199 def __init__(self, runqueue, rqdata):
192 RunQueueSchedulerSpeed.__init__(self, runqueue, rqdata) 200 RunQueueSchedulerSpeed.__init__(self, runqueue, rqdata)
193 201
194 #FIXME - whilst this groups all fnids together it does not reorder the 202 #FIXME - whilst this groups all fns together it does not reorder the
195 #fnid groups optimally. 203 #fn groups optimally.
196 204
197 basemap = copy.deepcopy(self.prio_map) 205 basemap = copy.deepcopy(self.prio_map)
198 self.prio_map = [] 206 self.prio_map = []
199 while (len(basemap) > 0): 207 while (len(basemap) > 0):
200 entry = basemap.pop(0) 208 entry = basemap.pop(0)
201 self.prio_map.append(entry) 209 self.prio_map.append(entry)
202 fnid = self.rqdata.runq_fnid[entry] 210 fn = fn_from_tid(entry)
203 todel = [] 211 todel = []
204 for entry in basemap: 212 for entry in basemap:
205 entry_fnid = self.rqdata.runq_fnid[entry] 213 entry_fn = fn_from_tid(entry)
206 if entry_fnid == fnid: 214 if entry_fn == fn:
207 todel.append(basemap.index(entry)) 215 todel.append(basemap.index(entry))
208 self.prio_map.append(entry) 216 self.prio_map.append(entry)
209 todel.reverse() 217 todel.reverse()
210 for idx in todel: 218 for idx in todel:
211 del basemap[idx] 219 del basemap[idx]
212 220
221class RunTaskEntry(object):
222 def __init__(self):
223 self.depends = set()
224 self.revdeps = set()
225 self.hash = None
226 self.task = None
227 self.weight = 1
228
213class RunQueueData: 229class RunQueueData:
214 """ 230 """
215 BitBake Run Queue implementation 231 BitBake Run Queue implementation
@@ -228,48 +244,29 @@ class RunQueueData:
228 self.reset() 244 self.reset()
229 245
230 def reset(self): 246 def reset(self):
231 self.runq_fnid = [] 247 self.runtaskentries = {}
232 self.runq_task = []
233 self.runq_depends = []
234 self.runq_revdeps = []
235 self.runq_hash = []
236 248
237 def runq_depends_names(self, ids): 249 def runq_depends_names(self, ids):
238 import re 250 import re
239 ret = [] 251 ret = []
240 for id in self.runq_depends[ids]: 252 for id in ids:
241 nam = os.path.basename(self.get_user_idstring(id)) 253 nam = os.path.basename(id)
242 nam = re.sub("_[^,]*,", ",", nam) 254 nam = re.sub("_[^,]*,", ",", nam)
243 ret.extend([nam]) 255 ret.extend([nam])
244 return ret 256 return ret
245 257
246 def get_task_name(self, task): 258 def get_task_hash(self, tid):
247 return self.runq_task[task] 259 return self.runtaskentries[tid].hash
248
249 def get_task_file(self, task):
250 return self.taskData.fn_index[self.runq_fnid[task]]
251 260
252 def get_task_hash(self, task): 261 def get_user_idstring(self, tid, task_name_suffix = ""):
253 return self.runq_hash[task] 262 return tid + task_name_suffix
254
255 def get_user_idstring(self, task, task_name_suffix = ""):
256 fn = self.taskData.fn_index[self.runq_fnid[task]]
257 taskname = self.runq_task[task] + task_name_suffix
258 return "%s, %s" % (fn, taskname)
259 263
260 def get_short_user_idstring(self, task, task_name_suffix = ""): 264 def get_short_user_idstring(self, task, task_name_suffix = ""):
261 fn = self.taskData.fn_index[self.runq_fnid[task]] 265 fn = fn_from_tid(task)
262 pn = self.dataCache.pkg_fn[fn] 266 pn = self.dataCache.pkg_fn[fn]
263 taskname = self.runq_task[task] + task_name_suffix 267 taskname = taskname_from_tid(task) + task_name_suffix
264 return "%s:%s" % (pn, taskname) 268 return "%s:%s" % (pn, taskname)
265 269
266
267 def get_task_id(self, fnid, taskname):
268 for listid in range(len(self.runq_fnid)):
269 if self.runq_fnid[listid] == fnid and self.runq_task[listid] == taskname:
270 return listid
271 return None
272
273 def circular_depchains_handler(self, tasks): 270 def circular_depchains_handler(self, tasks):
274 """ 271 """
275 Some tasks aren't buildable, likely due to circular dependency issues. 272 Some tasks aren't buildable, likely due to circular dependency issues.
@@ -314,11 +311,11 @@ class RunQueueData:
314 return True 311 return True
315 return False 312 return False
316 313
317 def find_chains(taskid, prev_chain): 314 def find_chains(tid, prev_chain):
318 prev_chain.append(taskid) 315 prev_chain.append(tid)
319 total_deps = [] 316 total_deps = []
320 total_deps.extend(self.runq_revdeps[taskid]) 317 total_deps.extend(self.runtaskentries[tid].revdeps)
321 for revdep in self.runq_revdeps[taskid]: 318 for revdep in self.runtaskentries[tid].revdeps:
322 if revdep in prev_chain: 319 if revdep in prev_chain:
323 idx = prev_chain.index(revdep) 320 idx = prev_chain.index(revdep)
324 # To prevent duplicates, reorder the chain to start with the lowest taskid 321 # To prevent duplicates, reorder the chain to start with the lowest taskid
@@ -329,7 +326,7 @@ class RunQueueData:
329 valid_chains.append(new_chain) 326 valid_chains.append(new_chain)
330 msgs.append("Dependency loop #%d found:\n" % len(valid_chains)) 327 msgs.append("Dependency loop #%d found:\n" % len(valid_chains))
331 for dep in new_chain: 328 for dep in new_chain:
332 msgs.append(" Task %s (%s) (dependent Tasks %s)\n" % (dep, self.get_user_idstring(dep), self.runq_depends_names(dep))) 329 msgs.append(" Task %s (dependent Tasks %s)\n" % (dep, self.runq_depends_names(self.runtaskentries[dep].depends)))
333 msgs.append("\n") 330 msgs.append("\n")
334 if len(valid_chains) > 10: 331 if len(valid_chains) > 10:
335 msgs.append("Aborted dependency loops search after 10 matches.\n") 332 msgs.append("Aborted dependency loops search after 10 matches.\n")
@@ -366,25 +363,25 @@ class RunQueueData:
366 possible to execute due to circular dependencies. 363 possible to execute due to circular dependencies.
367 """ 364 """
368 365
369 numTasks = len(self.runq_fnid) 366 numTasks = len(self.runtaskentries)
370 weight = [] 367 weight = {}
371 deps_left = [] 368 deps_left = {}
372 task_done = [] 369 task_done = {}
373 370
374 for listid in range(numTasks): 371 for tid in self.runtaskentries:
375 task_done.append(False) 372 task_done[tid] = False
376 weight.append(1) 373 weight[tid] = 1
377 deps_left.append(len(self.runq_revdeps[listid])) 374 deps_left[tid] = len(self.runtaskentries[tid].revdeps)
378 375
379 for listid in endpoints: 376 for tid in endpoints:
380 weight[listid] = 10 377 weight[tid] = 10
381 task_done[listid] = True 378 task_done[tid] = True
382 379
383 while True: 380 while True:
384 next_points = [] 381 next_points = []
385 for listid in endpoints: 382 for tid in endpoints:
386 for revdep in self.runq_depends[listid]: 383 for revdep in self.runtaskentries[tid].depends:
387 weight[revdep] = weight[revdep] + weight[listid] 384 weight[revdep] = weight[revdep] + weight[tid]
388 deps_left[revdep] = deps_left[revdep] - 1 385 deps_left[revdep] = deps_left[revdep] - 1
389 if deps_left[revdep] == 0: 386 if deps_left[revdep] == 0:
390 next_points.append(revdep) 387 next_points.append(revdep)
@@ -395,14 +392,15 @@ class RunQueueData:
395 392
396 # Circular dependency sanity check 393 # Circular dependency sanity check
397 problem_tasks = [] 394 problem_tasks = []
398 for task in range(numTasks): 395 for tid in self.runtaskentries:
399 if task_done[task] is False or deps_left[task] != 0: 396 if task_done[tid] is False or deps_left[tid] != 0:
400 problem_tasks.append(task) 397 problem_tasks.append(tid)
401 logger.debug(2, "Task %s (%s) is not buildable", task, self.get_user_idstring(task)) 398 logger.debug(2, "Task %s is not buildable", tid)
402 logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n", task_done[task], deps_left[task]) 399 logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n", task_done[tid], deps_left[tid])
400 self.runtaskentries[tid].weight = weight[tid]
403 401
404 if problem_tasks: 402 if problem_tasks:
405 message = "Unbuildable tasks were found.\n" 403 message = "%s unbuildable tasks were found.\n" % len(problem_tasks)
406 message = message + "These are usually caused by circular dependencies and any circular dependency chains found will be printed below. Increase the debug level to see a list of unbuildable tasks.\n\n" 404 message = message + "These are usually caused by circular dependencies and any circular dependency chains found will be printed below. Increase the debug level to see a list of unbuildable tasks.\n\n"
407 message = message + "Identifying dependency loops (this may take a short while)...\n" 405 message = message + "Identifying dependency loops (this may take a short while)...\n"
408 logger.error(message) 406 logger.error(message)
@@ -422,14 +420,14 @@ class RunQueueData:
422 to optimise the execution order. 420 to optimise the execution order.
423 """ 421 """
424 422
425 runq_build = [] 423 runq_build = {}
426 recursivetasks = {} 424 recursivetasks = {}
427 recursiveitasks = {} 425 recursiveitasks = {}
428 recursivetasksselfref = set() 426 recursivetasksselfref = set()
429 427
430 taskData = self.taskData 428 taskData = self.taskData
431 429
432 if len(taskData.tasks_name) == 0: 430 if len(taskData.taskentries) == 0:
433 # Nothing to do 431 # Nothing to do
434 return 0 432 return 0
435 433
@@ -447,160 +445,159 @@ class RunQueueData:
447 # rdeptast, recrdeptask, idepends). 445 # rdeptast, recrdeptask, idepends).
448 446
449 def add_build_dependencies(depids, tasknames, depends): 447 def add_build_dependencies(depids, tasknames, depends):
450 for depid in depids: 448 for depname in depids:
451 # Won't be in build_targets if ASSUME_PROVIDED 449 # Won't be in build_targets if ASSUME_PROVIDED
452 if depid not in taskData.build_targets: 450 if depname not in taskData.build_targets or not taskData.build_targets[depname]:
453 continue 451 continue
454 depdata = taskData.build_targets[depid][0] 452 depdata = taskData.build_targets[depname][0]
455 if depdata is None: 453 if depdata is None:
456 continue 454 continue
457 for taskname in tasknames: 455 for taskname in tasknames:
458 taskid = taskData.gettask_id_fromfnid(depdata, taskname) 456 t = depdata + ":" + taskname
459 if taskid is not None: 457 if t in taskData.taskentries:
460 depends.add(taskid) 458 depends.add(t)
461 459
462 def add_runtime_dependencies(depids, tasknames, depends): 460 def add_runtime_dependencies(depids, tasknames, depends):
463 for depid in depids: 461 for depname in depids:
464 if depid not in taskData.run_targets: 462 if depname not in taskData.run_targets or not taskData.run_targets[depname]:
465 continue 463 continue
466 depdata = taskData.run_targets[depid][0] 464 depdata = taskData.run_targets[depname][0]
467 if depdata is None: 465 if depdata is None:
468 continue 466 continue
469 for taskname in tasknames: 467 for taskname in tasknames:
470 taskid = taskData.gettask_id_fromfnid(depdata, taskname) 468 t = depdata + ":" + taskname
471 if taskid is not None: 469 if t in taskData.taskentries:
472 depends.add(taskid) 470 depends.add(t)
473 471
474 def add_resolved_dependencies(depids, tasknames, depends): 472 def add_resolved_dependencies(fn, tasknames, depends):
475 for depid in depids: 473 for taskname in tasknames:
476 for taskname in tasknames: 474 tid = fn + ":" + taskname
477 taskid = taskData.gettask_id_fromfnid(depid, taskname) 475 if tid in self.runtaskentries:
478 if taskid is not None: 476 depends.add(tid)
479 depends.add(taskid) 477
478 for tid in taskData.taskentries:
479
480 fn = fn_from_tid(tid)
481 taskname = taskname_from_tid(tid)
480 482
481 for task in range(len(taskData.tasks_name)):
482 depends = set() 483 depends = set()
483 fnid = taskData.tasks_fnid[task]
484 fn = taskData.fn_index[fnid]
485 task_deps = self.dataCache.task_deps[fn] 484 task_deps = self.dataCache.task_deps[fn]
486 485
487 #logger.debug(2, "Processing %s:%s", fn, taskData.tasks_name[task]) 486 self.runtaskentries[tid] = RunTaskEntry()
488 487
489 if fnid not in taskData.failed_fnids: 488 #logger.debug(2, "Processing %s:%s", fn, taskname)
489
490 if fn not in taskData.failed_fns:
490 491
491 # Resolve task internal dependencies 492 # Resolve task internal dependencies
492 # 493 #
493 # e.g. addtask before X after Y 494 # e.g. addtask before X after Y
494 depends = set(taskData.tasks_tdepends[task]) 495 depends.update(taskData.taskentries[tid].tdepends)
495 496
496 # Resolve 'deptask' dependencies 497 # Resolve 'deptask' dependencies
497 # 498 #
498 # e.g. do_sometask[deptask] = "do_someothertask" 499 # e.g. do_sometask[deptask] = "do_someothertask"
499 # (makes sure sometask runs after someothertask of all DEPENDS) 500 # (makes sure sometask runs after someothertask of all DEPENDS)
500 if 'deptask' in task_deps and taskData.tasks_name[task] in task_deps['deptask']: 501 if 'deptask' in task_deps and taskname in task_deps['deptask']:
501 tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() 502 tasknames = task_deps['deptask'][taskname].split()
502 add_build_dependencies(taskData.depids[fnid], tasknames, depends) 503 add_build_dependencies(taskData.depids[fn], tasknames, depends)
503 504
504 # Resolve 'rdeptask' dependencies 505 # Resolve 'rdeptask' dependencies
505 # 506 #
506 # e.g. do_sometask[rdeptask] = "do_someothertask" 507 # e.g. do_sometask[rdeptask] = "do_someothertask"
507 # (makes sure sometask runs after someothertask of all RDEPENDS) 508 # (makes sure sometask runs after someothertask of all RDEPENDS)
508 if 'rdeptask' in task_deps and taskData.tasks_name[task] in task_deps['rdeptask']: 509 if 'rdeptask' in task_deps and taskname in task_deps['rdeptask']:
509 tasknames = task_deps['rdeptask'][taskData.tasks_name[task]].split() 510 tasknames = task_deps['rdeptask'][taskname].split()
510 add_runtime_dependencies(taskData.rdepids[fnid], tasknames, depends) 511 add_runtime_dependencies(taskData.rdepids[fn], tasknames, depends)
511 512
512 # Resolve inter-task dependencies 513 # Resolve inter-task dependencies
513 # 514 #
514 # e.g. do_sometask[depends] = "targetname:do_someothertask" 515 # e.g. do_sometask[depends] = "targetname:do_someothertask"
515 # (makes sure sometask runs after targetname's someothertask) 516 # (makes sure sometask runs after targetname's someothertask)
516 idepends = taskData.tasks_idepends[task] 517 idepends = taskData.taskentries[tid].idepends
517 for (depid, idependtask) in idepends: 518 for (depname, idependtask) in idepends:
518 if depid in taskData.build_targets and not depid in taskData.failed_deps: 519 if depname in taskData.build_targets and taskData.build_targets[depname] and not depname in taskData.failed_deps:
519 # Won't be in build_targets if ASSUME_PROVIDED 520 # Won't be in build_targets if ASSUME_PROVIDED
520 depdata = taskData.build_targets[depid][0] 521 depdata = taskData.build_targets[depname][0]
521 if depdata is not None: 522 if depdata is not None:
522 taskid = taskData.gettask_id_fromfnid(depdata, idependtask) 523 t = depdata + ":" + idependtask
523 if taskid is None: 524 depends.add(t)
524 bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in %s" % (taskData.tasks_name[task], fn, idependtask, taskData.fn_index[depdata])) 525 if t not in taskData.taskentries:
525 depends.add(taskid) 526 bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in %s" % (taskname, fn, idependtask, depdata))
526 irdepends = taskData.tasks_irdepends[task] 527 irdepends = taskData.taskentries[tid].irdepends
527 for (depid, idependtask) in irdepends: 528 for (depname, idependtask) in irdepends:
528 if depid in taskData.run_targets: 529 if depname in taskData.run_targets:
529 # Won't be in run_targets if ASSUME_PROVIDED 530 # Won't be in run_targets if ASSUME_PROVIDED
530 depdata = taskData.run_targets[depid][0] 531 depdata = taskData.run_targets[depname][0]
531 if depdata is not None: 532 if depdata is not None:
532 taskid = taskData.gettask_id_fromfnid(depdata, idependtask) 533 t = depdata + ":" + idependtask
533 if taskid is None: 534 depends.add(t)
534 bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s in %s" % (taskData.tasks_name[task], fn, idependtask, taskData.fn_index[depdata])) 535 if t not in taskData.taskentries:
535 depends.add(taskid) 536 bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s in %s" % (taskname, fn, idependtask, depdata))
536 537
537 # Resolve recursive 'recrdeptask' dependencies (Part A) 538 # Resolve recursive 'recrdeptask' dependencies (Part A)
538 # 539 #
539 # e.g. do_sometask[recrdeptask] = "do_someothertask" 540 # e.g. do_sometask[recrdeptask] = "do_someothertask"
540 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) 541 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
541 # We cover the recursive part of the dependencies below 542 # We cover the recursive part of the dependencies below
542 if 'recrdeptask' in task_deps and taskData.tasks_name[task] in task_deps['recrdeptask']: 543 if 'recrdeptask' in task_deps and taskname in task_deps['recrdeptask']:
543 tasknames = task_deps['recrdeptask'][taskData.tasks_name[task]].split() 544 tasknames = task_deps['recrdeptask'][taskname].split()
544 recursivetasks[task] = tasknames 545 recursivetasks[tid] = tasknames
545 add_build_dependencies(taskData.depids[fnid], tasknames, depends) 546 add_build_dependencies(taskData.depids[fn], tasknames, depends)
546 add_runtime_dependencies(taskData.rdepids[fnid], tasknames, depends) 547 add_runtime_dependencies(taskData.rdepids[fn], tasknames, depends)
547 if taskData.tasks_name[task] in tasknames: 548 if taskname in tasknames:
548 recursivetasksselfref.add(task) 549 recursivetasksselfref.add(tid)
549 550
550 if 'recideptask' in task_deps and taskData.tasks_name[task] in task_deps['recideptask']: 551 if 'recideptask' in task_deps and taskname in task_deps['recideptask']:
551 recursiveitasks[task] = [] 552 recursiveitasks[tid] = []
552 for t in task_deps['recideptask'][taskData.tasks_name[task]].split(): 553 for t in task_deps['recideptask'][taskname].split():
553 newdep = taskData.gettask_id_fromfnid(fnid, t) 554 newdep = fn + ":" + t
554 recursiveitasks[task].append(newdep) 555 recursiveitasks[tid].append(newdep)
555 556
556 self.runq_fnid.append(taskData.tasks_fnid[task]) 557 self.runtaskentries[tid].depends = depends
557 self.runq_task.append(taskData.tasks_name[task])
558 self.runq_depends.append(depends)
559 self.runq_revdeps.append(set())
560 self.runq_hash.append("")
561
562 runq_build.append(0)
563 558
564 # Resolve recursive 'recrdeptask' dependencies (Part B) 559 # Resolve recursive 'recrdeptask' dependencies (Part B)
565 # 560 #
566 # e.g. do_sometask[recrdeptask] = "do_someothertask" 561 # e.g. do_sometask[recrdeptask] = "do_someothertask"
567 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) 562 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
568 # We need to do this separately since we need all of self.runq_depends to be complete before this is processed 563 # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed
569 extradeps = {} 564 extradeps = {}
570 for task in recursivetasks: 565 for tid in recursivetasks:
571 extradeps[task] = set(self.runq_depends[task]) 566 extradeps[tid] = set(self.runtaskentries[tid].depends)
572 tasknames = recursivetasks[task] 567
568 tasknames = recursivetasks[tid]
573 seendeps = set() 569 seendeps = set()
574 seenfnid = []
575 570
576 def generate_recdeps(t): 571 def generate_recdeps(t):
577 newdeps = set() 572 newdeps = set()
578 add_resolved_dependencies([taskData.tasks_fnid[t]], tasknames, newdeps) 573 add_resolved_dependencies(fn_from_tid(t), tasknames, newdeps)
579 extradeps[task].update(newdeps) 574 extradeps[tid].update(newdeps)
580 seendeps.add(t) 575 seendeps.add(t)
581 newdeps.add(t) 576 newdeps.add(t)
582 for i in newdeps: 577 for i in newdeps:
583 for n in self.runq_depends[i]: 578 task = self.runtaskentries[i].task
579 for n in self.runtaskentries[i].depends:
584 if n not in seendeps: 580 if n not in seendeps:
585 generate_recdeps(n) 581 generate_recdeps(n)
586 generate_recdeps(task) 582 generate_recdeps(tid)
587 583
588 if task in recursiveitasks: 584 if tid in recursiveitasks:
589 for dep in recursiveitasks[task]: 585 for dep in recursiveitasks[tid]:
590 generate_recdeps(dep) 586 generate_recdeps(dep)
591 587
592 # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work 588 # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work
593 for task in recursivetasks: 589 for tid in recursivetasks:
594 extradeps[task].difference_update(recursivetasksselfref) 590 extradeps[tid].difference_update(recursivetasksselfref)
595 591
596 for task in range(len(taskData.tasks_name)): 592 for tid in self.runtaskentries:
593 task = self.runtaskentries[tid].task
597 # Add in extra dependencies 594 # Add in extra dependencies
598 if task in extradeps: 595 if tid in extradeps:
599 self.runq_depends[task] = extradeps[task] 596 self.runtaskentries[tid].depends = extradeps[tid]
600 # Remove all self references 597 # Remove all self references
601 if task in self.runq_depends[task]: 598 if tid in self.runtaskentries[tid].depends:
602 logger.debug(2, "Task %s (%s %s) contains self reference! %s", task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], self.runq_depends[task]) 599 logger.debug(2, "Task %s contains self reference!", tid)
603 self.runq_depends[task].remove(task) 600 self.runtaskentries[tid].depends.remove(tid)
604 601
605 # Step B - Mark all active tasks 602 # Step B - Mark all active tasks
606 # 603 #
@@ -610,33 +607,30 @@ class RunQueueData:
610 607
611 logger.verbose("Marking Active Tasks") 608 logger.verbose("Marking Active Tasks")
612 609
613 def mark_active(listid, depth): 610 def mark_active(tid, depth):
614 """ 611 """
615 Mark an item as active along with its depends 612 Mark an item as active along with its depends
616 (calls itself recursively) 613 (calls itself recursively)
617 """ 614 """
618 615
619 if runq_build[listid] == 1: 616 if tid in runq_build:
620 return 617 return
621 618
622 runq_build[listid] = 1 619 runq_build[tid] = 1
623 620
624 depends = self.runq_depends[listid] 621 depends = self.runtaskentries[tid].depends
625 for depend in depends: 622 for depend in depends:
626 mark_active(depend, depth+1) 623 mark_active(depend, depth+1)
627 624
628 self.target_pairs = [] 625 self.target_pairs = []
629 for target in self.targets: 626 for target in self.targets:
630 targetid = taskData.getbuild_id(target[0]) 627 if target[0] not in taskData.build_targets or not taskData.build_targets[target[0]]:
631
632 if targetid not in taskData.build_targets:
633 continue 628 continue
634 629
635 if targetid in taskData.failed_deps: 630 if target[0] in taskData.failed_deps:
636 continue 631 continue
637 632
638 fnid = taskData.build_targets[targetid][0] 633 fn = taskData.build_targets[target[0]][0]
639 fn = taskData.fn_index[fnid]
640 task = target[1] 634 task = target[1]
641 parents = False 635 parents = False
642 if task.endswith('-'): 636 if task.endswith('-'):
@@ -645,87 +639,72 @@ class RunQueueData:
645 639
646 self.target_pairs.append((fn, task)) 640 self.target_pairs.append((fn, task))
647 641
648 if fnid in taskData.failed_fnids: 642 if fn in taskData.failed_fns:
649 continue 643 continue
650 644
651 if task not in taskData.tasks_lookup[fnid]: 645 tid = fn + ":" + task
646 if tid not in taskData.taskentries:
652 import difflib 647 import difflib
653 close_matches = difflib.get_close_matches(task, taskData.tasks_lookup[fnid], cutoff=0.7) 648 tasks = []
649 for x in taskData.taskentries:
650 if x.startswith(fn + ":"):
651 tasks.append(taskname_from_tid(x))
652 close_matches = difflib.get_close_matches(task, tasks, cutoff=0.7)
654 if close_matches: 653 if close_matches:
655 extra = ". Close matches:\n %s" % "\n ".join(close_matches) 654 extra = ". Close matches:\n %s" % "\n ".join(close_matches)
656 else: 655 else:
657 extra = "" 656 extra = ""
658 bb.msg.fatal("RunQueue", "Task %s does not exist for target %s%s" % (task, target[0], extra)) 657 bb.msg.fatal("RunQueue", "Task %s does not exist for target %s%s" % (task, target[0], extra))
659 658
660 # For tasks called "XXXX-", ony run their dependencies 659 # For tasks called "XXXX-", ony run their dependencies
661 listid = taskData.tasks_lookup[fnid][task]
662 if parents: 660 if parents:
663 for i in self.runq_depends[listid]: 661 for i in self.runtaskentries[tid].depends:
664 mark_active(i, 1) 662 mark_active(i, 1)
665 else: 663 else:
666 mark_active(listid, 1) 664 mark_active(tid, 1)
667 665
668 # Step C - Prune all inactive tasks 666 # Step C - Prune all inactive tasks
669 # 667 #
670 # Once all active tasks are marked, prune the ones we don't need. 668 # Once all active tasks are marked, prune the ones we don't need.
671 669
672 maps = []
673 delcount = 0 670 delcount = 0
674 for listid in range(len(self.runq_fnid)): 671 for tid in list(self.runtaskentries.keys()):
675 if runq_build[listid-delcount] == 1: 672 if tid not in runq_build:
676 maps.append(listid-delcount) 673 del self.runtaskentries[tid]
677 else: 674 delcount += 1
678 del self.runq_fnid[listid-delcount]
679 del self.runq_task[listid-delcount]
680 del self.runq_depends[listid-delcount]
681 del runq_build[listid-delcount]
682 del self.runq_revdeps[listid-delcount]
683 del self.runq_hash[listid-delcount]
684 delcount = delcount + 1
685 maps.append(-1)
686 675
687 # 676 #
688 # Step D - Sanity checks and computation 677 # Step D - Sanity checks and computation
689 # 678 #
690 679
691 # Check to make sure we still have tasks to run 680 # Check to make sure we still have tasks to run
692 if len(self.runq_fnid) == 0: 681 if len(self.runtaskentries) == 0:
693 if not taskData.abort: 682 if not taskData.abort:
694 bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.") 683 bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.")
695 else: 684 else:
696 bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.") 685 bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.")
697 686
698 logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runq_fnid)) 687 logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runtaskentries))
699
700 # Remap the dependencies to account for the deleted tasks
701 # Check we didn't delete a task we depend on
702 for listid in range(len(self.runq_fnid)):
703 newdeps = []
704 origdeps = self.runq_depends[listid]
705 for origdep in origdeps:
706 if maps[origdep] == -1:
707 bb.msg.fatal("RunQueue", "Invalid mapping - Should never happen!")
708 newdeps.append(maps[origdep])
709 self.runq_depends[listid] = set(newdeps)
710 688
711 logger.verbose("Assign Weightings") 689 logger.verbose("Assign Weightings")
712 690
713 # Generate a list of reverse dependencies to ease future calculations 691 # Generate a list of reverse dependencies to ease future calculations
714 for listid in range(len(self.runq_fnid)): 692 for tid in self.runtaskentries:
715 for dep in self.runq_depends[listid]: 693 for dep in self.runtaskentries[tid].depends:
716 self.runq_revdeps[dep].add(listid) 694 self.runtaskentries[dep].revdeps.add(tid)
717 695
718 # Identify tasks at the end of dependency chains 696 # Identify tasks at the end of dependency chains
719 # Error on circular dependency loops (length two) 697 # Error on circular dependency loops (length two)
720 endpoints = [] 698 endpoints = []
721 for listid in range(len(self.runq_fnid)): 699 for tid in self.runtaskentries:
722 revdeps = self.runq_revdeps[listid] 700 revdeps = self.runtaskentries[tid].revdeps
723 if len(revdeps) == 0: 701 if len(revdeps) == 0:
724 endpoints.append(listid) 702 endpoints.append(tid)
725 for dep in revdeps: 703 for dep in revdeps:
726 if dep in self.runq_depends[listid]: 704 if dep in self.runtaskentries[tid].depends:
727 #self.dump_data(taskData) 705 #self.dump_data(taskData)
728 bb.msg.fatal("RunQueue", "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep], taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid])) 706 bb.msg.fatal("RunQueue", "Task %s has circular dependency on %s" % (tid, dep))
707
729 708
730 logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints)) 709 logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
731 710
@@ -736,8 +715,8 @@ class RunQueueData:
736 # Sanity Check - Check for multiple tasks building the same provider 715 # Sanity Check - Check for multiple tasks building the same provider
737 prov_list = {} 716 prov_list = {}
738 seen_fn = [] 717 seen_fn = []
739 for task in range(len(self.runq_fnid)): 718 for tid in self.runtaskentries:
740 fn = taskData.fn_index[self.runq_fnid[task]] 719 fn = fn_from_tid(tid)
741 if fn in seen_fn: 720 if fn in seen_fn:
742 continue 721 continue
743 seen_fn.append(fn) 722 seen_fn.append(fn)
@@ -766,15 +745,15 @@ class RunQueueData:
766 commondeps = None 745 commondeps = None
767 for provfn in prov_list[prov]: 746 for provfn in prov_list[prov]:
768 deps = set() 747 deps = set()
769 for task, fnid in enumerate(self.runq_fnid): 748 for tid in self.runtaskentries:
770 fn = taskData.fn_index[fnid] 749 fn = fn_from_tid(tid)
771 if fn != provfn: 750 if fn != provfn:
772 continue 751 continue
773 for dep in self.runq_revdeps[task]: 752 for dep in self.runtaskentries[tid].revdeps:
774 fn = taskData.fn_index[self.runq_fnid[dep]] 753 fn = fn_from_tid(dep)
775 if fn == provfn: 754 if fn == provfn:
776 continue 755 continue
777 deps.add(self.get_short_user_idstring(dep)) 756 deps.add(dep)
778 if not commondeps: 757 if not commondeps:
779 commondeps = set(deps) 758 commondeps = set(deps)
780 else: 759 else:
@@ -827,27 +806,26 @@ class RunQueueData:
827 # Create a whitelist usable by the stamp checks 806 # Create a whitelist usable by the stamp checks
828 stampfnwhitelist = [] 807 stampfnwhitelist = []
829 for entry in self.stampwhitelist.split(): 808 for entry in self.stampwhitelist.split():
830 entryid = self.taskData.getbuild_id(entry) 809 if entry not in self.taskData.build_targets:
831 if entryid not in self.taskData.build_targets:
832 continue 810 continue
833 fnid = self.taskData.build_targets[entryid][0] 811 fn = self.taskData.build_targets[entry][0]
834 fn = self.taskData.fn_index[fnid]
835 stampfnwhitelist.append(fn) 812 stampfnwhitelist.append(fn)
836 self.stampfnwhitelist = stampfnwhitelist 813 self.stampfnwhitelist = stampfnwhitelist
837 814
838 # Iterate over the task list looking for tasks with a 'setscene' function 815 # Iterate over the task list looking for tasks with a 'setscene' function
839 self.runq_setscene = [] 816 self.runq_setscene_tids = []
840 if not self.cooker.configuration.nosetscene: 817 if not self.cooker.configuration.nosetscene:
841 for task in range(len(self.runq_fnid)): 818 for tid in self.runtaskentries:
842 setscene = taskData.gettask_id(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task] + "_setscene", False) 819 setscenetid = tid + "_setscene"
843 if not setscene: 820 if setscenetid not in taskData.taskentries:
844 continue 821 continue
845 self.runq_setscene.append(task) 822 task = self.runtaskentries[tid].task
823 self.runq_setscene_tids.append(tid)
846 824
847 def invalidate_task(fn, taskname, error_nostamp): 825 def invalidate_task(fn, taskname, error_nostamp):
848 taskdep = self.dataCache.task_deps[fn] 826 taskdep = self.dataCache.task_deps[fn]
849 fnid = self.taskData.getfn_id(fn) 827 tid = fn + ":" + taskname
850 if taskname not in taskData.tasks_lookup[fnid]: 828 if tid not in taskData.taskentries:
851 logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname) 829 logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname)
852 if 'nostamp' in taskdep and taskname in taskdep['nostamp']: 830 if 'nostamp' in taskdep and taskname in taskdep['nostamp']:
853 if error_nostamp: 831 if error_nostamp:
@@ -882,43 +860,38 @@ class RunQueueData:
882 860
883 # Iterate over the task list and call into the siggen code 861 # Iterate over the task list and call into the siggen code
884 dealtwith = set() 862 dealtwith = set()
885 todeal = set(range(len(self.runq_fnid))) 863 todeal = set(self.runtaskentries)
886 while len(todeal) > 0: 864 while len(todeal) > 0:
887 for task in todeal.copy(): 865 for tid in todeal.copy():
888 if len(self.runq_depends[task] - dealtwith) == 0: 866 if len(self.runtaskentries[tid].depends - dealtwith) == 0:
889 dealtwith.add(task) 867 dealtwith.add(tid)
890 todeal.remove(task) 868 todeal.remove(tid)
891 procdep = [] 869 procdep = []
892 for dep in self.runq_depends[task]: 870 for dep in self.runtaskentries[tid].depends:
893 procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep]) 871 procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep))
894 self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache) 872 self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(fn_from_tid(tid), taskname_from_tid(tid), procdep, self.dataCache)
873 task = self.runtaskentries[tid].task
895 874
896 bb.parse.siggen.writeout_file_checksum_cache() 875 bb.parse.siggen.writeout_file_checksum_cache()
897 return len(self.runq_fnid) 876 return len(self.runtaskentries)
898 877
899 def dump_data(self, taskQueue): 878 def dump_data(self, taskQueue):
900 """ 879 """
901 Dump some debug information on the internal data structures 880 Dump some debug information on the internal data structures
902 """ 881 """
903 logger.debug(3, "run_tasks:") 882 logger.debug(3, "run_tasks:")
904 for task in range(len(self.rqdata.runq_task)): 883 for tid in self.runtaskentries:
905 logger.debug(3, " (%s)%s - %s: %s Deps %s RevDeps %s", task, 884 logger.debug(3, " %s: %s Deps %s RevDeps %s", tid,
906 taskQueue.fn_index[self.rqdata.runq_fnid[task]], 885 self.runtaskentries[tid].weight,
907 self.rqdata.runq_task[task], 886 self.runtaskentries[tid].depends,
908 self.rqdata.runq_weight[task], 887 self.runtaskentries[tid].revdeps)
909 self.rqdata.runq_depends[task],
910 self.rqdata.runq_revdeps[task])
911 888
912 logger.debug(3, "sorted_tasks:") 889 logger.debug(3, "sorted_tasks:")
913 for task1 in range(len(self.rqdata.runq_task)): 890 for tid in self.prio_map:
914 if task1 in self.prio_map: 891 logger.debug(3, " %s: %s Deps %s RevDeps %s", tid,
915 task = self.prio_map[task1] 892 self.runtaskentries[tid].weight,
916 logger.debug(3, " (%s)%s - %s: %s Deps %s RevDeps %s", task, 893 self.runtaskentries[tid].depends,
917 taskQueue.fn_index[self.rqdata.runq_fnid[task]], 894 self.runtaskentries[tid].revdeps)
918 self.rqdata.runq_task[task],
919 self.rqdata.runq_weight[task],
920 self.rqdata.runq_depends[task],
921 self.rqdata.runq_revdeps[task])
922 895
923class RunQueue: 896class RunQueue:
924 def __init__(self, cooker, cfgData, dataCache, taskData, targets): 897 def __init__(self, cooker, cfgData, dataCache, taskData, targets):
@@ -929,7 +902,7 @@ class RunQueue:
929 902
930 self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile" 903 self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile"
931 self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None 904 self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None
932 self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION", True) or None 905 self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION2", True) or None
933 self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None 906 self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None
934 907
935 self.state = runQueuePrepare 908 self.state = runQueuePrepare
@@ -961,13 +934,17 @@ class RunQueue:
961 bb.utils.nonblockingfd(worker.stdout) 934 bb.utils.nonblockingfd(worker.stdout)
962 workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec) 935 workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
963 936
937 runqhash = {}
938 for tid in self.rqdata.runtaskentries:
939 runqhash[tid] = self.rqdata.runtaskentries[tid].hash
940
964 workerdata = { 941 workerdata = {
965 "taskdeps" : self.rqdata.dataCache.task_deps, 942 "taskdeps" : self.rqdata.dataCache.task_deps,
966 "fakerootenv" : self.rqdata.dataCache.fakerootenv, 943 "fakerootenv" : self.rqdata.dataCache.fakerootenv,
967 "fakerootdirs" : self.rqdata.dataCache.fakerootdirs, 944 "fakerootdirs" : self.rqdata.dataCache.fakerootdirs,
968 "fakerootnoenv" : self.rqdata.dataCache.fakerootnoenv, 945 "fakerootnoenv" : self.rqdata.dataCache.fakerootnoenv,
969 "sigdata" : bb.parse.siggen.get_taskdata(), 946 "sigdata" : bb.parse.siggen.get_taskdata(),
970 "runq_hash" : self.rqdata.runq_hash, 947 "runq_hash" : runqhash,
971 "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel, 948 "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel,
972 "logdefaultverbose" : bb.msg.loggerDefaultVerbose, 949 "logdefaultverbose" : bb.msg.loggerDefaultVerbose,
973 "logdefaultverboselogs" : bb.msg.loggerVerboseLogs, 950 "logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
@@ -1033,7 +1010,7 @@ class RunQueue:
1033 fds.append(self.fakeworkerpipe.input) 1010 fds.append(self.fakeworkerpipe.input)
1034 return fds 1011 return fds
1035 1012
1036 def check_stamp_task(self, task, taskname = None, recurse = False, cache = None): 1013 def check_stamp_task(self, tid, taskname = None, recurse = False, cache = None):
1037 def get_timestamp(f): 1014 def get_timestamp(f):
1038 try: 1015 try:
1039 if not os.access(f, os.F_OK): 1016 if not os.access(f, os.F_OK):
@@ -1050,9 +1027,9 @@ class RunQueue:
1050 if self.stamppolicy == "whitelist": 1027 if self.stamppolicy == "whitelist":
1051 stampwhitelist = self.rqdata.stampfnwhitelist 1028 stampwhitelist = self.rqdata.stampfnwhitelist
1052 1029
1053 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] 1030 fn = fn_from_tid(tid)
1054 if taskname is None: 1031 if taskname is None:
1055 taskname = self.rqdata.runq_task[task] 1032 taskname = taskname_from_tid(tid)
1056 1033
1057 stampfile = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) 1034 stampfile = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
1058 1035
@@ -1074,10 +1051,10 @@ class RunQueue:
1074 1051
1075 iscurrent = True 1052 iscurrent = True
1076 t1 = get_timestamp(stampfile) 1053 t1 = get_timestamp(stampfile)
1077 for dep in self.rqdata.runq_depends[task]: 1054 for dep in self.rqdata.runtaskentries[tid].depends:
1078 if iscurrent: 1055 if iscurrent:
1079 fn2 = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[dep]] 1056 fn2 = fn_from_tid(dep)
1080 taskname2 = self.rqdata.runq_task[dep] 1057 taskname2 = taskname_from_tid(dep)
1081 stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCache, fn2) 1058 stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCache, fn2)
1082 stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCache, fn2) 1059 stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCache, fn2)
1083 t2 = get_timestamp(stampfile2) 1060 t2 = get_timestamp(stampfile2)
@@ -1104,7 +1081,7 @@ class RunQueue:
1104 iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache) 1081 iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache)
1105 cache[dep] = iscurrent 1082 cache[dep] = iscurrent
1106 if recurse: 1083 if recurse:
1107 cache[task] = iscurrent 1084 cache[tid] = iscurrent
1108 return iscurrent 1085 return iscurrent
1109 1086
1110 def _execute_runqueue(self): 1087 def _execute_runqueue(self):
@@ -1171,9 +1148,9 @@ class RunQueue:
1171 1148
1172 if self.state is runQueueFailed: 1149 if self.state is runQueueFailed:
1173 if not self.rqdata.taskData.tryaltconfigs: 1150 if not self.rqdata.taskData.tryaltconfigs:
1174 raise bb.runqueue.TaskFailure(self.rqexe.failed_fnids) 1151 raise bb.runqueue.TaskFailure(self.rqexe.failed_fns)
1175 for fnid in self.rqexe.failed_fnids: 1152 for fn in self.rqexe.failed_fns:
1176 self.rqdata.taskData.fail_fnid(fnid) 1153 self.rqdata.taskData.fail_fn(fn)
1177 self.rqdata.reset() 1154 self.rqdata.reset()
1178 1155
1179 if self.state is runQueueComplete: 1156 if self.state is runQueueComplete:
@@ -1220,11 +1197,11 @@ class RunQueue:
1220 def dump_signatures(self, options): 1197 def dump_signatures(self, options):
1221 done = set() 1198 done = set()
1222 bb.note("Reparsing files to collect dependency data") 1199 bb.note("Reparsing files to collect dependency data")
1223 for task in range(len(self.rqdata.runq_fnid)): 1200 for tid in self.rqdata.runtaskentries:
1224 if self.rqdata.runq_fnid[task] not in done: 1201 fn = fn_from_tid(tid)
1225 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] 1202 if fn not in done:
1226 the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn), self.cooker.data) 1203 the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn), self.cooker.data)
1227 done.add(self.rqdata.runq_fnid[task]) 1204 done.add(fn)
1228 1205
1229 bb.parse.siggen.dump_sigs(self.rqdata.dataCache, options) 1206 bb.parse.siggen.dump_sigs(self.rqdata.dataCache, options)
1230 1207
@@ -1242,20 +1219,20 @@ class RunQueue:
1242 stamppresent = [] 1219 stamppresent = []
1243 valid_new = set() 1220 valid_new = set()
1244 1221
1245 for task in range(len(self.rqdata.runq_fnid)): 1222 for tid in self.rqdata.runtaskentries:
1246 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] 1223 fn = fn_from_tid(tid)
1247 taskname = self.rqdata.runq_task[task] 1224 taskname = taskname_from_tid(tid)
1248 taskdep = self.rqdata.dataCache.task_deps[fn] 1225 taskdep = self.rqdata.dataCache.task_deps[fn]
1249 1226
1250 if 'noexec' in taskdep and taskname in taskdep['noexec']: 1227 if 'noexec' in taskdep and taskname in taskdep['noexec']:
1251 noexec.append(task) 1228 noexec.append(tid)
1252 continue 1229 continue
1253 1230
1254 sq_fn.append(fn) 1231 sq_fn.append(fn)
1255 sq_hashfn.append(self.rqdata.dataCache.hashfn[fn]) 1232 sq_hashfn.append(self.rqdata.dataCache.hashfn[fn])
1256 sq_hash.append(self.rqdata.runq_hash[task]) 1233 sq_hash.append(self.rqdata.runtaskentries[tid].hash)
1257 sq_taskname.append(taskname) 1234 sq_taskname.append(taskname)
1258 sq_task.append(task) 1235 sq_task.append(tid)
1259 locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data } 1236 locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data }
1260 try: 1237 try:
1261 call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)" 1238 call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)"
@@ -1270,13 +1247,13 @@ class RunQueue:
1270 # Tasks which are both setscene and noexec never care about dependencies 1247 # Tasks which are both setscene and noexec never care about dependencies
1271 # We therefore find tasks which are setscene and noexec and mark their 1248 # We therefore find tasks which are setscene and noexec and mark their
1272 # unique dependencies as valid. 1249 # unique dependencies as valid.
1273 for task in noexec: 1250 for tid in noexec:
1274 if task not in self.rqdata.runq_setscene: 1251 if tid not in self.rqdata.runq_setscene_tids:
1275 continue 1252 continue
1276 for dep in self.rqdata.runq_depends[task]: 1253 for dep in self.rqdata.runtaskentries[tid].depends:
1277 hasnoexecparents = True 1254 hasnoexecparents = True
1278 for dep2 in self.rqdata.runq_revdeps[dep]: 1255 for dep2 in self.rqdata.runtaskentries[dep].revdeps:
1279 if dep2 in self.rqdata.runq_setscene and dep2 in noexec: 1256 if dep2 in self.rqdata.runq_setscene_tids and dep2 in noexec:
1280 continue 1257 continue
1281 hasnoexecparents = False 1258 hasnoexecparents = False
1282 break 1259 break
@@ -1284,30 +1261,30 @@ class RunQueue:
1284 valid_new.add(dep) 1261 valid_new.add(dep)
1285 1262
1286 invalidtasks = set() 1263 invalidtasks = set()
1287 for task in range(len(self.rqdata.runq_fnid)): 1264 for tid in self.rqdata.runtaskentries:
1288 if task not in valid_new and task not in noexec: 1265 if tid not in valid_new and tid not in noexec:
1289 invalidtasks.add(task) 1266 invalidtasks.add(tid)
1290 1267
1291 found = set() 1268 found = set()
1292 processed = set() 1269 processed = set()
1293 for task in invalidtasks: 1270 for tid in invalidtasks:
1294 toprocess = set([task]) 1271 toprocess = set([tid])
1295 while toprocess: 1272 while toprocess:
1296 next = set() 1273 next = set()
1297 for t in toprocess: 1274 for t in toprocess:
1298 for dep in self.rqdata.runq_depends[t]: 1275 for dep in self.rqdata.runtaskentries[t].depends:
1299 if dep in invalidtasks: 1276 if dep in invalidtasks:
1300 found.add(task) 1277 found.add(tid)
1301 if dep not in processed: 1278 if dep not in processed:
1302 processed.add(dep) 1279 processed.add(dep)
1303 next.add(dep) 1280 next.add(dep)
1304 toprocess = next 1281 toprocess = next
1305 if task in found: 1282 if tid in found:
1306 toprocess = set() 1283 toprocess = set()
1307 1284
1308 tasklist = [] 1285 tasklist = []
1309 for task in invalidtasks.difference(found): 1286 for tid in invalidtasks.difference(found):
1310 tasklist.append(self.rqdata.get_user_idstring(task)) 1287 tasklist.append(tid)
1311 1288
1312 if tasklist: 1289 if tasklist:
1313 bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist)) 1290 bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist))
@@ -1331,11 +1308,11 @@ class RunQueue:
1331 return recout 1308 return recout
1332 1309
1333 1310
1334 for task in invalidtasks: 1311 for tid in invalidtasks:
1335 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] 1312 fn = fn_from_tid(tid)
1336 pn = self.rqdata.dataCache.pkg_fn[fn] 1313 pn = self.rqdata.dataCache.pkg_fn[fn]
1337 taskname = self.rqdata.runq_task[task] 1314 taskname = taskname_from_tid(tid)
1338 h = self.rqdata.runq_hash[task] 1315 h = self.rqdata.runtaskentries[tid].hash
1339 matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData) 1316 matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData)
1340 match = None 1317 match = None
1341 for m in matches: 1318 for m in matches:
@@ -1367,7 +1344,7 @@ class RunQueueExecute:
1367 1344
1368 self.build_stamps = {} 1345 self.build_stamps = {}
1369 self.build_stamps2 = [] 1346 self.build_stamps2 = []
1370 self.failed_fnids = [] 1347 self.failed_fns = []
1371 1348
1372 self.stampcache = {} 1349 self.stampcache = {}
1373 1350
@@ -1401,7 +1378,7 @@ class RunQueueExecute:
1401 except IOError: 1378 except IOError:
1402 # worker must have died? 1379 # worker must have died?
1403 pass 1380 pass
1404 if len(self.failed_fnids) != 0: 1381 if len(self.failed_fns) != 0:
1405 self.rq.state = runQueueFailed 1382 self.rq.state = runQueueFailed
1406 return 1383 return
1407 1384
@@ -1416,7 +1393,7 @@ class RunQueueExecute:
1416 self.rq.read_workers() 1393 self.rq.read_workers()
1417 return self.rq.active_fds() 1394 return self.rq.active_fds()
1418 1395
1419 if len(self.failed_fnids) != 0: 1396 if len(self.failed_fns) != 0:
1420 self.rq.state = runQueueFailed 1397 self.rq.state = runQueueFailed
1421 return True 1398 return True
1422 1399
@@ -1430,13 +1407,9 @@ class RunQueueExecute:
1430 taskdata = {} 1407 taskdata = {}
1431 taskdeps.add(task) 1408 taskdeps.add(task)
1432 for dep in taskdeps: 1409 for dep in taskdeps:
1433 if setscene: 1410 fn = fn_from_tid(dep)
1434 depid = self.rqdata.runq_setscene[dep]
1435 else:
1436 depid = dep
1437 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[depid]]
1438 pn = self.rqdata.dataCache.pkg_fn[fn] 1411 pn = self.rqdata.dataCache.pkg_fn[fn]
1439 taskname = self.rqdata.runq_task[depid] 1412 taskname = taskname_from_tid(dep)
1440 taskdata[dep] = [pn, taskname, fn] 1413 taskdata[dep] = [pn, taskname, fn]
1441 call = self.rq.depvalidate + "(task, taskdata, notneeded, d)" 1414 call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
1442 locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data } 1415 locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data }
@@ -1456,30 +1429,30 @@ class RunQueueExecuteTasks(RunQueueExecute):
1456 def __init__(self, rq): 1429 def __init__(self, rq):
1457 RunQueueExecute.__init__(self, rq) 1430 RunQueueExecute.__init__(self, rq)
1458 1431
1459 self.stats = RunQueueStats(len(self.rqdata.runq_fnid)) 1432 self.stats = RunQueueStats(len(self.rqdata.runtaskentries))
1460 1433
1461 self.stampcache = {} 1434 self.stampcache = {}
1462 1435
1463 initial_covered = self.rq.scenequeue_covered.copy() 1436 initial_covered = self.rq.scenequeue_covered.copy()
1464 1437
1465 # Mark initial buildable tasks 1438 # Mark initial buildable tasks
1466 for task in range(self.stats.total): 1439 for tid in self.rqdata.runtaskentries:
1467 if len(self.rqdata.runq_depends[task]) == 0: 1440 if len(self.rqdata.runtaskentries[tid].depends) == 0:
1468 self.runq_buildable.add(task) 1441 self.runq_buildable.add(tid)
1469 if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered): 1442 if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered):
1470 self.rq.scenequeue_covered.add(task) 1443 self.rq.scenequeue_covered.add(tid)
1471 1444
1472 found = True 1445 found = True
1473 while found: 1446 while found:
1474 found = False 1447 found = False
1475 for task in range(self.stats.total): 1448 for tid in self.rqdata.runtaskentries:
1476 if task in self.rq.scenequeue_covered: 1449 if tid in self.rq.scenequeue_covered:
1477 continue 1450 continue
1478 logger.debug(1, 'Considering %s (%s): %s' % (task, self.rqdata.get_user_idstring(task), str(self.rqdata.runq_revdeps[task]))) 1451 logger.debug(1, 'Considering %s: %s' % (tid, str(self.rqdata.runtaskentries[tid].revdeps)))
1479 1452
1480 if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered): 1453 if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered):
1481 found = True 1454 found = True
1482 self.rq.scenequeue_covered.add(task) 1455 self.rq.scenequeue_covered.add(tid)
1483 1456
1484 logger.debug(1, 'Skip list (pre setsceneverify) %s', sorted(self.rq.scenequeue_covered)) 1457 logger.debug(1, 'Skip list (pre setsceneverify) %s', sorted(self.rq.scenequeue_covered))
1485 1458
@@ -1487,35 +1460,33 @@ class RunQueueExecuteTasks(RunQueueExecute):
1487 covered_remove = set() 1460 covered_remove = set()
1488 if self.rq.setsceneverify: 1461 if self.rq.setsceneverify:
1489 invalidtasks = [] 1462 invalidtasks = []
1490 for task in range(len(self.rqdata.runq_task)): 1463 tasknames = {}
1491 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] 1464 fns = {}
1492 taskname = self.rqdata.runq_task[task] 1465 for tid in self.rqdata.runtaskentries:
1466 fn = fn_from_tid(tid)
1467 taskname = taskname_from_tid(tid)
1493 taskdep = self.rqdata.dataCache.task_deps[fn] 1468 taskdep = self.rqdata.dataCache.task_deps[fn]
1494 1469 fns[tid] = fn
1470 tasknames[tid] = taskname
1495 if 'noexec' in taskdep and taskname in taskdep['noexec']: 1471 if 'noexec' in taskdep and taskname in taskdep['noexec']:
1496 continue 1472 continue
1497 if self.rq.check_stamp_task(task, taskname + "_setscene", cache=self.stampcache): 1473 if self.rq.check_stamp_task(tid, taskname + "_setscene", cache=self.stampcache):
1498 logger.debug(2, 'Setscene stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(task)) 1474 logger.debug(2, 'Setscene stamp current for task %s', tid)
1499 continue 1475 continue
1500 if self.rq.check_stamp_task(task, taskname, recurse = True, cache=self.stampcache): 1476 if self.rq.check_stamp_task(tid, taskname, recurse = True, cache=self.stampcache):
1501 logger.debug(2, 'Normal stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(task)) 1477 logger.debug(2, 'Normal stamp current for task %s', tid)
1502 continue 1478 continue
1503 invalidtasks.append(task) 1479 invalidtasks.append(tid)
1504 1480
1505 call = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d, invalidtasks=invalidtasks)" 1481 call = self.rq.setsceneverify + "(covered, tasknames, fns, d, invalidtasks=invalidtasks)"
1506 call2 = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d)" 1482 locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : tasknames, "fns" : fns, "d" : self.cooker.expanded_data, "invalidtasks" : invalidtasks }
1507 locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : self.rqdata.runq_task, "fnids" : self.rqdata.runq_fnid, "fns" : self.rqdata.taskData.fn_index, "d" : self.cooker.expanded_data, "invalidtasks" : invalidtasks } 1483 covered_remove = bb.utils.better_eval(call, locs)
1508 # Backwards compatibility with older versions without invalidtasks
1509 try:
1510 covered_remove = bb.utils.better_eval(call, locs)
1511 except TypeError:
1512 covered_remove = bb.utils.better_eval(call2, locs)
1513 1484
1514 def removecoveredtask(task): 1485 def removecoveredtask(tid):
1515 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] 1486 fn = fn_from_tid(tid)
1516 taskname = self.rqdata.runq_task[task] + '_setscene' 1487 taskname = taskname_from_tid(tid) + '_setscene'
1517 bb.build.del_stamp(taskname, self.rqdata.dataCache, fn) 1488 bb.build.del_stamp(taskname, self.rqdata.dataCache, fn)
1518 self.rq.scenequeue_covered.remove(task) 1489 self.rq.scenequeue_covered.remove(tid)
1519 1490
1520 toremove = covered_remove 1491 toremove = covered_remove
1521 for task in toremove: 1492 for task in toremove:
@@ -1524,7 +1495,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1524 covered_remove = [] 1495 covered_remove = []
1525 for task in toremove: 1496 for task in toremove:
1526 removecoveredtask(task) 1497 removecoveredtask(task)
1527 for deptask in self.rqdata.runq_depends[task]: 1498 for deptask in self.rqdata.runtaskentries[task].depends:
1528 if deptask not in self.rq.scenequeue_covered: 1499 if deptask not in self.rq.scenequeue_covered:
1529 continue 1500 continue
1530 if deptask in toremove or deptask in covered_remove or deptask in initial_covered: 1501 if deptask in toremove or deptask in covered_remove or deptask in initial_covered:
@@ -1580,20 +1551,20 @@ class RunQueueExecuteTasks(RunQueueExecute):
1580 completed dependencies as buildable 1551 completed dependencies as buildable
1581 """ 1552 """
1582 self.runq_complete.add(task) 1553 self.runq_complete.add(task)
1583 for revdep in self.rqdata.runq_revdeps[task]: 1554 for revdep in self.rqdata.runtaskentries[task].revdeps:
1584 if revdep in self.runq_running: 1555 if revdep in self.runq_running:
1585 continue 1556 continue
1586 if revdep in self.runq_buildable: 1557 if revdep in self.runq_buildable:
1587 continue 1558 continue
1588 alldeps = 1 1559 alldeps = 1
1589 for dep in self.rqdata.runq_depends[revdep]: 1560 for dep in self.rqdata.runtaskentries[revdep].depends:
1590 if dep not in self.runq_complete: 1561 if dep not in self.runq_complete:
1591 alldeps = 0 1562 alldeps = 0
1592 if alldeps == 1: 1563 if alldeps == 1:
1593 self.setbuildable(revdep) 1564 self.setbuildable(revdep)
1594 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]] 1565 fn = fn_from_tid(revdep)
1595 taskname = self.rqdata.runq_task[revdep] 1566 taskname = taskname_from_tid(revdep)
1596 logger.debug(1, "Marking task %s (%s, %s) as buildable", revdep, fn, taskname) 1567 logger.debug(1, "Marking task %s as buildable", revdep)
1597 1568
1598 def task_complete(self, task): 1569 def task_complete(self, task):
1599 self.stats.taskCompleted() 1570 self.stats.taskCompleted()
@@ -1606,8 +1577,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
1606 Updates the state engine with the failure 1577 Updates the state engine with the failure
1607 """ 1578 """
1608 self.stats.taskFailed() 1579 self.stats.taskFailed()
1609 fnid = self.rqdata.runq_fnid[task] 1580 fn = fn_from_tid(task)
1610 self.failed_fnids.append(fnid) 1581 self.failed_fns.append(fn)
1611 bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData) 1582 bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData)
1612 if self.rqdata.taskData.abort: 1583 if self.rqdata.taskData.abort:
1613 self.rq.state = runQueueCleanUp 1584 self.rq.state = runQueueCleanUp
@@ -1626,7 +1597,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1626 """ 1597 """
1627 1598
1628 self.rq.read_workers() 1599 self.rq.read_workers()
1629 1600
1630 1601
1631 if self.stats.total == 0: 1602 if self.stats.total == 0:
1632 # nothing to do 1603 # nothing to do
@@ -1634,18 +1605,17 @@ class RunQueueExecuteTasks(RunQueueExecute):
1634 1605
1635 task = self.sched.next() 1606 task = self.sched.next()
1636 if task is not None: 1607 if task is not None:
1637 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] 1608 fn = fn_from_tid(task)
1638 taskname = self.rqdata.runq_task[task] 1609 taskname = taskname_from_tid(task)
1639 1610
1640 if task in self.rq.scenequeue_covered: 1611 if task in self.rq.scenequeue_covered:
1641 logger.debug(2, "Setscene covered task %s (%s)", task, 1612 logger.debug(2, "Setscene covered task %s", task)
1642 self.rqdata.get_user_idstring(task))
1643 self.task_skip(task, "covered") 1613 self.task_skip(task, "covered")
1644 return True 1614 return True
1645 1615
1646 if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): 1616 if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
1647 logger.debug(2, "Stamp current task %s (%s)", task, 1617 logger.debug(2, "Stamp current task %s", task)
1648 self.rqdata.get_user_idstring(task)) 1618
1649 self.task_skip(task, "existing") 1619 self.task_skip(task, "existing")
1650 return True 1620 return True
1651 1621
@@ -1672,7 +1642,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1672 try: 1642 try:
1673 self.rq.start_fakeworker(self) 1643 self.rq.start_fakeworker(self)
1674 except OSError as exc: 1644 except OSError as exc:
1675 logger.critical("Failed to spawn fakeroot worker to run %s:%s: %s" % (fn, taskname, str(exc))) 1645 logger.critical("Failed to spawn fakeroot worker to run %s: %s" % (task, str(exc)))
1676 self.rq.state = runQueueFailed 1646 self.rq.state = runQueueFailed
1677 return True 1647 return True
1678 self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + b"</runtask>") 1648 self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + b"</runtask>")
@@ -1682,7 +1652,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1682 self.rq.worker.stdin.flush() 1652 self.rq.worker.stdin.flush()
1683 1653
1684 self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) 1654 self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
1685 self.build_stamps2.append(self.build_stamps[task]) 1655 self.build_stamps2.append(self.build_stamps[task])
1686 self.runq_running.add(task) 1656 self.runq_running.add(task)
1687 self.stats.taskActive() 1657 self.stats.taskActive()
1688 if self.stats.active < self.number_tasks: 1658 if self.stats.active < self.number_tasks:
@@ -1692,12 +1662,12 @@ class RunQueueExecuteTasks(RunQueueExecute):
1692 self.rq.read_workers() 1662 self.rq.read_workers()
1693 return self.rq.active_fds() 1663 return self.rq.active_fds()
1694 1664
1695 if len(self.failed_fnids) != 0: 1665 if len(self.failed_fns) != 0:
1696 self.rq.state = runQueueFailed 1666 self.rq.state = runQueueFailed
1697 return True 1667 return True
1698 1668
1699 # Sanity Checks 1669 # Sanity Checks
1700 for task in range(self.stats.total): 1670 for task in self.rqdata.runtaskentries:
1701 if task not in self.runq_buildable: 1671 if task not in self.runq_buildable:
1702 logger.error("Task %s never buildable!", task) 1672 logger.error("Task %s never buildable!", task)
1703 if task not in self.runq_running: 1673 if task not in self.runq_running:
@@ -1710,15 +1680,15 @@ class RunQueueExecuteTasks(RunQueueExecute):
1710 1680
1711 def build_taskdepdata(self, task): 1681 def build_taskdepdata(self, task):
1712 taskdepdata = {} 1682 taskdepdata = {}
1713 next = self.rqdata.runq_depends[task] 1683 next = self.rqdata.runtaskentries[task].depends
1714 next.add(task) 1684 next.add(task)
1715 while next: 1685 while next:
1716 additional = [] 1686 additional = []
1717 for revdep in next: 1687 for revdep in next:
1718 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]] 1688 fn = fn_from_tid(revdep)
1719 pn = self.rqdata.dataCache.pkg_fn[fn] 1689 pn = self.rqdata.dataCache.pkg_fn[fn]
1720 taskname = self.rqdata.runq_task[revdep] 1690 taskname = taskname_from_tid(revdep)
1721 deps = self.rqdata.runq_depends[revdep] 1691 deps = self.rqdata.runtaskentries[revdep].depends
1722 provides = self.rqdata.dataCache.fn_provides[fn] 1692 provides = self.rqdata.dataCache.fn_provides[fn]
1723 taskdepdata[revdep] = [pn, taskname, fn, deps, provides] 1693 taskdepdata[revdep] = [pn, taskname, fn, deps, provides]
1724 for revdep2 in deps: 1694 for revdep2 in deps:
@@ -1738,16 +1708,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1738 self.scenequeue_notneeded = set() 1708 self.scenequeue_notneeded = set()
1739 1709
1740 # If we don't have any setscene functions, skip this step 1710 # If we don't have any setscene functions, skip this step
1741 if len(self.rqdata.runq_setscene) == 0: 1711 if len(self.rqdata.runq_setscene_tids) == 0:
1742 rq.scenequeue_covered = set() 1712 rq.scenequeue_covered = set()
1743 rq.state = runQueueRunInit 1713 rq.state = runQueueRunInit
1744 return 1714 return
1745 1715
1746 self.stats = RunQueueStats(len(self.rqdata.runq_setscene)) 1716 self.stats = RunQueueStats(len(self.rqdata.runq_setscene_tids))
1747 1717
1748 sq_revdeps = [] 1718 sq_revdeps = {}
1749 sq_revdeps_new = [] 1719 sq_revdeps_new = {}
1750 sq_revdeps_squash = [] 1720 sq_revdeps_squash = {}
1751 self.sq_harddeps = {} 1721 self.sq_harddeps = {}
1752 1722
1753 # We need to construct a dependency graph for the setscene functions. Intermediate 1723 # We need to construct a dependency graph for the setscene functions. Intermediate
@@ -1757,18 +1727,21 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1757 1727
1758 # First process the chains up to the first setscene task. 1728 # First process the chains up to the first setscene task.
1759 endpoints = {} 1729 endpoints = {}
1760 for task in range(len(self.rqdata.runq_fnid)): 1730 for tid in self.rqdata.runtaskentries:
1761 sq_revdeps.append(copy.copy(self.rqdata.runq_revdeps[task])) 1731 sq_revdeps[tid] = copy.copy(self.rqdata.runtaskentries[tid].revdeps)
1762 sq_revdeps_new.append(set()) 1732 sq_revdeps_new[tid] = set()
1763 if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene: 1733 if (len(sq_revdeps[tid]) == 0) and tid not in self.rqdata.runq_setscene_tids:
1764 endpoints[task] = set() 1734 #bb.warn("Added endpoint %s" % (tid))
1735 endpoints[tid] = set()
1765 1736
1766 # Secondly process the chains between setscene tasks. 1737 # Secondly process the chains between setscene tasks.
1767 for task in self.rqdata.runq_setscene: 1738 for tid in self.rqdata.runq_setscene_tids:
1768 for dep in self.rqdata.runq_depends[task]: 1739 #bb.warn("Added endpoint 2 %s" % (tid))
1740 for dep in self.rqdata.runtaskentries[tid].depends:
1769 if dep not in endpoints: 1741 if dep not in endpoints:
1770 endpoints[dep] = set() 1742 endpoints[dep] = set()
1771 endpoints[dep].add(task) 1743 #bb.warn(" Added endpoint 3 %s" % (dep))
1744 endpoints[dep].add(tid)
1772 1745
1773 def process_endpoints(endpoints): 1746 def process_endpoints(endpoints):
1774 newendpoints = {} 1747 newendpoints = {}
@@ -1779,15 +1752,15 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1779 if sq_revdeps_new[point]: 1752 if sq_revdeps_new[point]:
1780 tasks |= sq_revdeps_new[point] 1753 tasks |= sq_revdeps_new[point]
1781 sq_revdeps_new[point] = set() 1754 sq_revdeps_new[point] = set()
1782 if point in self.rqdata.runq_setscene: 1755 if point in self.rqdata.runq_setscene_tids:
1783 sq_revdeps_new[point] = tasks 1756 sq_revdeps_new[point] = tasks
1784 tasks = set() 1757 tasks = set()
1785 for dep in self.rqdata.runq_depends[point]: 1758 for dep in self.rqdata.runtaskentries[point].depends:
1786 if point in sq_revdeps[dep]: 1759 if point in sq_revdeps[dep]:
1787 sq_revdeps[dep].remove(point) 1760 sq_revdeps[dep].remove(point)
1788 if tasks: 1761 if tasks:
1789 sq_revdeps_new[dep] |= tasks 1762 sq_revdeps_new[dep] |= tasks
1790 if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in self.rqdata.runq_setscene: 1763 if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in self.rqdata.runq_setscene_tids:
1791 newendpoints[dep] = task 1764 newendpoints[dep] = task
1792 if len(newendpoints) != 0: 1765 if len(newendpoints) != 0:
1793 process_endpoints(newendpoints) 1766 process_endpoints(newendpoints)
@@ -1797,8 +1770,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1797 # Build a list of setscene tasks which are "unskippable" 1770 # Build a list of setscene tasks which are "unskippable"
1798 # These are direct endpoints referenced by the build 1771 # These are direct endpoints referenced by the build
1799 endpoints2 = {} 1772 endpoints2 = {}
1800 sq_revdeps2 = [] 1773 sq_revdeps2 = {}
1801 sq_revdeps_new2 = [] 1774 sq_revdeps_new2 = {}
1802 def process_endpoints2(endpoints): 1775 def process_endpoints2(endpoints):
1803 newendpoints = {} 1776 newendpoints = {}
1804 for point, task in endpoints.items(): 1777 for point, task in endpoints.items():
@@ -1808,84 +1781,85 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1808 if sq_revdeps_new2[point]: 1781 if sq_revdeps_new2[point]:
1809 tasks |= sq_revdeps_new2[point] 1782 tasks |= sq_revdeps_new2[point]
1810 sq_revdeps_new2[point] = set() 1783 sq_revdeps_new2[point] = set()
1811 if point in self.rqdata.runq_setscene: 1784 if point in self.rqdata.runq_setscene_tids:
1812 sq_revdeps_new2[point] = tasks 1785 sq_revdeps_new2[point] = tasks
1813 for dep in self.rqdata.runq_depends[point]: 1786 for dep in self.rqdata.runtaskentries[point].depends:
1814 if point in sq_revdeps2[dep]: 1787 if point in sq_revdeps2[dep]:
1815 sq_revdeps2[dep].remove(point) 1788 sq_revdeps2[dep].remove(point)
1816 if tasks: 1789 if tasks:
1817 sq_revdeps_new2[dep] |= tasks 1790 sq_revdeps_new2[dep] |= tasks
1818 if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in self.rqdata.runq_setscene: 1791 if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in self.rqdata.runq_setscene_tids:
1819 newendpoints[dep] = tasks 1792 newendpoints[dep] = tasks
1820 if len(newendpoints) != 0: 1793 if len(newendpoints) != 0:
1821 process_endpoints2(newendpoints) 1794 process_endpoints2(newendpoints)
1822 for task in range(len(self.rqdata.runq_fnid)): 1795 for tid in self.rqdata.runtaskentries:
1823 sq_revdeps2.append(copy.copy(self.rqdata.runq_revdeps[task])) 1796 sq_revdeps2[tid] = copy.copy(self.rqdata.runtaskentries[tid].revdeps)
1824 sq_revdeps_new2.append(set()) 1797 sq_revdeps_new2[tid] = set()
1825 if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene: 1798 if (len(sq_revdeps2[tid]) == 0) and tid not in self.rqdata.runq_setscene_tids:
1826 endpoints2[task] = set() 1799 endpoints2[tid] = set()
1827 process_endpoints2(endpoints2) 1800 process_endpoints2(endpoints2)
1828 self.unskippable = [] 1801 self.unskippable = []
1829 for task in self.rqdata.runq_setscene: 1802 for tid in self.rqdata.runq_setscene_tids:
1830 if sq_revdeps_new2[task]: 1803 if sq_revdeps_new2[tid]:
1831 self.unskippable.append(self.rqdata.runq_setscene.index(task)) 1804 self.unskippable.append(tid)
1832 1805
1833 for task in range(len(self.rqdata.runq_fnid)): 1806 for tid in self.rqdata.runtaskentries:
1834 if task in self.rqdata.runq_setscene: 1807 if tid in self.rqdata.runq_setscene_tids:
1835 deps = set() 1808 deps = set()
1836 for dep in sq_revdeps_new[task]: 1809 for dep in sq_revdeps_new[tid]:
1837 deps.add(self.rqdata.runq_setscene.index(dep)) 1810 deps.add(dep)
1838 sq_revdeps_squash.append(deps) 1811 sq_revdeps_squash[tid] = deps
1839 elif len(sq_revdeps_new[task]) != 0: 1812 elif len(sq_revdeps_new[tid]) != 0:
1840 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") 1813 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
1841 1814
1842 # Resolve setscene inter-task dependencies 1815 # Resolve setscene inter-task dependencies
1843 # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" 1816 # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
1844 # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies 1817 # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies
1845 for task in self.rqdata.runq_setscene: 1818 for tid in self.rqdata.runq_setscene_tids:
1846 realid = self.rqdata.taskData.gettask_id(self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]], self.rqdata.runq_task[task] + "_setscene", False) 1819 realtid = tid + "_setscene"
1847 idepends = self.rqdata.taskData.tasks_idepends[realid] 1820 idepends = self.rqdata.taskData.taskentries[realtid].idepends
1848 for (depid, idependtask) in idepends: 1821 for (depname, idependtask) in idepends:
1849 if depid not in self.rqdata.taskData.build_targets: 1822
1823 if depname not in self.rqdata.taskData.build_targets:
1850 continue 1824 continue
1851 1825
1852 depdata = self.rqdata.taskData.build_targets[depid][0] 1826 depfn = self.rqdata.taskData.build_targets[depname][0]
1853 if depdata is None: 1827 if depfn is None:
1854 continue 1828 continue
1855 dep = self.rqdata.taskData.fn_index[depdata] 1829 deptid = depfn + ":" + idependtask.replace("_setscene", "")
1856 taskid = self.rqdata.get_task_id(self.rqdata.taskData.getfn_id(dep), idependtask.replace("_setscene", "")) 1830 if deptid not in self.rqdata.runtaskentries:
1857 if taskid is None: 1831 bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask))
1858 bb.msg.fatal("RunQueue", "Task %s_setscene depends upon non-existent task %s:%s" % (self.rqdata.get_user_idstring(task), dep, idependtask))
1859 1832
1860 if not self.rqdata.runq_setscene.index(taskid) in self.sq_harddeps: 1833 if not deptid in self.sq_harddeps:
1861 self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)] = set() 1834 self.sq_harddeps[deptid] = set()
1862 self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)].add(self.rqdata.runq_setscene.index(task)) 1835 self.sq_harddeps[deptid].add(tid)
1863 1836
1864 sq_revdeps_squash[self.rqdata.runq_setscene.index(task)].add(self.rqdata.runq_setscene.index(taskid)) 1837 sq_revdeps_squash[tid].add(deptid)
1865 # Have to zero this to avoid circular dependencies 1838 # Have to zero this to avoid circular dependencies
1866 sq_revdeps_squash[self.rqdata.runq_setscene.index(taskid)] = set() 1839 sq_revdeps_squash[deptid] = set()
1867 1840
1868 for task in self.sq_harddeps: 1841 for task in self.sq_harddeps:
1869 for dep in self.sq_harddeps[task]: 1842 for dep in self.sq_harddeps[task]:
1870 sq_revdeps_squash[dep].add(task) 1843 sq_revdeps_squash[dep].add(task)
1871 1844
1872 #for task in range(len(sq_revdeps_squash)): 1845 #for tid in sq_revdeps_squash:
1873 # realtask = self.rqdata.runq_setscene[task] 1846 # for dep in sq_revdeps_squash[tid]:
1874 # bb.warn("Task %s: %s_setscene is %s " % (task, self.rqdata.get_user_idstring(realtask) , sq_revdeps_squash[task])) 1847 # data = data + "\n %s" % dep
1848 # bb.warn("Task %s_setscene: is %s " % (tid, data
1875 1849
1876 self.sq_deps = [] 1850 self.sq_deps = {}
1877 self.sq_revdeps = sq_revdeps_squash 1851 self.sq_revdeps = sq_revdeps_squash
1878 self.sq_revdeps2 = copy.deepcopy(self.sq_revdeps) 1852 self.sq_revdeps2 = copy.deepcopy(self.sq_revdeps)
1879 1853
1880 for task in range(len(self.sq_revdeps)): 1854 for tid in self.sq_revdeps:
1881 self.sq_deps.append(set()) 1855 self.sq_deps[tid] = set()
1882 for task in range(len(self.sq_revdeps)): 1856 for tid in self.sq_revdeps:
1883 for dep in self.sq_revdeps[task]: 1857 for dep in self.sq_revdeps[tid]:
1884 self.sq_deps[dep].add(task) 1858 self.sq_deps[dep].add(tid)
1885 1859
1886 for task in range(len(self.sq_revdeps)): 1860 for tid in self.sq_revdeps:
1887 if len(self.sq_revdeps[task]) == 0: 1861 if len(self.sq_revdeps[tid]) == 0:
1888 self.runq_buildable.add(task) 1862 self.runq_buildable.add(tid)
1889 1863
1890 self.outrightfail = [] 1864 self.outrightfail = []
1891 if self.rq.hashvalidate: 1865 if self.rq.hashvalidate:
@@ -1896,35 +1870,35 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1896 sq_task = [] 1870 sq_task = []
1897 noexec = [] 1871 noexec = []
1898 stamppresent = [] 1872 stamppresent = []
1899 for task in range(len(self.sq_revdeps)): 1873 for tid in self.sq_revdeps:
1900 realtask = self.rqdata.runq_setscene[task] 1874 fn = fn_from_tid(tid)
1901 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]] 1875 taskname = taskname_from_tid(tid)
1902 taskname = self.rqdata.runq_task[realtask] 1876
1903 taskdep = self.rqdata.dataCache.task_deps[fn] 1877 taskdep = self.rqdata.dataCache.task_deps[fn]
1904 1878
1905 if 'noexec' in taskdep and taskname in taskdep['noexec']: 1879 if 'noexec' in taskdep and taskname in taskdep['noexec']:
1906 noexec.append(task) 1880 noexec.append(tid)
1907 self.task_skip(task) 1881 self.task_skip(tid)
1908 bb.build.make_stamp(taskname + "_setscene", self.rqdata.dataCache, fn) 1882 bb.build.make_stamp(taskname + "_setscene", self.rqdata.dataCache, fn)
1909 continue 1883 continue
1910 1884
1911 if self.rq.check_stamp_task(realtask, taskname + "_setscene", cache=self.stampcache): 1885 if self.rq.check_stamp_task(tid, taskname + "_setscene", cache=self.stampcache):
1912 logger.debug(2, 'Setscene stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(realtask)) 1886 logger.debug(2, 'Setscene stamp current for task %s', tid)
1913 stamppresent.append(task) 1887 stamppresent.append(tid)
1914 self.task_skip(task) 1888 self.task_skip(tid)
1915 continue 1889 continue
1916 1890
1917 if self.rq.check_stamp_task(realtask, taskname, recurse = True, cache=self.stampcache): 1891 if self.rq.check_stamp_task(tid, taskname, recurse = True, cache=self.stampcache):
1918 logger.debug(2, 'Normal stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(realtask)) 1892 logger.debug(2, 'Normal stamp current for task %s', tid)
1919 stamppresent.append(task) 1893 stamppresent.append(tid)
1920 self.task_skip(task) 1894 self.task_skip(tid)
1921 continue 1895 continue
1922 1896
1923 sq_fn.append(fn) 1897 sq_fn.append(fn)
1924 sq_hashfn.append(self.rqdata.dataCache.hashfn[fn]) 1898 sq_hashfn.append(self.rqdata.dataCache.hashfn[fn])
1925 sq_hash.append(self.rqdata.runq_hash[realtask]) 1899 sq_hash.append(self.rqdata.runtaskentries[tid].hash)
1926 sq_taskname.append(taskname) 1900 sq_taskname.append(taskname)
1927 sq_task.append(task) 1901 sq_task.append(tid)
1928 call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)" 1902 call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
1929 locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data } 1903 locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data }
1930 valid = bb.utils.better_eval(call, locs) 1904 valid = bb.utils.better_eval(call, locs)
@@ -1933,12 +1907,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1933 for v in valid: 1907 for v in valid:
1934 valid_new.append(sq_task[v]) 1908 valid_new.append(sq_task[v])
1935 1909
1936 for task in range(len(self.sq_revdeps)): 1910 for tid in self.sq_revdeps:
1937 if task not in valid_new and task not in noexec: 1911 if tid not in valid_new and tid not in noexec:
1938 realtask = self.rqdata.runq_setscene[task] 1912 logger.debug(2, 'No package found, so skipping setscene task %s', tid)
1939 logger.debug(2, 'No package found, so skipping setscene task %s', 1913 self.outrightfail.append(tid)
1940 self.rqdata.get_user_idstring(realtask))
1941 self.outrightfail.append(task)
1942 1914
1943 logger.info('Executing SetScene Tasks') 1915 logger.info('Executing SetScene Tasks')
1944 1916
@@ -1947,9 +1919,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1947 def scenequeue_updatecounters(self, task, fail = False): 1919 def scenequeue_updatecounters(self, task, fail = False):
1948 for dep in self.sq_deps[task]: 1920 for dep in self.sq_deps[task]:
1949 if fail and task in self.sq_harddeps and dep in self.sq_harddeps[task]: 1921 if fail and task in self.sq_harddeps and dep in self.sq_harddeps[task]:
1950 realtask = self.rqdata.runq_setscene[task] 1922 logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
1951 realdep = self.rqdata.runq_setscene[dep]
1952 logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (self.rqdata.get_user_idstring(realtask), self.rqdata.get_user_idstring(realdep)))
1953 self.scenequeue_updatecounters(dep, fail) 1923 self.scenequeue_updatecounters(dep, fail)
1954 continue 1924 continue
1955 if task not in self.sq_revdeps2[dep]: 1925 if task not in self.sq_revdeps2[dep]:
@@ -1966,10 +1936,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1966 completed dependencies as buildable 1936 completed dependencies as buildable
1967 """ 1937 """
1968 1938
1969 index = self.rqdata.runq_setscene[task] 1939 logger.debug(1, 'Found task %s which could be accelerated', task)
1970 logger.debug(1, 'Found task %s which could be accelerated',
1971 self.rqdata.get_user_idstring(index))
1972
1973 self.scenequeue_covered.add(task) 1940 self.scenequeue_covered.add(task)
1974 self.scenequeue_updatecounters(task) 1941 self.scenequeue_updatecounters(task)
1975 1942
@@ -1989,7 +1956,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1989 self.runq_buildable.add(task) 1956 self.runq_buildable.add(task)
1990 self.stats.taskCompleted() 1957 self.stats.taskCompleted()
1991 self.stats.taskSkipped() 1958 self.stats.taskSkipped()
1992 index = self.rqdata.runq_setscene[task]
1993 self.scenequeue_notcovered.add(task) 1959 self.scenequeue_notcovered.add(task)
1994 self.scenequeue_updatecounters(task, True) 1960 self.scenequeue_updatecounters(task, True)
1995 1961
@@ -2010,20 +1976,19 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2010 task = None 1976 task = None
2011 if self.stats.active < self.number_tasks: 1977 if self.stats.active < self.number_tasks:
2012 # Find the next setscene to run 1978 # Find the next setscene to run
2013 for nexttask in range(self.stats.total): 1979 for nexttask in self.rqdata.runq_setscene_tids:
2014 if nexttask in self.runq_buildable and nexttask not in self.runq_running: 1980 if nexttask in self.runq_buildable and nexttask not in self.runq_running:
2015 if nexttask in self.unskippable: 1981 if nexttask in self.unskippable:
2016 logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask])) 1982 logger.debug(2, "Setscene task %s is unskippable" % nexttask)
2017 if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True): 1983 if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True):
2018 realtask = self.rqdata.runq_setscene[nexttask] 1984 fn = fn_from_tid(nexttask)
2019 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]]
2020 foundtarget = False 1985 foundtarget = False
2021 for target in self.rqdata.target_pairs: 1986 for target in self.rqdata.target_pairs:
2022 if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]: 1987 if target[0] == fn and target[1] == taskname_from_tid(nexttask):
2023 foundtarget = True 1988 foundtarget = True
2024 break 1989 break
2025 if not foundtarget: 1990 if not foundtarget:
2026 logger.debug(2, "Skipping setscene for task %s" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask])) 1991 logger.debug(2, "Skipping setscene for task %s" % nexttask)
2027 self.task_skip(nexttask) 1992 self.task_skip(nexttask)
2028 self.scenequeue_notneeded.add(nexttask) 1993 self.scenequeue_notneeded.add(nexttask)
2029 return True 1994 return True
@@ -2033,25 +1998,21 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2033 task = nexttask 1998 task = nexttask
2034 break 1999 break
2035 if task is not None: 2000 if task is not None:
2036 realtask = self.rqdata.runq_setscene[task] 2001 fn = fn_from_tid(task)
2037 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]] 2002 taskname = taskname_from_tid(task) + "_setscene"
2038 2003 if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache):
2039 taskname = self.rqdata.runq_task[realtask] + "_setscene" 2004 logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task)
2040 if self.rq.check_stamp_task(realtask, self.rqdata.runq_task[realtask], recurse = True, cache=self.stampcache):
2041 logger.debug(2, 'Stamp for underlying task %s(%s) is current, so skipping setscene variant',
2042 task, self.rqdata.get_user_idstring(realtask))
2043 self.task_failoutright(task) 2005 self.task_failoutright(task)
2044 return True 2006 return True
2045 2007
2046 if self.cooker.configuration.force: 2008 if self.cooker.configuration.force:
2047 for target in self.rqdata.target_pairs: 2009 for target in self.rqdata.target_pairs:
2048 if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]: 2010 if target[0] == fn and target[1] == taskname_from_tid(task):
2049 self.task_failoutright(task) 2011 self.task_failoutright(task)
2050 return True 2012 return True
2051 2013
2052 if self.rq.check_stamp_task(realtask, taskname, cache=self.stampcache): 2014 if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
2053 logger.debug(2, 'Setscene stamp current task %s(%s), so skip it and its dependencies', 2015 logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task)
2054 task, self.rqdata.get_user_idstring(realtask))
2055 self.task_skip(task) 2016 self.task_skip(task)
2056 return True 2017 return True
2057 2018
@@ -2062,10 +2023,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2062 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: 2023 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
2063 if not self.rq.fakeworker: 2024 if not self.rq.fakeworker:
2064 self.rq.start_fakeworker(self) 2025 self.rq.start_fakeworker(self)
2065 self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>") 2026 self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
2066 self.rq.fakeworker.stdin.flush() 2027 self.rq.fakeworker.stdin.flush()
2067 else: 2028 else:
2068 self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>") 2029 self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
2069 self.rq.worker.stdin.flush() 2030 self.rq.worker.stdin.flush()
2070 2031
2071 self.runq_running.add(task) 2032 self.runq_running.add(task)
@@ -2077,17 +2038,17 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2077 self.rq.read_workers() 2038 self.rq.read_workers()
2078 return self.rq.active_fds() 2039 return self.rq.active_fds()
2079 2040
2080 #for task in range(self.stats.total): 2041 #for tid in self.sq_revdeps:
2081 # if task not in self.runq_running: 2042 # if tid not in self.runq_running:
2082 # buildable = task in self.runq_buildable 2043 # buildable = tid in self.runq_buildable
2083 # revdeps = self.sq_revdeps[task] 2044 # revdeps = self.sq_revdeps[tid]
2084 # bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task]))) 2045 # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps)))
2085 2046
2086 # Convert scenequeue_covered task numbers into full taskgraph ids 2047 # Convert scenequeue_covered task numbers into full taskgraph ids
2087 oldcovered = self.scenequeue_covered 2048 oldcovered = self.scenequeue_covered
2088 self.rq.scenequeue_covered = set() 2049 self.rq.scenequeue_covered = set()
2089 for task in oldcovered: 2050 for task in oldcovered:
2090 self.rq.scenequeue_covered.add(self.rqdata.runq_setscene[task]) 2051 self.rq.scenequeue_covered.add(task)
2091 2052
2092 logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered)) 2053 logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered))
2093 2054
@@ -2099,8 +2060,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2099 return True 2060 return True
2100 2061
2101 def runqueue_process_waitpid(self, task, status): 2062 def runqueue_process_waitpid(self, task, status):
2102 task = self.rq.rqdata.runq_setscene.index(task)
2103
2104 RunQueueExecute.runqueue_process_waitpid(self, task, status) 2063 RunQueueExecute.runqueue_process_waitpid(self, task, status)
2105 2064
2106class TaskFailure(Exception): 2065class TaskFailure(Exception):
@@ -2127,9 +2086,9 @@ class runQueueEvent(bb.event.Event):
2127 """ 2086 """
2128 def __init__(self, task, stats, rq): 2087 def __init__(self, task, stats, rq):
2129 self.taskid = task 2088 self.taskid = task
2130 self.taskstring = rq.rqdata.get_user_idstring(task) 2089 self.taskstring = task
2131 self.taskname = rq.rqdata.get_task_name(task) 2090 self.taskname = taskname_from_tid(task)
2132 self.taskfile = rq.rqdata.get_task_file(task) 2091 self.taskfile = fn_from_tid(task)
2133 self.taskhash = rq.rqdata.get_task_hash(task) 2092 self.taskhash = rq.rqdata.get_task_hash(task)
2134 self.stats = stats.copy() 2093 self.stats = stats.copy()
2135 bb.event.Event.__init__(self) 2094 bb.event.Event.__init__(self)
@@ -2140,11 +2099,10 @@ class sceneQueueEvent(runQueueEvent):
2140 """ 2099 """
2141 def __init__(self, task, stats, rq, noexec=False): 2100 def __init__(self, task, stats, rq, noexec=False):
2142 runQueueEvent.__init__(self, task, stats, rq) 2101 runQueueEvent.__init__(self, task, stats, rq)
2143 realtask = rq.rqdata.runq_setscene[task] 2102 self.taskstring = task + "_setscene"
2144 self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene") 2103 self.taskname = taskname_from_tid(task) + "_setscene"
2145 self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene" 2104 self.taskfile = fn_from_tid(task)
2146 self.taskfile = rq.rqdata.get_task_file(realtask) 2105 self.taskhash = rq.rqdata.get_task_hash(task)
2147 self.taskhash = rq.rqdata.get_task_hash(realtask)
2148 2106
2149class runQueueTaskStarted(runQueueEvent): 2107class runQueueTaskStarted(runQueueEvent):
2150 """ 2108 """
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
index 65628c86f2..d8bdbcabf9 100644
--- a/bitbake/lib/bb/taskdata.py
+++ b/bitbake/lib/bb/taskdata.py
@@ -37,27 +37,24 @@ def re_match_strings(target, strings):
37 return any(name == target or re.match(name, target) 37 return any(name == target or re.match(name, target)
38 for name in strings) 38 for name in strings)
39 39
40class TaskEntry:
41 def __init__(self):
42 self.tdepends = []
43 self.idepends = []
44 self.irdepends = []
45
40class TaskData: 46class TaskData:
41 """ 47 """
42 BitBake Task Data implementation 48 BitBake Task Data implementation
43 """ 49 """
44 def __init__(self, abort = True, tryaltconfigs = False, skiplist = None, allowincomplete = False): 50 def __init__(self, abort = True, tryaltconfigs = False, skiplist = None, allowincomplete = False):
45 self.build_names_index = []
46 self.run_names_index = []
47 self.fn_index = []
48
49 self.build_targets = {} 51 self.build_targets = {}
50 self.run_targets = {} 52 self.run_targets = {}
51 53
52 self.external_targets = [] 54 self.external_targets = []
53 55
54 self.tasks_fnid = [] 56 self.seenfns = []
55 self.tasks_name = [] 57 self.taskentries = {}
56 self.tasks_tdepends = []
57 self.tasks_idepends = []
58 self.tasks_irdepends = []
59 # Cache to speed up task ID lookups
60 self.tasks_lookup = {}
61 58
62 self.depids = {} 59 self.depids = {}
63 self.rdepids = {} 60 self.rdepids = {}
@@ -66,7 +63,7 @@ class TaskData:
66 63
67 self.failed_deps = [] 64 self.failed_deps = []
68 self.failed_rdeps = [] 65 self.failed_rdeps = []
69 self.failed_fnids = [] 66 self.failed_fns = []
70 67
71 self.abort = abort 68 self.abort = abort
72 self.tryaltconfigs = tryaltconfigs 69 self.tryaltconfigs = tryaltconfigs
@@ -74,88 +71,6 @@ class TaskData:
74 71
75 self.skiplist = skiplist 72 self.skiplist = skiplist
76 73
77 def getbuild_id(self, name):
78 """
79 Return an ID number for the build target name.
80 If it doesn't exist, create one.
81 """
82 if not name in self.build_names_index:
83 self.build_names_index.append(name)
84 return len(self.build_names_index) - 1
85
86 return self.build_names_index.index(name)
87
88 def getrun_id(self, name):
89 """
90 Return an ID number for the run target name.
91 If it doesn't exist, create one.
92 """
93 if not name in self.run_names_index:
94 self.run_names_index.append(name)
95 return len(self.run_names_index) - 1
96
97 return self.run_names_index.index(name)
98
99 def getfn_id(self, name):
100 """
101 Return an ID number for the filename.
102 If it doesn't exist, create one.
103 """
104 if not name in self.fn_index:
105 self.fn_index.append(name)
106 return len(self.fn_index) - 1
107
108 return self.fn_index.index(name)
109
110 def gettask_ids(self, fnid):
111 """
112 Return an array of the ID numbers matching a given fnid.
113 """
114 ids = []
115 if fnid in self.tasks_lookup:
116 for task in self.tasks_lookup[fnid]:
117 ids.append(self.tasks_lookup[fnid][task])
118 return ids
119
120 def gettask_id_fromfnid(self, fnid, task):
121 """
122 Return an ID number for the task matching fnid and task.
123 """
124 if fnid in self.tasks_lookup:
125 if task in self.tasks_lookup[fnid]:
126 return self.tasks_lookup[fnid][task]
127
128 return None
129
130 def gettask_id(self, fn, task, create = True):
131 """
132 Return an ID number for the task matching fn and task.
133 If it doesn't exist, create one by default.
134 Optionally return None instead.
135 """
136 fnid = self.getfn_id(fn)
137
138 if fnid in self.tasks_lookup:
139 if task in self.tasks_lookup[fnid]:
140 return self.tasks_lookup[fnid][task]
141
142 if not create:
143 return None
144
145 self.tasks_name.append(task)
146 self.tasks_fnid.append(fnid)
147 self.tasks_tdepends.append([])
148 self.tasks_idepends.append([])
149 self.tasks_irdepends.append([])
150
151 listid = len(self.tasks_name) - 1
152
153 if fnid not in self.tasks_lookup:
154 self.tasks_lookup[fnid] = {}
155 self.tasks_lookup[fnid][task] = listid
156
157 return listid
158
159 def add_tasks(self, fn, dataCache): 74 def add_tasks(self, fn, dataCache):
160 """ 75 """
161 Add tasks for a given fn to the database 76 Add tasks for a given fn to the database
@@ -163,29 +78,31 @@ class TaskData:
163 78
164 task_deps = dataCache.task_deps[fn] 79 task_deps = dataCache.task_deps[fn]
165 80
166 fnid = self.getfn_id(fn) 81 if fn in self.failed_fns:
167
168 if fnid in self.failed_fnids:
169 bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...") 82 bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
170 83
171 # Check if we've already seen this fn 84 # Check if we've already seen this fn
172 if fnid in self.tasks_fnid: 85 if fn in self.seenfns:
173 return 86 return
174 87
88 self.seenfns.append(fn)
89
175 self.add_extra_deps(fn, dataCache) 90 self.add_extra_deps(fn, dataCache)
176 91
177 for task in task_deps['tasks']: 92 for task in task_deps['tasks']:
178 93
94 tid = "%s:%s" % (fn, task)
95 self.taskentries[tid] = TaskEntry()
96
179 # Work out task dependencies 97 # Work out task dependencies
180 parentids = [] 98 parentids = []
181 for dep in task_deps['parents'][task]: 99 for dep in task_deps['parents'][task]:
182 if dep not in task_deps['tasks']: 100 if dep not in task_deps['tasks']:
183 bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep)) 101 bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep, dep))
184 continue 102 continue
185 parentid = self.gettask_id(fn, dep) 103 parentid = "%s:%s" % (fn, dep)
186 parentids.append(parentid) 104 parentids.append(parentid)
187 taskid = self.gettask_id(fn, task) 105 self.taskentries[tid].tdepends.extend(parentids)
188 self.tasks_tdepends[taskid].extend(parentids)
189 106
190 # Touch all intertask dependencies 107 # Touch all intertask dependencies
191 if 'depends' in task_deps and task in task_deps['depends']: 108 if 'depends' in task_deps and task in task_deps['depends']:
@@ -194,29 +111,30 @@ class TaskData:
194 if dep: 111 if dep:
195 if ":" not in dep: 112 if ":" not in dep:
196 bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep)) 113 bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep))
197 ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1])) 114 ids.append(((dep.split(":")[0]), dep.split(":")[1]))
198 self.tasks_idepends[taskid].extend(ids) 115 self.seen_build_target(dep.split(":")[0])
116 self.taskentries[tid].idepends.extend(ids)
199 if 'rdepends' in task_deps and task in task_deps['rdepends']: 117 if 'rdepends' in task_deps and task in task_deps['rdepends']:
200 ids = [] 118 ids = []
201 for dep in task_deps['rdepends'][task].split(): 119 for dep in task_deps['rdepends'][task].split():
202 if dep: 120 if dep:
203 if ":" not in dep: 121 if ":" not in dep:
204 bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep)) 122 bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':' character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep))
205 ids.append(((self.getrun_id(dep.split(":")[0])), dep.split(":")[1])) 123 ids.append(((dep.split(":")[0]), dep.split(":")[1]))
206 self.tasks_irdepends[taskid].extend(ids) 124 self.seen_run_target(dep.split(":")[0])
207 125 self.taskentries[tid].irdepends.extend(ids)
208 126
209 # Work out build dependencies 127 # Work out build dependencies
210 if not fnid in self.depids: 128 if not fn in self.depids:
211 dependids = {} 129 dependids = set()
212 for depend in dataCache.deps[fn]: 130 for depend in dataCache.deps[fn]:
213 dependids[self.getbuild_id(depend)] = None 131 dependids.add(depend)
214 self.depids[fnid] = dependids.keys() 132 self.depids[fn] = list(dependids)
215 logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn) 133 logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
216 134
217 # Work out runtime dependencies 135 # Work out runtime dependencies
218 if not fnid in self.rdepids: 136 if not fn in self.rdepids:
219 rdependids = {} 137 rdependids = set()
220 rdepends = dataCache.rundeps[fn] 138 rdepends = dataCache.rundeps[fn]
221 rrecs = dataCache.runrecs[fn] 139 rrecs = dataCache.runrecs[fn]
222 rdependlist = [] 140 rdependlist = []
@@ -224,24 +142,26 @@ class TaskData:
224 for package in rdepends: 142 for package in rdepends:
225 for rdepend in rdepends[package]: 143 for rdepend in rdepends[package]:
226 rdependlist.append(rdepend) 144 rdependlist.append(rdepend)
227 rdependids[self.getrun_id(rdepend)] = None 145 rdependids.add(rdepend)
228 for package in rrecs: 146 for package in rrecs:
229 for rdepend in rrecs[package]: 147 for rdepend in rrecs[package]:
230 rreclist.append(rdepend) 148 rreclist.append(rdepend)
231 rdependids[self.getrun_id(rdepend)] = None 149 rdependids.add(rdepend)
232 if rdependlist: 150 if rdependlist:
233 logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn) 151 logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
234 if rreclist: 152 if rreclist:
235 logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn) 153 logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
236 self.rdepids[fnid] = rdependids.keys() 154 self.rdepids[fn] = list(rdependids)
237 155
238 for dep in self.depids[fnid]: 156 for dep in self.depids[fn]:
157 self.seen_build_target(dep)
239 if dep in self.failed_deps: 158 if dep in self.failed_deps:
240 self.fail_fnid(fnid) 159 self.fail_fn(fn)
241 return 160 return
242 for dep in self.rdepids[fnid]: 161 for dep in self.rdepids[fn]:
162 self.seen_run_target(dep)
243 if dep in self.failed_rdeps: 163 if dep in self.failed_rdeps:
244 self.fail_fnid(fnid) 164 self.fail_fn(fn)
245 return 165 return
246 166
247 def add_extra_deps(self, fn, dataCache): 167 def add_extra_deps(self, fn, dataCache):
@@ -263,9 +183,7 @@ class TaskData:
263 """ 183 """
264 Have we a build target matching this name? 184 Have we a build target matching this name?
265 """ 185 """
266 targetid = self.getbuild_id(target) 186 if target in self.build_targets and self.build_targets[target]:
267
268 if targetid in self.build_targets:
269 return True 187 return True
270 return False 188 return False
271 189
@@ -273,50 +191,54 @@ class TaskData:
273 """ 191 """
274 Have we a runtime target matching this name? 192 Have we a runtime target matching this name?
275 """ 193 """
276 targetid = self.getrun_id(target) 194 if target in self.run_targets and self.run_targets[target]:
277
278 if targetid in self.run_targets:
279 return True 195 return True
280 return False 196 return False
281 197
198 def seen_build_target(self, name):
199 """
200 Maintain a list of build targets
201 """
202 if name not in self.build_targets:
203 self.build_targets[name] = []
204
282 def add_build_target(self, fn, item): 205 def add_build_target(self, fn, item):
283 """ 206 """
284 Add a build target. 207 Add a build target.
285 If already present, append the provider fn to the list 208 If already present, append the provider fn to the list
286 """ 209 """
287 targetid = self.getbuild_id(item) 210 if item in self.build_targets:
288 fnid = self.getfn_id(fn) 211 if fn in self.build_targets[item]:
289
290 if targetid in self.build_targets:
291 if fnid in self.build_targets[targetid]:
292 return 212 return
293 self.build_targets[targetid].append(fnid) 213 self.build_targets[item].append(fn)
294 return 214 return
295 self.build_targets[targetid] = [fnid] 215 self.build_targets[item] = [fn]
216
217 def seen_run_target(self, name):
218 """
219 Maintain a list of runtime build targets
220 """
221 if name not in self.run_targets:
222 self.run_targets[name] = []
296 223
297 def add_runtime_target(self, fn, item): 224 def add_runtime_target(self, fn, item):
298 """ 225 """
299 Add a runtime target. 226 Add a runtime target.
300 If already present, append the provider fn to the list 227 If already present, append the provider fn to the list
301 """ 228 """
302 targetid = self.getrun_id(item) 229 if item in self.run_targets:
303 fnid = self.getfn_id(fn) 230 if fn in self.run_targets[item]:
304
305 if targetid in self.run_targets:
306 if fnid in self.run_targets[targetid]:
307 return 231 return
308 self.run_targets[targetid].append(fnid) 232 self.run_targets[item].append(fn)
309 return 233 return
310 self.run_targets[targetid] = [fnid] 234 self.run_targets[item] = [fn]
311 235
312 def mark_external_target(self, item): 236 def mark_external_target(self, target):
313 """ 237 """
314 Mark a build target as being externally requested 238 Mark a build target as being externally requested
315 """ 239 """
316 targetid = self.getbuild_id(item) 240 if target not in self.external_targets:
317 241 self.external_targets.append(target)
318 if targetid not in self.external_targets:
319 self.external_targets.append(targetid)
320 242
321 def get_unresolved_build_targets(self, dataCache): 243 def get_unresolved_build_targets(self, dataCache):
322 """ 244 """
@@ -324,12 +246,12 @@ class TaskData:
324 are unknown. 246 are unknown.
325 """ 247 """
326 unresolved = [] 248 unresolved = []
327 for target in self.build_names_index: 249 for target in self.build_targets:
328 if re_match_strings(target, dataCache.ignored_dependencies): 250 if re_match_strings(target, dataCache.ignored_dependencies):
329 continue 251 continue
330 if self.build_names_index.index(target) in self.failed_deps: 252 if target in self.failed_deps:
331 continue 253 continue
332 if not self.have_build_target(target): 254 if not self.build_targets[target]:
333 unresolved.append(target) 255 unresolved.append(target)
334 return unresolved 256 return unresolved
335 257
@@ -339,12 +261,12 @@ class TaskData:
339 are unknown. 261 are unknown.
340 """ 262 """
341 unresolved = [] 263 unresolved = []
342 for target in self.run_names_index: 264 for target in self.run_targets:
343 if re_match_strings(target, dataCache.ignored_dependencies): 265 if re_match_strings(target, dataCache.ignored_dependencies):
344 continue 266 continue
345 if self.run_names_index.index(target) in self.failed_rdeps: 267 if target in self.failed_rdeps:
346 continue 268 continue
347 if not self.have_runtime_target(target): 269 if not self.run_targets[target]:
348 unresolved.append(target) 270 unresolved.append(target)
349 return unresolved 271 return unresolved
350 272
@@ -352,50 +274,26 @@ class TaskData:
352 """ 274 """
353 Return a list of providers of item 275 Return a list of providers of item
354 """ 276 """
355 targetid = self.getbuild_id(item) 277 return self.build_targets[item]
356 278
357 return self.build_targets[targetid] 279 def get_dependees(self, item):
358
359 def get_dependees(self, itemid):
360 """ 280 """
361 Return a list of targets which depend on item 281 Return a list of targets which depend on item
362 """ 282 """
363 dependees = [] 283 dependees = []
364 for fnid in self.depids: 284 for fn in self.depids:
365 if itemid in self.depids[fnid]: 285 if item in self.depids[fn]:
366 dependees.append(fnid) 286 dependees.append(fn)
367 return dependees
368
369 def get_dependees_str(self, item):
370 """
371 Return a list of targets which depend on item as a user readable string
372 """
373 itemid = self.getbuild_id(item)
374 dependees = []
375 for fnid in self.depids:
376 if itemid in self.depids[fnid]:
377 dependees.append(self.fn_index[fnid])
378 return dependees 287 return dependees
379 288
380 def get_rdependees(self, itemid): 289 def get_rdependees(self, item):
381 """ 290 """
382 Return a list of targets which depend on runtime item 291 Return a list of targets which depend on runtime item
383 """ 292 """
384 dependees = [] 293 dependees = []
385 for fnid in self.rdepids: 294 for fn in self.rdepids:
386 if itemid in self.rdepids[fnid]: 295 if item in self.rdepids[fn]:
387 dependees.append(fnid) 296 dependees.append(fn)
388 return dependees
389
390 def get_rdependees_str(self, item):
391 """
392 Return a list of targets which depend on runtime item as a user readable string
393 """
394 itemid = self.getrun_id(item)
395 dependees = []
396 for fnid in self.rdepids:
397 if itemid in self.rdepids[fnid]:
398 dependees.append(self.fn_index[fnid])
399 return dependees 297 return dependees
400 298
401 def get_reasons(self, item, runtime=False): 299 def get_reasons(self, item, runtime=False):
@@ -431,7 +329,7 @@ class TaskData:
431 except bb.providers.NoProvider: 329 except bb.providers.NoProvider:
432 if self.abort: 330 if self.abort:
433 raise 331 raise
434 self.remove_buildtarget(self.getbuild_id(item)) 332 self.remove_buildtarget(item)
435 333
436 self.mark_external_target(item) 334 self.mark_external_target(item)
437 335
@@ -453,7 +351,7 @@ class TaskData:
453 new = dataCache.pkg_fn[fn] + " RPROVIDES " + item 351 new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
454 if new not in close_matches: 352 if new not in close_matches:
455 close_matches.append(new) 353 close_matches.append(new)
456 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData) 354 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
457 raise bb.providers.NoProvider(item) 355 raise bb.providers.NoProvider(item)
458 356
459 if self.have_build_target(item): 357 if self.have_build_target(item):
@@ -462,10 +360,10 @@ class TaskData:
462 all_p = dataCache.providers[item] 360 all_p = dataCache.providers[item]
463 361
464 eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache) 362 eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
465 eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids] 363 eligible = [p for p in eligible if not p in self.failed_fns]
466 364
467 if not eligible: 365 if not eligible:
468 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData) 366 bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No eligible PROVIDERs exist for '%s'" % item]), cfgData)
469 raise bb.providers.NoProvider(item) 367 raise bb.providers.NoProvider(item)
470 368
471 if len(eligible) > 1 and foundUnique == False: 369 if len(eligible) > 1 and foundUnique == False:
@@ -477,8 +375,7 @@ class TaskData:
477 self.consider_msgs_cache.append(item) 375 self.consider_msgs_cache.append(item)
478 376
479 for fn in eligible: 377 for fn in eligible:
480 fnid = self.getfn_id(fn) 378 if fn in self.failed_fns:
481 if fnid in self.failed_fnids:
482 continue 379 continue
483 logger.debug(2, "adding %s to satisfy %s", fn, item) 380 logger.debug(2, "adding %s to satisfy %s", fn, item)
484 self.add_build_target(fn, item) 381 self.add_build_target(fn, item)
@@ -502,14 +399,14 @@ class TaskData:
502 all_p = bb.providers.getRuntimeProviders(dataCache, item) 399 all_p = bb.providers.getRuntimeProviders(dataCache, item)
503 400
504 if not all_p: 401 if not all_p:
505 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=self.get_reasons(item, True)), cfgData) 402 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=self.get_reasons(item, True)), cfgData)
506 raise bb.providers.NoRProvider(item) 403 raise bb.providers.NoRProvider(item)
507 404
508 eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache) 405 eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
509 eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids] 406 eligible = [p for p in eligible if not p in self.failed_fns]
510 407
511 if not eligible: 408 if not eligible:
512 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData) 409 bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item), reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
513 raise bb.providers.NoRProvider(item) 410 raise bb.providers.NoRProvider(item)
514 411
515 if len(eligible) > 1 and numberPreferred == 0: 412 if len(eligible) > 1 and numberPreferred == 0:
@@ -531,82 +428,80 @@ class TaskData:
531 428
532 # run through the list until we find one that we can build 429 # run through the list until we find one that we can build
533 for fn in eligible: 430 for fn in eligible:
534 fnid = self.getfn_id(fn) 431 if fn in self.failed_fns:
535 if fnid in self.failed_fnids:
536 continue 432 continue
537 logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item) 433 logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
538 self.add_runtime_target(fn, item) 434 self.add_runtime_target(fn, item)
539 self.add_tasks(fn, dataCache) 435 self.add_tasks(fn, dataCache)
540 436
541 def fail_fnid(self, fnid, missing_list=None): 437 def fail_fn(self, fn, missing_list=None):
542 """ 438 """
543 Mark a file as failed (unbuildable) 439 Mark a file as failed (unbuildable)
544 Remove any references from build and runtime provider lists 440 Remove any references from build and runtime provider lists
545 441
546 missing_list, A list of missing requirements for this target 442 missing_list, A list of missing requirements for this target
547 """ 443 """
548 if fnid in self.failed_fnids: 444 if fn in self.failed_fns:
549 return 445 return
550 if not missing_list: 446 if not missing_list:
551 missing_list = [] 447 missing_list = []
552 logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid]) 448 logger.debug(1, "File '%s' is unbuildable, removing...", fn)
553 self.failed_fnids.append(fnid) 449 self.failed_fns.append(fn)
554 for target in self.build_targets: 450 for target in self.build_targets:
555 if fnid in self.build_targets[target]: 451 if fn in self.build_targets[target]:
556 self.build_targets[target].remove(fnid) 452 self.build_targets[target].remove(fn)
557 if len(self.build_targets[target]) == 0: 453 if len(self.build_targets[target]) == 0:
558 self.remove_buildtarget(target, missing_list) 454 self.remove_buildtarget(target, missing_list)
559 for target in self.run_targets: 455 for target in self.run_targets:
560 if fnid in self.run_targets[target]: 456 if fn in self.run_targets[target]:
561 self.run_targets[target].remove(fnid) 457 self.run_targets[target].remove(fn)
562 if len(self.run_targets[target]) == 0: 458 if len(self.run_targets[target]) == 0:
563 self.remove_runtarget(target, missing_list) 459 self.remove_runtarget(target, missing_list)
564 460
565 def remove_buildtarget(self, targetid, missing_list=None): 461 def remove_buildtarget(self, target, missing_list=None):
566 """ 462 """
567 Mark a build target as failed (unbuildable) 463 Mark a build target as failed (unbuildable)
568 Trigger removal of any files that have this as a dependency 464 Trigger removal of any files that have this as a dependency
569 """ 465 """
570 if not missing_list: 466 if not missing_list:
571 missing_list = [self.build_names_index[targetid]] 467 missing_list = [target]
572 else: 468 else:
573 missing_list = [self.build_names_index[targetid]] + missing_list 469 missing_list = [target] + missing_list
574 logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.build_names_index[targetid], missing_list) 470 logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
575 self.failed_deps.append(targetid) 471 self.failed_deps.append(target)
576 dependees = self.get_dependees(targetid) 472 dependees = self.get_dependees(target)
577 for fnid in dependees: 473 for fn in dependees:
578 self.fail_fnid(fnid, missing_list) 474 self.fail_fn(fn, missing_list)
579 for taskid in range(len(self.tasks_idepends)): 475 for tid in self.taskentries:
580 idepends = self.tasks_idepends[taskid] 476 for (idepend, idependtask) in self.taskentries[tid].idepends:
581 for (idependid, idependtask) in idepends: 477 if idepend == target:
582 if idependid == targetid: 478 fn = tid.rsplit(":",1)[0]
583 self.fail_fnid(self.tasks_fnid[taskid], missing_list) 479 self.fail_fn(fn, missing_list)
584 480
585 if self.abort and targetid in self.external_targets: 481 if self.abort and target in self.external_targets:
586 target = self.build_names_index[targetid]
587 logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list) 482 logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
588 raise bb.providers.NoProvider(target) 483 raise bb.providers.NoProvider(target)
589 484
590 def remove_runtarget(self, targetid, missing_list=None): 485 def remove_runtarget(self, target, missing_list=None):
591 """ 486 """
592 Mark a run target as failed (unbuildable) 487 Mark a run target as failed (unbuildable)
593 Trigger removal of any files that have this as a dependency 488 Trigger removal of any files that have this as a dependency
594 """ 489 """
595 if not missing_list: 490 if not missing_list:
596 missing_list = [self.run_names_index[targetid]] 491 missing_list = [target]
597 else: 492 else:
598 missing_list = [self.run_names_index[targetid]] + missing_list 493 missing_list = [target] + missing_list
599 494
600 logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", self.run_names_index[targetid], missing_list) 495 logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain was: %s", target, missing_list)
601 self.failed_rdeps.append(targetid) 496 self.failed_rdeps.append(target)
602 dependees = self.get_rdependees(targetid) 497 dependees = self.get_rdependees(target)
603 for fnid in dependees: 498 for fn in dependees:
604 self.fail_fnid(fnid, missing_list) 499 self.fail_fn(fn, missing_list)
605 for taskid in range(len(self.tasks_irdepends)): 500 for tid in self.taskentries:
606 irdepends = self.tasks_irdepends[taskid] 501 for (idepend, idependtask) in self.taskentries[tid].irdepends:
607 for (idependid, idependtask) in irdepends: 502 if idepend == target:
608 if idependid == targetid: 503 fn = tid.rsplit(":",1)[0]
609 self.fail_fnid(self.tasks_fnid[taskid], missing_list) 504 self.fail_fn(fn, missing_list)
610 505
611 def add_unresolved(self, cfgData, dataCache): 506 def add_unresolved(self, cfgData, dataCache):
612 """ 507 """
@@ -620,17 +515,16 @@ class TaskData:
620 self.add_provider_internal(cfgData, dataCache, target) 515 self.add_provider_internal(cfgData, dataCache, target)
621 added = added + 1 516 added = added + 1
622 except bb.providers.NoProvider: 517 except bb.providers.NoProvider:
623 targetid = self.getbuild_id(target) 518 if self.abort and target in self.external_targets and not self.allowincomplete:
624 if self.abort and targetid in self.external_targets and not self.allowincomplete:
625 raise 519 raise
626 if not self.allowincomplete: 520 if not self.allowincomplete:
627 self.remove_buildtarget(targetid) 521 self.remove_buildtarget(target)
628 for target in self.get_unresolved_run_targets(dataCache): 522 for target in self.get_unresolved_run_targets(dataCache):
629 try: 523 try:
630 self.add_rprovider(cfgData, dataCache, target) 524 self.add_rprovider(cfgData, dataCache, target)
631 added = added + 1 525 added = added + 1
632 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider): 526 except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
633 self.remove_runtarget(self.getrun_id(target)) 527 self.remove_runtarget(target)
634 logger.debug(1, "Resolved " + str(added) + " extra dependencies") 528 logger.debug(1, "Resolved " + str(added) + " extra dependencies")
635 if added == 0: 529 if added == 0:
636 break 530 break
@@ -638,13 +532,13 @@ class TaskData:
638 532
639 def get_providermap(self, prefix=None): 533 def get_providermap(self, prefix=None):
640 provmap = {} 534 provmap = {}
641 for name in self.build_names_index: 535 for name in self.build_targets:
642 if prefix and not name.startswith(prefix): 536 if prefix and not name.startswith(prefix):
643 continue 537 continue
644 if self.have_build_target(name): 538 if self.have_build_target(name):
645 provider = self.get_provider(name) 539 provider = self.get_provider(name)
646 if provider: 540 if provider:
647 provmap[name] = self.fn_index[provider[0]] 541 provmap[name] = provider[0]
648 return provmap 542 return provmap
649 543
650 def dump_data(self): 544 def dump_data(self):
@@ -652,39 +546,37 @@ class TaskData:
652 Dump some debug information on the internal data structures 546 Dump some debug information on the internal data structures
653 """ 547 """
654 logger.debug(3, "build_names:") 548 logger.debug(3, "build_names:")
655 logger.debug(3, ", ".join(self.build_names_index)) 549 logger.debug(3, ", ".join(self.build_targets))
656 550
657 logger.debug(3, "run_names:") 551 logger.debug(3, "run_names:")
658 logger.debug(3, ", ".join(self.run_names_index)) 552 logger.debug(3, ", ".join(self.run_targets))
659 553
660 logger.debug(3, "build_targets:") 554 logger.debug(3, "build_targets:")
661 for buildid in range(len(self.build_names_index)): 555 for target in self.build_targets:
662 target = self.build_names_index[buildid]
663 targets = "None" 556 targets = "None"
664 if buildid in self.build_targets: 557 if target in self.build_targets:
665 targets = self.build_targets[buildid] 558 targets = self.build_targets[target]
666 logger.debug(3, " (%s)%s: %s", buildid, target, targets) 559 logger.debug(3, " %s: %s", target, targets)
667 560
668 logger.debug(3, "run_targets:") 561 logger.debug(3, "run_targets:")
669 for runid in range(len(self.run_names_index)): 562 for target in self.run_targets:
670 target = self.run_names_index[runid]
671 targets = "None" 563 targets = "None"
672 if runid in self.run_targets: 564 if target in self.run_targets:
673 targets = self.run_targets[runid] 565 targets = self.run_targets[target]
674 logger.debug(3, " (%s)%s: %s", runid, target, targets) 566 logger.debug(3, " %s: %s", target, targets)
675 567
676 logger.debug(3, "tasks:") 568 logger.debug(3, "tasks:")
677 for task in range(len(self.tasks_name)): 569 for tid in self.taskentries:
678 logger.debug(3, " (%s)%s - %s: %s", 570 logger.debug(3, " %s: %s %s %s",
679 task, 571 tid,
680 self.fn_index[self.tasks_fnid[task]], 572 self.taskentries[tid].idepends,
681 self.tasks_name[task], 573 self.taskentries[tid].irdepends,
682 self.tasks_tdepends[task]) 574 self.taskentries[tid].tdepends)
683 575
684 logger.debug(3, "dependency ids (per fn):") 576 logger.debug(3, "dependency ids (per fn):")
685 for fnid in self.depids: 577 for fn in self.depids:
686 logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.depids[fnid]) 578 logger.debug(3, " %s: %s", fn, self.depids[fn])
687 579
688 logger.debug(3, "runtime dependency ids (per fn):") 580 logger.debug(3, "runtime dependency ids (per fn):")
689 for fnid in self.rdepids: 581 for fn in self.rdepids:
690 logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.rdepids[fnid]) 582 logger.debug(3, " %s: %s", fn, self.rdepids[fn])