diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2016-06-12 23:55:48 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2016-06-15 08:35:07 +0100 |
commit | af04a52bf7d7ae94c187e85028a80dee6d7853f0 (patch) | |
tree | d0c65de172d7ded2e1ba28f09c271da97e10cbe9 /bitbake/lib/bb/runqueue.py | |
parent | 29b4fb17966bfaec74b41cce533725c04ec1078f (diff) | |
download | poky-af04a52bf7d7ae94c187e85028a80dee6d7853f0.tar.gz |
bitbake: taskdata/runqueue: Rewrite without use of ID indirection
I'm not sure what possesed me when I wrote this code originally but its
indirection of everyting to use numeric IDs and position dependent lists
is horrific. Given the way python internals work, its completely and
utterly pointless from performance perspective. It also makes the code
hard to understand and debug since any numeric ID has to be translated
into something human readable.
The hard part is that the IDs are infectous and spread from taskdata
into runqueue and even partly into cooker for the dependency graph
processing. The only real way to deal with this is to convert everything
to use a more sane data structure.
This patch:
* Uses "<fn>:<taskname>" as the ID for tasks rather than a number
* Changes to dict() based structures rather than position dependent lists
* Drops the build name, runtime name and filename ID indexes
On the most part there shouldn't be user visible changes. Sadly we did
leak datastructures to the setscene verify function which has to be
rewritten. To handle this, the variable name used to specifiy the version
changes from BB_SETSCENE_VERIFY_FUNCTION to BB_SETSCENE_VERIFY_FUNCTION2
allowing multiple versions of bitbake to work with suitably written
metadata. Anyone with custom schedulers may also need to change them.
I believe the benefits in code readability and easier debugging far
outweigh those issues though. It also means we have a saner codebase
to add multiconfig support on top of.
During development, I did have some of the original code coexisting with
the new data stores to allow comparision of the data and check it was
working correcty, particuarly for taskdata. I have also compared
task-depends.dot files before and after the change. There should be no
functionality changes in this patch, its purely a data structure change
and that is visible in the patch.
(Bitbake rev: 2c88afb60da54e58f555411a7bd7b006b0c29306)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 958 |
1 files changed, 458 insertions, 500 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index de618ff1d8..b32e8e694d 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -42,6 +42,12 @@ logger = logging.getLogger("BitBake.RunQueue") | |||
42 | 42 | ||
43 | __find_md5__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])' ) | 43 | __find_md5__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])' ) |
44 | 44 | ||
45 | def fn_from_tid(tid): | ||
46 | return tid.rsplit(":", 1)[0] | ||
47 | |||
48 | def taskname_from_tid(tid): | ||
49 | return tid.rsplit(":", 1)[1] | ||
50 | |||
45 | class RunQueueStats: | 51 | class RunQueueStats: |
46 | """ | 52 | """ |
47 | Holds statistics on the tasks handled by the associated runQueue | 53 | Holds statistics on the tasks handled by the associated runQueue |
@@ -97,19 +103,18 @@ class RunQueueScheduler(object): | |||
97 | """ | 103 | """ |
98 | self.rq = runqueue | 104 | self.rq = runqueue |
99 | self.rqdata = rqdata | 105 | self.rqdata = rqdata |
100 | self.numTasks = len(self.rqdata.runq_fnid) | 106 | self.numTasks = len(self.rqdata.runtaskentries) |
101 | 107 | ||
102 | self.prio_map = [] | 108 | self.prio_map = [self.rqdata.runtaskentries.keys()] |
103 | self.prio_map.extend(range(self.numTasks)) | ||
104 | 109 | ||
105 | self.buildable = [] | 110 | self.buildable = [] |
106 | self.stamps = {} | 111 | self.stamps = {} |
107 | for taskid in range(self.numTasks): | 112 | for tid in self.rqdata.runtaskentries: |
108 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]] | 113 | fn = fn_from_tid(tid) |
109 | taskname = self.rqdata.runq_task[taskid] | 114 | taskname = taskname_from_tid(tid) |
110 | self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) | 115 | self.stamps[tid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) |
111 | if taskid in self.rq.runq_buildable: | 116 | if tid in self.rq.runq_buildable: |
112 | self.buildable.append(taskid) | 117 | self.buildable.append(tid) |
113 | 118 | ||
114 | self.rev_prio_map = None | 119 | self.rev_prio_map = None |
115 | 120 | ||
@@ -121,26 +126,26 @@ class RunQueueScheduler(object): | |||
121 | if not self.buildable: | 126 | if not self.buildable: |
122 | return None | 127 | return None |
123 | if len(self.buildable) == 1: | 128 | if len(self.buildable) == 1: |
124 | taskid = self.buildable[0] | 129 | tid = self.buildable[0] |
125 | stamp = self.stamps[taskid] | 130 | stamp = self.stamps[tid] |
126 | if stamp not in self.rq.build_stamps.values(): | 131 | if stamp not in self.rq.build_stamps.values(): |
127 | return taskid | 132 | return tid |
128 | 133 | ||
129 | if not self.rev_prio_map: | 134 | if not self.rev_prio_map: |
130 | self.rev_prio_map = list(range(self.numTasks)) | 135 | self.rev_prio_map = {} |
131 | for taskid in range(self.numTasks): | 136 | for tid in self.rqdata.runtaskentries: |
132 | self.rev_prio_map[self.prio_map[taskid]] = taskid | 137 | self.rev_prio_map[tid] = self.prio_map.index(tid) |
133 | 138 | ||
134 | best = None | 139 | best = None |
135 | bestprio = None | 140 | bestprio = None |
136 | for taskid in self.buildable: | 141 | for tid in self.buildable: |
137 | prio = self.rev_prio_map[taskid] | 142 | prio = self.rev_prio_map[tid] |
138 | if bestprio is None or bestprio > prio: | 143 | if bestprio is None or bestprio > prio: |
139 | stamp = self.stamps[taskid] | 144 | stamp = self.stamps[tid] |
140 | if stamp in self.rq.build_stamps.values(): | 145 | if stamp in self.rq.build_stamps.values(): |
141 | continue | 146 | continue |
142 | bestprio = prio | 147 | bestprio = prio |
143 | best = taskid | 148 | best = tid |
144 | 149 | ||
145 | return best | 150 | return best |
146 | 151 | ||
@@ -167,14 +172,17 @@ class RunQueueSchedulerSpeed(RunQueueScheduler): | |||
167 | """ | 172 | """ |
168 | RunQueueScheduler.__init__(self, runqueue, rqdata) | 173 | RunQueueScheduler.__init__(self, runqueue, rqdata) |
169 | 174 | ||
170 | sortweight = sorted(copy.deepcopy(self.rqdata.runq_weight)) | 175 | weights = {} |
171 | copyweight = copy.deepcopy(self.rqdata.runq_weight) | 176 | for tid in self.rqdata.runtaskentries: |
172 | self.prio_map = [] | 177 | weight = self.rqdata.runtaskentries[tid].weight |
178 | if not weight in weights: | ||
179 | weights[weight] = [] | ||
180 | weights[weight].append(tid) | ||
173 | 181 | ||
174 | for weight in sortweight: | 182 | self.prio_map = [] |
175 | idx = copyweight.index(weight) | 183 | for weight in sorted(weights): |
176 | self.prio_map.append(idx) | 184 | for w in weights[weight]: |
177 | copyweight[idx] = -1 | 185 | self.prio_map.append(w) |
178 | 186 | ||
179 | self.prio_map.reverse() | 187 | self.prio_map.reverse() |
180 | 188 | ||
@@ -191,25 +199,33 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): | |||
191 | def __init__(self, runqueue, rqdata): | 199 | def __init__(self, runqueue, rqdata): |
192 | RunQueueSchedulerSpeed.__init__(self, runqueue, rqdata) | 200 | RunQueueSchedulerSpeed.__init__(self, runqueue, rqdata) |
193 | 201 | ||
194 | #FIXME - whilst this groups all fnids together it does not reorder the | 202 | #FIXME - whilst this groups all fns together it does not reorder the |
195 | #fnid groups optimally. | 203 | #fn groups optimally. |
196 | 204 | ||
197 | basemap = copy.deepcopy(self.prio_map) | 205 | basemap = copy.deepcopy(self.prio_map) |
198 | self.prio_map = [] | 206 | self.prio_map = [] |
199 | while (len(basemap) > 0): | 207 | while (len(basemap) > 0): |
200 | entry = basemap.pop(0) | 208 | entry = basemap.pop(0) |
201 | self.prio_map.append(entry) | 209 | self.prio_map.append(entry) |
202 | fnid = self.rqdata.runq_fnid[entry] | 210 | fn = fn_from_tid(entry) |
203 | todel = [] | 211 | todel = [] |
204 | for entry in basemap: | 212 | for entry in basemap: |
205 | entry_fnid = self.rqdata.runq_fnid[entry] | 213 | entry_fn = fn_from_tid(entry) |
206 | if entry_fnid == fnid: | 214 | if entry_fn == fn: |
207 | todel.append(basemap.index(entry)) | 215 | todel.append(basemap.index(entry)) |
208 | self.prio_map.append(entry) | 216 | self.prio_map.append(entry) |
209 | todel.reverse() | 217 | todel.reverse() |
210 | for idx in todel: | 218 | for idx in todel: |
211 | del basemap[idx] | 219 | del basemap[idx] |
212 | 220 | ||
221 | class RunTaskEntry(object): | ||
222 | def __init__(self): | ||
223 | self.depends = set() | ||
224 | self.revdeps = set() | ||
225 | self.hash = None | ||
226 | self.task = None | ||
227 | self.weight = 1 | ||
228 | |||
213 | class RunQueueData: | 229 | class RunQueueData: |
214 | """ | 230 | """ |
215 | BitBake Run Queue implementation | 231 | BitBake Run Queue implementation |
@@ -228,48 +244,29 @@ class RunQueueData: | |||
228 | self.reset() | 244 | self.reset() |
229 | 245 | ||
230 | def reset(self): | 246 | def reset(self): |
231 | self.runq_fnid = [] | 247 | self.runtaskentries = {} |
232 | self.runq_task = [] | ||
233 | self.runq_depends = [] | ||
234 | self.runq_revdeps = [] | ||
235 | self.runq_hash = [] | ||
236 | 248 | ||
237 | def runq_depends_names(self, ids): | 249 | def runq_depends_names(self, ids): |
238 | import re | 250 | import re |
239 | ret = [] | 251 | ret = [] |
240 | for id in self.runq_depends[ids]: | 252 | for id in ids: |
241 | nam = os.path.basename(self.get_user_idstring(id)) | 253 | nam = os.path.basename(id) |
242 | nam = re.sub("_[^,]*,", ",", nam) | 254 | nam = re.sub("_[^,]*,", ",", nam) |
243 | ret.extend([nam]) | 255 | ret.extend([nam]) |
244 | return ret | 256 | return ret |
245 | 257 | ||
246 | def get_task_name(self, task): | 258 | def get_task_hash(self, tid): |
247 | return self.runq_task[task] | 259 | return self.runtaskentries[tid].hash |
248 | |||
249 | def get_task_file(self, task): | ||
250 | return self.taskData.fn_index[self.runq_fnid[task]] | ||
251 | 260 | ||
252 | def get_task_hash(self, task): | 261 | def get_user_idstring(self, tid, task_name_suffix = ""): |
253 | return self.runq_hash[task] | 262 | return tid + task_name_suffix |
254 | |||
255 | def get_user_idstring(self, task, task_name_suffix = ""): | ||
256 | fn = self.taskData.fn_index[self.runq_fnid[task]] | ||
257 | taskname = self.runq_task[task] + task_name_suffix | ||
258 | return "%s, %s" % (fn, taskname) | ||
259 | 263 | ||
260 | def get_short_user_idstring(self, task, task_name_suffix = ""): | 264 | def get_short_user_idstring(self, task, task_name_suffix = ""): |
261 | fn = self.taskData.fn_index[self.runq_fnid[task]] | 265 | fn = fn_from_tid(task) |
262 | pn = self.dataCache.pkg_fn[fn] | 266 | pn = self.dataCache.pkg_fn[fn] |
263 | taskname = self.runq_task[task] + task_name_suffix | 267 | taskname = taskname_from_tid(task) + task_name_suffix |
264 | return "%s:%s" % (pn, taskname) | 268 | return "%s:%s" % (pn, taskname) |
265 | 269 | ||
266 | |||
267 | def get_task_id(self, fnid, taskname): | ||
268 | for listid in range(len(self.runq_fnid)): | ||
269 | if self.runq_fnid[listid] == fnid and self.runq_task[listid] == taskname: | ||
270 | return listid | ||
271 | return None | ||
272 | |||
273 | def circular_depchains_handler(self, tasks): | 270 | def circular_depchains_handler(self, tasks): |
274 | """ | 271 | """ |
275 | Some tasks aren't buildable, likely due to circular dependency issues. | 272 | Some tasks aren't buildable, likely due to circular dependency issues. |
@@ -314,11 +311,11 @@ class RunQueueData: | |||
314 | return True | 311 | return True |
315 | return False | 312 | return False |
316 | 313 | ||
317 | def find_chains(taskid, prev_chain): | 314 | def find_chains(tid, prev_chain): |
318 | prev_chain.append(taskid) | 315 | prev_chain.append(tid) |
319 | total_deps = [] | 316 | total_deps = [] |
320 | total_deps.extend(self.runq_revdeps[taskid]) | 317 | total_deps.extend(self.runtaskentries[tid].revdeps) |
321 | for revdep in self.runq_revdeps[taskid]: | 318 | for revdep in self.runtaskentries[tid].revdeps: |
322 | if revdep in prev_chain: | 319 | if revdep in prev_chain: |
323 | idx = prev_chain.index(revdep) | 320 | idx = prev_chain.index(revdep) |
324 | # To prevent duplicates, reorder the chain to start with the lowest taskid | 321 | # To prevent duplicates, reorder the chain to start with the lowest taskid |
@@ -329,7 +326,7 @@ class RunQueueData: | |||
329 | valid_chains.append(new_chain) | 326 | valid_chains.append(new_chain) |
330 | msgs.append("Dependency loop #%d found:\n" % len(valid_chains)) | 327 | msgs.append("Dependency loop #%d found:\n" % len(valid_chains)) |
331 | for dep in new_chain: | 328 | for dep in new_chain: |
332 | msgs.append(" Task %s (%s) (dependent Tasks %s)\n" % (dep, self.get_user_idstring(dep), self.runq_depends_names(dep))) | 329 | msgs.append(" Task %s (dependent Tasks %s)\n" % (dep, self.runq_depends_names(self.runtaskentries[dep].depends))) |
333 | msgs.append("\n") | 330 | msgs.append("\n") |
334 | if len(valid_chains) > 10: | 331 | if len(valid_chains) > 10: |
335 | msgs.append("Aborted dependency loops search after 10 matches.\n") | 332 | msgs.append("Aborted dependency loops search after 10 matches.\n") |
@@ -366,25 +363,25 @@ class RunQueueData: | |||
366 | possible to execute due to circular dependencies. | 363 | possible to execute due to circular dependencies. |
367 | """ | 364 | """ |
368 | 365 | ||
369 | numTasks = len(self.runq_fnid) | 366 | numTasks = len(self.runtaskentries) |
370 | weight = [] | 367 | weight = {} |
371 | deps_left = [] | 368 | deps_left = {} |
372 | task_done = [] | 369 | task_done = {} |
373 | 370 | ||
374 | for listid in range(numTasks): | 371 | for tid in self.runtaskentries: |
375 | task_done.append(False) | 372 | task_done[tid] = False |
376 | weight.append(1) | 373 | weight[tid] = 1 |
377 | deps_left.append(len(self.runq_revdeps[listid])) | 374 | deps_left[tid] = len(self.runtaskentries[tid].revdeps) |
378 | 375 | ||
379 | for listid in endpoints: | 376 | for tid in endpoints: |
380 | weight[listid] = 10 | 377 | weight[tid] = 10 |
381 | task_done[listid] = True | 378 | task_done[tid] = True |
382 | 379 | ||
383 | while True: | 380 | while True: |
384 | next_points = [] | 381 | next_points = [] |
385 | for listid in endpoints: | 382 | for tid in endpoints: |
386 | for revdep in self.runq_depends[listid]: | 383 | for revdep in self.runtaskentries[tid].depends: |
387 | weight[revdep] = weight[revdep] + weight[listid] | 384 | weight[revdep] = weight[revdep] + weight[tid] |
388 | deps_left[revdep] = deps_left[revdep] - 1 | 385 | deps_left[revdep] = deps_left[revdep] - 1 |
389 | if deps_left[revdep] == 0: | 386 | if deps_left[revdep] == 0: |
390 | next_points.append(revdep) | 387 | next_points.append(revdep) |
@@ -395,14 +392,15 @@ class RunQueueData: | |||
395 | 392 | ||
396 | # Circular dependency sanity check | 393 | # Circular dependency sanity check |
397 | problem_tasks = [] | 394 | problem_tasks = [] |
398 | for task in range(numTasks): | 395 | for tid in self.runtaskentries: |
399 | if task_done[task] is False or deps_left[task] != 0: | 396 | if task_done[tid] is False or deps_left[tid] != 0: |
400 | problem_tasks.append(task) | 397 | problem_tasks.append(tid) |
401 | logger.debug(2, "Task %s (%s) is not buildable", task, self.get_user_idstring(task)) | 398 | logger.debug(2, "Task %s is not buildable", tid) |
402 | logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n", task_done[task], deps_left[task]) | 399 | logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n", task_done[tid], deps_left[tid]) |
400 | self.runtaskentries[tid].weight = weight[tid] | ||
403 | 401 | ||
404 | if problem_tasks: | 402 | if problem_tasks: |
405 | message = "Unbuildable tasks were found.\n" | 403 | message = "%s unbuildable tasks were found.\n" % len(problem_tasks) |
406 | message = message + "These are usually caused by circular dependencies and any circular dependency chains found will be printed below. Increase the debug level to see a list of unbuildable tasks.\n\n" | 404 | message = message + "These are usually caused by circular dependencies and any circular dependency chains found will be printed below. Increase the debug level to see a list of unbuildable tasks.\n\n" |
407 | message = message + "Identifying dependency loops (this may take a short while)...\n" | 405 | message = message + "Identifying dependency loops (this may take a short while)...\n" |
408 | logger.error(message) | 406 | logger.error(message) |
@@ -422,14 +420,14 @@ class RunQueueData: | |||
422 | to optimise the execution order. | 420 | to optimise the execution order. |
423 | """ | 421 | """ |
424 | 422 | ||
425 | runq_build = [] | 423 | runq_build = {} |
426 | recursivetasks = {} | 424 | recursivetasks = {} |
427 | recursiveitasks = {} | 425 | recursiveitasks = {} |
428 | recursivetasksselfref = set() | 426 | recursivetasksselfref = set() |
429 | 427 | ||
430 | taskData = self.taskData | 428 | taskData = self.taskData |
431 | 429 | ||
432 | if len(taskData.tasks_name) == 0: | 430 | if len(taskData.taskentries) == 0: |
433 | # Nothing to do | 431 | # Nothing to do |
434 | return 0 | 432 | return 0 |
435 | 433 | ||
@@ -447,160 +445,159 @@ class RunQueueData: | |||
447 | # rdeptast, recrdeptask, idepends). | 445 | # rdeptast, recrdeptask, idepends). |
448 | 446 | ||
449 | def add_build_dependencies(depids, tasknames, depends): | 447 | def add_build_dependencies(depids, tasknames, depends): |
450 | for depid in depids: | 448 | for depname in depids: |
451 | # Won't be in build_targets if ASSUME_PROVIDED | 449 | # Won't be in build_targets if ASSUME_PROVIDED |
452 | if depid not in taskData.build_targets: | 450 | if depname not in taskData.build_targets or not taskData.build_targets[depname]: |
453 | continue | 451 | continue |
454 | depdata = taskData.build_targets[depid][0] | 452 | depdata = taskData.build_targets[depname][0] |
455 | if depdata is None: | 453 | if depdata is None: |
456 | continue | 454 | continue |
457 | for taskname in tasknames: | 455 | for taskname in tasknames: |
458 | taskid = taskData.gettask_id_fromfnid(depdata, taskname) | 456 | t = depdata + ":" + taskname |
459 | if taskid is not None: | 457 | if t in taskData.taskentries: |
460 | depends.add(taskid) | 458 | depends.add(t) |
461 | 459 | ||
462 | def add_runtime_dependencies(depids, tasknames, depends): | 460 | def add_runtime_dependencies(depids, tasknames, depends): |
463 | for depid in depids: | 461 | for depname in depids: |
464 | if depid not in taskData.run_targets: | 462 | if depname not in taskData.run_targets or not taskData.run_targets[depname]: |
465 | continue | 463 | continue |
466 | depdata = taskData.run_targets[depid][0] | 464 | depdata = taskData.run_targets[depname][0] |
467 | if depdata is None: | 465 | if depdata is None: |
468 | continue | 466 | continue |
469 | for taskname in tasknames: | 467 | for taskname in tasknames: |
470 | taskid = taskData.gettask_id_fromfnid(depdata, taskname) | 468 | t = depdata + ":" + taskname |
471 | if taskid is not None: | 469 | if t in taskData.taskentries: |
472 | depends.add(taskid) | 470 | depends.add(t) |
473 | 471 | ||
474 | def add_resolved_dependencies(depids, tasknames, depends): | 472 | def add_resolved_dependencies(fn, tasknames, depends): |
475 | for depid in depids: | 473 | for taskname in tasknames: |
476 | for taskname in tasknames: | 474 | tid = fn + ":" + taskname |
477 | taskid = taskData.gettask_id_fromfnid(depid, taskname) | 475 | if tid in self.runtaskentries: |
478 | if taskid is not None: | 476 | depends.add(tid) |
479 | depends.add(taskid) | 477 | |
478 | for tid in taskData.taskentries: | ||
479 | |||
480 | fn = fn_from_tid(tid) | ||
481 | taskname = taskname_from_tid(tid) | ||
480 | 482 | ||
481 | for task in range(len(taskData.tasks_name)): | ||
482 | depends = set() | 483 | depends = set() |
483 | fnid = taskData.tasks_fnid[task] | ||
484 | fn = taskData.fn_index[fnid] | ||
485 | task_deps = self.dataCache.task_deps[fn] | 484 | task_deps = self.dataCache.task_deps[fn] |
486 | 485 | ||
487 | #logger.debug(2, "Processing %s:%s", fn, taskData.tasks_name[task]) | 486 | self.runtaskentries[tid] = RunTaskEntry() |
488 | 487 | ||
489 | if fnid not in taskData.failed_fnids: | 488 | #logger.debug(2, "Processing %s:%s", fn, taskname) |
489 | |||
490 | if fn not in taskData.failed_fns: | ||
490 | 491 | ||
491 | # Resolve task internal dependencies | 492 | # Resolve task internal dependencies |
492 | # | 493 | # |
493 | # e.g. addtask before X after Y | 494 | # e.g. addtask before X after Y |
494 | depends = set(taskData.tasks_tdepends[task]) | 495 | depends.update(taskData.taskentries[tid].tdepends) |
495 | 496 | ||
496 | # Resolve 'deptask' dependencies | 497 | # Resolve 'deptask' dependencies |
497 | # | 498 | # |
498 | # e.g. do_sometask[deptask] = "do_someothertask" | 499 | # e.g. do_sometask[deptask] = "do_someothertask" |
499 | # (makes sure sometask runs after someothertask of all DEPENDS) | 500 | # (makes sure sometask runs after someothertask of all DEPENDS) |
500 | if 'deptask' in task_deps and taskData.tasks_name[task] in task_deps['deptask']: | 501 | if 'deptask' in task_deps and taskname in task_deps['deptask']: |
501 | tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() | 502 | tasknames = task_deps['deptask'][taskname].split() |
502 | add_build_dependencies(taskData.depids[fnid], tasknames, depends) | 503 | add_build_dependencies(taskData.depids[fn], tasknames, depends) |
503 | 504 | ||
504 | # Resolve 'rdeptask' dependencies | 505 | # Resolve 'rdeptask' dependencies |
505 | # | 506 | # |
506 | # e.g. do_sometask[rdeptask] = "do_someothertask" | 507 | # e.g. do_sometask[rdeptask] = "do_someothertask" |
507 | # (makes sure sometask runs after someothertask of all RDEPENDS) | 508 | # (makes sure sometask runs after someothertask of all RDEPENDS) |
508 | if 'rdeptask' in task_deps and taskData.tasks_name[task] in task_deps['rdeptask']: | 509 | if 'rdeptask' in task_deps and taskname in task_deps['rdeptask']: |
509 | tasknames = task_deps['rdeptask'][taskData.tasks_name[task]].split() | 510 | tasknames = task_deps['rdeptask'][taskname].split() |
510 | add_runtime_dependencies(taskData.rdepids[fnid], tasknames, depends) | 511 | add_runtime_dependencies(taskData.rdepids[fn], tasknames, depends) |
511 | 512 | ||
512 | # Resolve inter-task dependencies | 513 | # Resolve inter-task dependencies |
513 | # | 514 | # |
514 | # e.g. do_sometask[depends] = "targetname:do_someothertask" | 515 | # e.g. do_sometask[depends] = "targetname:do_someothertask" |
515 | # (makes sure sometask runs after targetname's someothertask) | 516 | # (makes sure sometask runs after targetname's someothertask) |
516 | idepends = taskData.tasks_idepends[task] | 517 | idepends = taskData.taskentries[tid].idepends |
517 | for (depid, idependtask) in idepends: | 518 | for (depname, idependtask) in idepends: |
518 | if depid in taskData.build_targets and not depid in taskData.failed_deps: | 519 | if depname in taskData.build_targets and taskData.build_targets[depname] and not depname in taskData.failed_deps: |
519 | # Won't be in build_targets if ASSUME_PROVIDED | 520 | # Won't be in build_targets if ASSUME_PROVIDED |
520 | depdata = taskData.build_targets[depid][0] | 521 | depdata = taskData.build_targets[depname][0] |
521 | if depdata is not None: | 522 | if depdata is not None: |
522 | taskid = taskData.gettask_id_fromfnid(depdata, idependtask) | 523 | t = depdata + ":" + idependtask |
523 | if taskid is None: | 524 | depends.add(t) |
524 | bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in %s" % (taskData.tasks_name[task], fn, idependtask, taskData.fn_index[depdata])) | 525 | if t not in taskData.taskentries: |
525 | depends.add(taskid) | 526 | bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in %s" % (taskname, fn, idependtask, depdata)) |
526 | irdepends = taskData.tasks_irdepends[task] | 527 | irdepends = taskData.taskentries[tid].irdepends |
527 | for (depid, idependtask) in irdepends: | 528 | for (depname, idependtask) in irdepends: |
528 | if depid in taskData.run_targets: | 529 | if depname in taskData.run_targets: |
529 | # Won't be in run_targets if ASSUME_PROVIDED | 530 | # Won't be in run_targets if ASSUME_PROVIDED |
530 | depdata = taskData.run_targets[depid][0] | 531 | depdata = taskData.run_targets[depname][0] |
531 | if depdata is not None: | 532 | if depdata is not None: |
532 | taskid = taskData.gettask_id_fromfnid(depdata, idependtask) | 533 | t = depdata + ":" + idependtask |
533 | if taskid is None: | 534 | depends.add(t) |
534 | bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s in %s" % (taskData.tasks_name[task], fn, idependtask, taskData.fn_index[depdata])) | 535 | if t not in taskData.taskentries: |
535 | depends.add(taskid) | 536 | bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s in %s" % (taskname, fn, idependtask, depdata)) |
536 | 537 | ||
537 | # Resolve recursive 'recrdeptask' dependencies (Part A) | 538 | # Resolve recursive 'recrdeptask' dependencies (Part A) |
538 | # | 539 | # |
539 | # e.g. do_sometask[recrdeptask] = "do_someothertask" | 540 | # e.g. do_sometask[recrdeptask] = "do_someothertask" |
540 | # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) | 541 | # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) |
541 | # We cover the recursive part of the dependencies below | 542 | # We cover the recursive part of the dependencies below |
542 | if 'recrdeptask' in task_deps and taskData.tasks_name[task] in task_deps['recrdeptask']: | 543 | if 'recrdeptask' in task_deps and taskname in task_deps['recrdeptask']: |
543 | tasknames = task_deps['recrdeptask'][taskData.tasks_name[task]].split() | 544 | tasknames = task_deps['recrdeptask'][taskname].split() |
544 | recursivetasks[task] = tasknames | 545 | recursivetasks[tid] = tasknames |
545 | add_build_dependencies(taskData.depids[fnid], tasknames, depends) | 546 | add_build_dependencies(taskData.depids[fn], tasknames, depends) |
546 | add_runtime_dependencies(taskData.rdepids[fnid], tasknames, depends) | 547 | add_runtime_dependencies(taskData.rdepids[fn], tasknames, depends) |
547 | if taskData.tasks_name[task] in tasknames: | 548 | if taskname in tasknames: |
548 | recursivetasksselfref.add(task) | 549 | recursivetasksselfref.add(tid) |
549 | 550 | ||
550 | if 'recideptask' in task_deps and taskData.tasks_name[task] in task_deps['recideptask']: | 551 | if 'recideptask' in task_deps and taskname in task_deps['recideptask']: |
551 | recursiveitasks[task] = [] | 552 | recursiveitasks[tid] = [] |
552 | for t in task_deps['recideptask'][taskData.tasks_name[task]].split(): | 553 | for t in task_deps['recideptask'][taskname].split(): |
553 | newdep = taskData.gettask_id_fromfnid(fnid, t) | 554 | newdep = fn + ":" + t |
554 | recursiveitasks[task].append(newdep) | 555 | recursiveitasks[tid].append(newdep) |
555 | 556 | ||
556 | self.runq_fnid.append(taskData.tasks_fnid[task]) | 557 | self.runtaskentries[tid].depends = depends |
557 | self.runq_task.append(taskData.tasks_name[task]) | ||
558 | self.runq_depends.append(depends) | ||
559 | self.runq_revdeps.append(set()) | ||
560 | self.runq_hash.append("") | ||
561 | |||
562 | runq_build.append(0) | ||
563 | 558 | ||
564 | # Resolve recursive 'recrdeptask' dependencies (Part B) | 559 | # Resolve recursive 'recrdeptask' dependencies (Part B) |
565 | # | 560 | # |
566 | # e.g. do_sometask[recrdeptask] = "do_someothertask" | 561 | # e.g. do_sometask[recrdeptask] = "do_someothertask" |
567 | # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) | 562 | # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) |
568 | # We need to do this separately since we need all of self.runq_depends to be complete before this is processed | 563 | # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed |
569 | extradeps = {} | 564 | extradeps = {} |
570 | for task in recursivetasks: | 565 | for tid in recursivetasks: |
571 | extradeps[task] = set(self.runq_depends[task]) | 566 | extradeps[tid] = set(self.runtaskentries[tid].depends) |
572 | tasknames = recursivetasks[task] | 567 | |
568 | tasknames = recursivetasks[tid] | ||
573 | seendeps = set() | 569 | seendeps = set() |
574 | seenfnid = [] | ||
575 | 570 | ||
576 | def generate_recdeps(t): | 571 | def generate_recdeps(t): |
577 | newdeps = set() | 572 | newdeps = set() |
578 | add_resolved_dependencies([taskData.tasks_fnid[t]], tasknames, newdeps) | 573 | add_resolved_dependencies(fn_from_tid(t), tasknames, newdeps) |
579 | extradeps[task].update(newdeps) | 574 | extradeps[tid].update(newdeps) |
580 | seendeps.add(t) | 575 | seendeps.add(t) |
581 | newdeps.add(t) | 576 | newdeps.add(t) |
582 | for i in newdeps: | 577 | for i in newdeps: |
583 | for n in self.runq_depends[i]: | 578 | task = self.runtaskentries[i].task |
579 | for n in self.runtaskentries[i].depends: | ||
584 | if n not in seendeps: | 580 | if n not in seendeps: |
585 | generate_recdeps(n) | 581 | generate_recdeps(n) |
586 | generate_recdeps(task) | 582 | generate_recdeps(tid) |
587 | 583 | ||
588 | if task in recursiveitasks: | 584 | if tid in recursiveitasks: |
589 | for dep in recursiveitasks[task]: | 585 | for dep in recursiveitasks[tid]: |
590 | generate_recdeps(dep) | 586 | generate_recdeps(dep) |
591 | 587 | ||
592 | # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work | 588 | # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work |
593 | for task in recursivetasks: | 589 | for tid in recursivetasks: |
594 | extradeps[task].difference_update(recursivetasksselfref) | 590 | extradeps[tid].difference_update(recursivetasksselfref) |
595 | 591 | ||
596 | for task in range(len(taskData.tasks_name)): | 592 | for tid in self.runtaskentries: |
593 | task = self.runtaskentries[tid].task | ||
597 | # Add in extra dependencies | 594 | # Add in extra dependencies |
598 | if task in extradeps: | 595 | if tid in extradeps: |
599 | self.runq_depends[task] = extradeps[task] | 596 | self.runtaskentries[tid].depends = extradeps[tid] |
600 | # Remove all self references | 597 | # Remove all self references |
601 | if task in self.runq_depends[task]: | 598 | if tid in self.runtaskentries[tid].depends: |
602 | logger.debug(2, "Task %s (%s %s) contains self reference! %s", task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], self.runq_depends[task]) | 599 | logger.debug(2, "Task %s contains self reference!", tid) |
603 | self.runq_depends[task].remove(task) | 600 | self.runtaskentries[tid].depends.remove(tid) |
604 | 601 | ||
605 | # Step B - Mark all active tasks | 602 | # Step B - Mark all active tasks |
606 | # | 603 | # |
@@ -610,33 +607,30 @@ class RunQueueData: | |||
610 | 607 | ||
611 | logger.verbose("Marking Active Tasks") | 608 | logger.verbose("Marking Active Tasks") |
612 | 609 | ||
613 | def mark_active(listid, depth): | 610 | def mark_active(tid, depth): |
614 | """ | 611 | """ |
615 | Mark an item as active along with its depends | 612 | Mark an item as active along with its depends |
616 | (calls itself recursively) | 613 | (calls itself recursively) |
617 | """ | 614 | """ |
618 | 615 | ||
619 | if runq_build[listid] == 1: | 616 | if tid in runq_build: |
620 | return | 617 | return |
621 | 618 | ||
622 | runq_build[listid] = 1 | 619 | runq_build[tid] = 1 |
623 | 620 | ||
624 | depends = self.runq_depends[listid] | 621 | depends = self.runtaskentries[tid].depends |
625 | for depend in depends: | 622 | for depend in depends: |
626 | mark_active(depend, depth+1) | 623 | mark_active(depend, depth+1) |
627 | 624 | ||
628 | self.target_pairs = [] | 625 | self.target_pairs = [] |
629 | for target in self.targets: | 626 | for target in self.targets: |
630 | targetid = taskData.getbuild_id(target[0]) | 627 | if target[0] not in taskData.build_targets or not taskData.build_targets[target[0]]: |
631 | |||
632 | if targetid not in taskData.build_targets: | ||
633 | continue | 628 | continue |
634 | 629 | ||
635 | if targetid in taskData.failed_deps: | 630 | if target[0] in taskData.failed_deps: |
636 | continue | 631 | continue |
637 | 632 | ||
638 | fnid = taskData.build_targets[targetid][0] | 633 | fn = taskData.build_targets[target[0]][0] |
639 | fn = taskData.fn_index[fnid] | ||
640 | task = target[1] | 634 | task = target[1] |
641 | parents = False | 635 | parents = False |
642 | if task.endswith('-'): | 636 | if task.endswith('-'): |
@@ -645,87 +639,72 @@ class RunQueueData: | |||
645 | 639 | ||
646 | self.target_pairs.append((fn, task)) | 640 | self.target_pairs.append((fn, task)) |
647 | 641 | ||
648 | if fnid in taskData.failed_fnids: | 642 | if fn in taskData.failed_fns: |
649 | continue | 643 | continue |
650 | 644 | ||
651 | if task not in taskData.tasks_lookup[fnid]: | 645 | tid = fn + ":" + task |
646 | if tid not in taskData.taskentries: | ||
652 | import difflib | 647 | import difflib |
653 | close_matches = difflib.get_close_matches(task, taskData.tasks_lookup[fnid], cutoff=0.7) | 648 | tasks = [] |
649 | for x in taskData.taskentries: | ||
650 | if x.startswith(fn + ":"): | ||
651 | tasks.append(taskname_from_tid(x)) | ||
652 | close_matches = difflib.get_close_matches(task, tasks, cutoff=0.7) | ||
654 | if close_matches: | 653 | if close_matches: |
655 | extra = ". Close matches:\n %s" % "\n ".join(close_matches) | 654 | extra = ". Close matches:\n %s" % "\n ".join(close_matches) |
656 | else: | 655 | else: |
657 | extra = "" | 656 | extra = "" |
658 | bb.msg.fatal("RunQueue", "Task %s does not exist for target %s%s" % (task, target[0], extra)) | 657 | bb.msg.fatal("RunQueue", "Task %s does not exist for target %s%s" % (task, target[0], extra)) |
659 | 658 | ||
660 | # For tasks called "XXXX-", ony run their dependencies | 659 | # For tasks called "XXXX-", ony run their dependencies |
661 | listid = taskData.tasks_lookup[fnid][task] | ||
662 | if parents: | 660 | if parents: |
663 | for i in self.runq_depends[listid]: | 661 | for i in self.runtaskentries[tid].depends: |
664 | mark_active(i, 1) | 662 | mark_active(i, 1) |
665 | else: | 663 | else: |
666 | mark_active(listid, 1) | 664 | mark_active(tid, 1) |
667 | 665 | ||
668 | # Step C - Prune all inactive tasks | 666 | # Step C - Prune all inactive tasks |
669 | # | 667 | # |
670 | # Once all active tasks are marked, prune the ones we don't need. | 668 | # Once all active tasks are marked, prune the ones we don't need. |
671 | 669 | ||
672 | maps = [] | ||
673 | delcount = 0 | 670 | delcount = 0 |
674 | for listid in range(len(self.runq_fnid)): | 671 | for tid in list(self.runtaskentries.keys()): |
675 | if runq_build[listid-delcount] == 1: | 672 | if tid not in runq_build: |
676 | maps.append(listid-delcount) | 673 | del self.runtaskentries[tid] |
677 | else: | 674 | delcount += 1 |
678 | del self.runq_fnid[listid-delcount] | ||
679 | del self.runq_task[listid-delcount] | ||
680 | del self.runq_depends[listid-delcount] | ||
681 | del runq_build[listid-delcount] | ||
682 | del self.runq_revdeps[listid-delcount] | ||
683 | del self.runq_hash[listid-delcount] | ||
684 | delcount = delcount + 1 | ||
685 | maps.append(-1) | ||
686 | 675 | ||
687 | # | 676 | # |
688 | # Step D - Sanity checks and computation | 677 | # Step D - Sanity checks and computation |
689 | # | 678 | # |
690 | 679 | ||
691 | # Check to make sure we still have tasks to run | 680 | # Check to make sure we still have tasks to run |
692 | if len(self.runq_fnid) == 0: | 681 | if len(self.runtaskentries) == 0: |
693 | if not taskData.abort: | 682 | if not taskData.abort: |
694 | bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.") | 683 | bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.") |
695 | else: | 684 | else: |
696 | bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.") | 685 | bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.") |
697 | 686 | ||
698 | logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runq_fnid)) | 687 | logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runtaskentries)) |
699 | |||
700 | # Remap the dependencies to account for the deleted tasks | ||
701 | # Check we didn't delete a task we depend on | ||
702 | for listid in range(len(self.runq_fnid)): | ||
703 | newdeps = [] | ||
704 | origdeps = self.runq_depends[listid] | ||
705 | for origdep in origdeps: | ||
706 | if maps[origdep] == -1: | ||
707 | bb.msg.fatal("RunQueue", "Invalid mapping - Should never happen!") | ||
708 | newdeps.append(maps[origdep]) | ||
709 | self.runq_depends[listid] = set(newdeps) | ||
710 | 688 | ||
711 | logger.verbose("Assign Weightings") | 689 | logger.verbose("Assign Weightings") |
712 | 690 | ||
713 | # Generate a list of reverse dependencies to ease future calculations | 691 | # Generate a list of reverse dependencies to ease future calculations |
714 | for listid in range(len(self.runq_fnid)): | 692 | for tid in self.runtaskentries: |
715 | for dep in self.runq_depends[listid]: | 693 | for dep in self.runtaskentries[tid].depends: |
716 | self.runq_revdeps[dep].add(listid) | 694 | self.runtaskentries[dep].revdeps.add(tid) |
717 | 695 | ||
718 | # Identify tasks at the end of dependency chains | 696 | # Identify tasks at the end of dependency chains |
719 | # Error on circular dependency loops (length two) | 697 | # Error on circular dependency loops (length two) |
720 | endpoints = [] | 698 | endpoints = [] |
721 | for listid in range(len(self.runq_fnid)): | 699 | for tid in self.runtaskentries: |
722 | revdeps = self.runq_revdeps[listid] | 700 | revdeps = self.runtaskentries[tid].revdeps |
723 | if len(revdeps) == 0: | 701 | if len(revdeps) == 0: |
724 | endpoints.append(listid) | 702 | endpoints.append(tid) |
725 | for dep in revdeps: | 703 | for dep in revdeps: |
726 | if dep in self.runq_depends[listid]: | 704 | if dep in self.runtaskentries[tid].depends: |
727 | #self.dump_data(taskData) | 705 | #self.dump_data(taskData) |
728 | bb.msg.fatal("RunQueue", "Task %s (%s) has circular dependency on %s (%s)" % (taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep], taskData.fn_index[self.runq_fnid[listid]], self.runq_task[listid])) | 706 | bb.msg.fatal("RunQueue", "Task %s has circular dependency on %s" % (tid, dep)) |
707 | |||
729 | 708 | ||
730 | logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints)) | 709 | logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints)) |
731 | 710 | ||
@@ -736,8 +715,8 @@ class RunQueueData: | |||
736 | # Sanity Check - Check for multiple tasks building the same provider | 715 | # Sanity Check - Check for multiple tasks building the same provider |
737 | prov_list = {} | 716 | prov_list = {} |
738 | seen_fn = [] | 717 | seen_fn = [] |
739 | for task in range(len(self.runq_fnid)): | 718 | for tid in self.runtaskentries: |
740 | fn = taskData.fn_index[self.runq_fnid[task]] | 719 | fn = fn_from_tid(tid) |
741 | if fn in seen_fn: | 720 | if fn in seen_fn: |
742 | continue | 721 | continue |
743 | seen_fn.append(fn) | 722 | seen_fn.append(fn) |
@@ -766,15 +745,15 @@ class RunQueueData: | |||
766 | commondeps = None | 745 | commondeps = None |
767 | for provfn in prov_list[prov]: | 746 | for provfn in prov_list[prov]: |
768 | deps = set() | 747 | deps = set() |
769 | for task, fnid in enumerate(self.runq_fnid): | 748 | for tid in self.runtaskentries: |
770 | fn = taskData.fn_index[fnid] | 749 | fn = fn_from_tid(tid) |
771 | if fn != provfn: | 750 | if fn != provfn: |
772 | continue | 751 | continue |
773 | for dep in self.runq_revdeps[task]: | 752 | for dep in self.runtaskentries[tid].revdeps: |
774 | fn = taskData.fn_index[self.runq_fnid[dep]] | 753 | fn = fn_from_tid(dep) |
775 | if fn == provfn: | 754 | if fn == provfn: |
776 | continue | 755 | continue |
777 | deps.add(self.get_short_user_idstring(dep)) | 756 | deps.add(dep) |
778 | if not commondeps: | 757 | if not commondeps: |
779 | commondeps = set(deps) | 758 | commondeps = set(deps) |
780 | else: | 759 | else: |
@@ -827,27 +806,26 @@ class RunQueueData: | |||
827 | # Create a whitelist usable by the stamp checks | 806 | # Create a whitelist usable by the stamp checks |
828 | stampfnwhitelist = [] | 807 | stampfnwhitelist = [] |
829 | for entry in self.stampwhitelist.split(): | 808 | for entry in self.stampwhitelist.split(): |
830 | entryid = self.taskData.getbuild_id(entry) | 809 | if entry not in self.taskData.build_targets: |
831 | if entryid not in self.taskData.build_targets: | ||
832 | continue | 810 | continue |
833 | fnid = self.taskData.build_targets[entryid][0] | 811 | fn = self.taskData.build_targets[entry][0] |
834 | fn = self.taskData.fn_index[fnid] | ||
835 | stampfnwhitelist.append(fn) | 812 | stampfnwhitelist.append(fn) |
836 | self.stampfnwhitelist = stampfnwhitelist | 813 | self.stampfnwhitelist = stampfnwhitelist |
837 | 814 | ||
838 | # Iterate over the task list looking for tasks with a 'setscene' function | 815 | # Iterate over the task list looking for tasks with a 'setscene' function |
839 | self.runq_setscene = [] | 816 | self.runq_setscene_tids = [] |
840 | if not self.cooker.configuration.nosetscene: | 817 | if not self.cooker.configuration.nosetscene: |
841 | for task in range(len(self.runq_fnid)): | 818 | for tid in self.runtaskentries: |
842 | setscene = taskData.gettask_id(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task] + "_setscene", False) | 819 | setscenetid = tid + "_setscene" |
843 | if not setscene: | 820 | if setscenetid not in taskData.taskentries: |
844 | continue | 821 | continue |
845 | self.runq_setscene.append(task) | 822 | task = self.runtaskentries[tid].task |
823 | self.runq_setscene_tids.append(tid) | ||
846 | 824 | ||
847 | def invalidate_task(fn, taskname, error_nostamp): | 825 | def invalidate_task(fn, taskname, error_nostamp): |
848 | taskdep = self.dataCache.task_deps[fn] | 826 | taskdep = self.dataCache.task_deps[fn] |
849 | fnid = self.taskData.getfn_id(fn) | 827 | tid = fn + ":" + taskname |
850 | if taskname not in taskData.tasks_lookup[fnid]: | 828 | if tid not in taskData.taskentries: |
851 | logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname) | 829 | logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname) |
852 | if 'nostamp' in taskdep and taskname in taskdep['nostamp']: | 830 | if 'nostamp' in taskdep and taskname in taskdep['nostamp']: |
853 | if error_nostamp: | 831 | if error_nostamp: |
@@ -882,43 +860,38 @@ class RunQueueData: | |||
882 | 860 | ||
883 | # Iterate over the task list and call into the siggen code | 861 | # Iterate over the task list and call into the siggen code |
884 | dealtwith = set() | 862 | dealtwith = set() |
885 | todeal = set(range(len(self.runq_fnid))) | 863 | todeal = set(self.runtaskentries) |
886 | while len(todeal) > 0: | 864 | while len(todeal) > 0: |
887 | for task in todeal.copy(): | 865 | for tid in todeal.copy(): |
888 | if len(self.runq_depends[task] - dealtwith) == 0: | 866 | if len(self.runtaskentries[tid].depends - dealtwith) == 0: |
889 | dealtwith.add(task) | 867 | dealtwith.add(tid) |
890 | todeal.remove(task) | 868 | todeal.remove(tid) |
891 | procdep = [] | 869 | procdep = [] |
892 | for dep in self.runq_depends[task]: | 870 | for dep in self.runtaskentries[tid].depends: |
893 | procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." + self.runq_task[dep]) | 871 | procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep)) |
894 | self.runq_hash[task] = bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep, self.dataCache) | 872 | self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(fn_from_tid(tid), taskname_from_tid(tid), procdep, self.dataCache) |
873 | task = self.runtaskentries[tid].task | ||
895 | 874 | ||
896 | bb.parse.siggen.writeout_file_checksum_cache() | 875 | bb.parse.siggen.writeout_file_checksum_cache() |
897 | return len(self.runq_fnid) | 876 | return len(self.runtaskentries) |
898 | 877 | ||
899 | def dump_data(self, taskQueue): | 878 | def dump_data(self, taskQueue): |
900 | """ | 879 | """ |
901 | Dump some debug information on the internal data structures | 880 | Dump some debug information on the internal data structures |
902 | """ | 881 | """ |
903 | logger.debug(3, "run_tasks:") | 882 | logger.debug(3, "run_tasks:") |
904 | for task in range(len(self.rqdata.runq_task)): | 883 | for tid in self.runtaskentries: |
905 | logger.debug(3, " (%s)%s - %s: %s Deps %s RevDeps %s", task, | 884 | logger.debug(3, " %s: %s Deps %s RevDeps %s", tid, |
906 | taskQueue.fn_index[self.rqdata.runq_fnid[task]], | 885 | self.runtaskentries[tid].weight, |
907 | self.rqdata.runq_task[task], | 886 | self.runtaskentries[tid].depends, |
908 | self.rqdata.runq_weight[task], | 887 | self.runtaskentries[tid].revdeps) |
909 | self.rqdata.runq_depends[task], | ||
910 | self.rqdata.runq_revdeps[task]) | ||
911 | 888 | ||
912 | logger.debug(3, "sorted_tasks:") | 889 | logger.debug(3, "sorted_tasks:") |
913 | for task1 in range(len(self.rqdata.runq_task)): | 890 | for tid in self.prio_map: |
914 | if task1 in self.prio_map: | 891 | logger.debug(3, " %s: %s Deps %s RevDeps %s", tid, |
915 | task = self.prio_map[task1] | 892 | self.runtaskentries[tid].weight, |
916 | logger.debug(3, " (%s)%s - %s: %s Deps %s RevDeps %s", task, | 893 | self.runtaskentries[tid].depends, |
917 | taskQueue.fn_index[self.rqdata.runq_fnid[task]], | 894 | self.runtaskentries[tid].revdeps) |
918 | self.rqdata.runq_task[task], | ||
919 | self.rqdata.runq_weight[task], | ||
920 | self.rqdata.runq_depends[task], | ||
921 | self.rqdata.runq_revdeps[task]) | ||
922 | 895 | ||
923 | class RunQueue: | 896 | class RunQueue: |
924 | def __init__(self, cooker, cfgData, dataCache, taskData, targets): | 897 | def __init__(self, cooker, cfgData, dataCache, taskData, targets): |
@@ -929,7 +902,7 @@ class RunQueue: | |||
929 | 902 | ||
930 | self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile" | 903 | self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile" |
931 | self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None | 904 | self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None |
932 | self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION", True) or None | 905 | self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION2", True) or None |
933 | self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None | 906 | self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None |
934 | 907 | ||
935 | self.state = runQueuePrepare | 908 | self.state = runQueuePrepare |
@@ -961,13 +934,17 @@ class RunQueue: | |||
961 | bb.utils.nonblockingfd(worker.stdout) | 934 | bb.utils.nonblockingfd(worker.stdout) |
962 | workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec) | 935 | workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec) |
963 | 936 | ||
937 | runqhash = {} | ||
938 | for tid in self.rqdata.runtaskentries: | ||
939 | runqhash[tid] = self.rqdata.runtaskentries[tid].hash | ||
940 | |||
964 | workerdata = { | 941 | workerdata = { |
965 | "taskdeps" : self.rqdata.dataCache.task_deps, | 942 | "taskdeps" : self.rqdata.dataCache.task_deps, |
966 | "fakerootenv" : self.rqdata.dataCache.fakerootenv, | 943 | "fakerootenv" : self.rqdata.dataCache.fakerootenv, |
967 | "fakerootdirs" : self.rqdata.dataCache.fakerootdirs, | 944 | "fakerootdirs" : self.rqdata.dataCache.fakerootdirs, |
968 | "fakerootnoenv" : self.rqdata.dataCache.fakerootnoenv, | 945 | "fakerootnoenv" : self.rqdata.dataCache.fakerootnoenv, |
969 | "sigdata" : bb.parse.siggen.get_taskdata(), | 946 | "sigdata" : bb.parse.siggen.get_taskdata(), |
970 | "runq_hash" : self.rqdata.runq_hash, | 947 | "runq_hash" : runqhash, |
971 | "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel, | 948 | "logdefaultdebug" : bb.msg.loggerDefaultDebugLevel, |
972 | "logdefaultverbose" : bb.msg.loggerDefaultVerbose, | 949 | "logdefaultverbose" : bb.msg.loggerDefaultVerbose, |
973 | "logdefaultverboselogs" : bb.msg.loggerVerboseLogs, | 950 | "logdefaultverboselogs" : bb.msg.loggerVerboseLogs, |
@@ -1033,7 +1010,7 @@ class RunQueue: | |||
1033 | fds.append(self.fakeworkerpipe.input) | 1010 | fds.append(self.fakeworkerpipe.input) |
1034 | return fds | 1011 | return fds |
1035 | 1012 | ||
1036 | def check_stamp_task(self, task, taskname = None, recurse = False, cache = None): | 1013 | def check_stamp_task(self, tid, taskname = None, recurse = False, cache = None): |
1037 | def get_timestamp(f): | 1014 | def get_timestamp(f): |
1038 | try: | 1015 | try: |
1039 | if not os.access(f, os.F_OK): | 1016 | if not os.access(f, os.F_OK): |
@@ -1050,9 +1027,9 @@ class RunQueue: | |||
1050 | if self.stamppolicy == "whitelist": | 1027 | if self.stamppolicy == "whitelist": |
1051 | stampwhitelist = self.rqdata.stampfnwhitelist | 1028 | stampwhitelist = self.rqdata.stampfnwhitelist |
1052 | 1029 | ||
1053 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] | 1030 | fn = fn_from_tid(tid) |
1054 | if taskname is None: | 1031 | if taskname is None: |
1055 | taskname = self.rqdata.runq_task[task] | 1032 | taskname = taskname_from_tid(tid) |
1056 | 1033 | ||
1057 | stampfile = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) | 1034 | stampfile = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) |
1058 | 1035 | ||
@@ -1074,10 +1051,10 @@ class RunQueue: | |||
1074 | 1051 | ||
1075 | iscurrent = True | 1052 | iscurrent = True |
1076 | t1 = get_timestamp(stampfile) | 1053 | t1 = get_timestamp(stampfile) |
1077 | for dep in self.rqdata.runq_depends[task]: | 1054 | for dep in self.rqdata.runtaskentries[tid].depends: |
1078 | if iscurrent: | 1055 | if iscurrent: |
1079 | fn2 = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[dep]] | 1056 | fn2 = fn_from_tid(dep) |
1080 | taskname2 = self.rqdata.runq_task[dep] | 1057 | taskname2 = taskname_from_tid(dep) |
1081 | stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCache, fn2) | 1058 | stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCache, fn2) |
1082 | stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCache, fn2) | 1059 | stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCache, fn2) |
1083 | t2 = get_timestamp(stampfile2) | 1060 | t2 = get_timestamp(stampfile2) |
@@ -1104,7 +1081,7 @@ class RunQueue: | |||
1104 | iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache) | 1081 | iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache) |
1105 | cache[dep] = iscurrent | 1082 | cache[dep] = iscurrent |
1106 | if recurse: | 1083 | if recurse: |
1107 | cache[task] = iscurrent | 1084 | cache[tid] = iscurrent |
1108 | return iscurrent | 1085 | return iscurrent |
1109 | 1086 | ||
1110 | def _execute_runqueue(self): | 1087 | def _execute_runqueue(self): |
@@ -1171,9 +1148,9 @@ class RunQueue: | |||
1171 | 1148 | ||
1172 | if self.state is runQueueFailed: | 1149 | if self.state is runQueueFailed: |
1173 | if not self.rqdata.taskData.tryaltconfigs: | 1150 | if not self.rqdata.taskData.tryaltconfigs: |
1174 | raise bb.runqueue.TaskFailure(self.rqexe.failed_fnids) | 1151 | raise bb.runqueue.TaskFailure(self.rqexe.failed_fns) |
1175 | for fnid in self.rqexe.failed_fnids: | 1152 | for fn in self.rqexe.failed_fns: |
1176 | self.rqdata.taskData.fail_fnid(fnid) | 1153 | self.rqdata.taskData.fail_fn(fn) |
1177 | self.rqdata.reset() | 1154 | self.rqdata.reset() |
1178 | 1155 | ||
1179 | if self.state is runQueueComplete: | 1156 | if self.state is runQueueComplete: |
@@ -1220,11 +1197,11 @@ class RunQueue: | |||
1220 | def dump_signatures(self, options): | 1197 | def dump_signatures(self, options): |
1221 | done = set() | 1198 | done = set() |
1222 | bb.note("Reparsing files to collect dependency data") | 1199 | bb.note("Reparsing files to collect dependency data") |
1223 | for task in range(len(self.rqdata.runq_fnid)): | 1200 | for tid in self.rqdata.runtaskentries: |
1224 | if self.rqdata.runq_fnid[task] not in done: | 1201 | fn = fn_from_tid(tid) |
1225 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] | 1202 | if fn not in done: |
1226 | the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn), self.cooker.data) | 1203 | the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn), self.cooker.data) |
1227 | done.add(self.rqdata.runq_fnid[task]) | 1204 | done.add(fn) |
1228 | 1205 | ||
1229 | bb.parse.siggen.dump_sigs(self.rqdata.dataCache, options) | 1206 | bb.parse.siggen.dump_sigs(self.rqdata.dataCache, options) |
1230 | 1207 | ||
@@ -1242,20 +1219,20 @@ class RunQueue: | |||
1242 | stamppresent = [] | 1219 | stamppresent = [] |
1243 | valid_new = set() | 1220 | valid_new = set() |
1244 | 1221 | ||
1245 | for task in range(len(self.rqdata.runq_fnid)): | 1222 | for tid in self.rqdata.runtaskentries: |
1246 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] | 1223 | fn = fn_from_tid(tid) |
1247 | taskname = self.rqdata.runq_task[task] | 1224 | taskname = taskname_from_tid(tid) |
1248 | taskdep = self.rqdata.dataCache.task_deps[fn] | 1225 | taskdep = self.rqdata.dataCache.task_deps[fn] |
1249 | 1226 | ||
1250 | if 'noexec' in taskdep and taskname in taskdep['noexec']: | 1227 | if 'noexec' in taskdep and taskname in taskdep['noexec']: |
1251 | noexec.append(task) | 1228 | noexec.append(tid) |
1252 | continue | 1229 | continue |
1253 | 1230 | ||
1254 | sq_fn.append(fn) | 1231 | sq_fn.append(fn) |
1255 | sq_hashfn.append(self.rqdata.dataCache.hashfn[fn]) | 1232 | sq_hashfn.append(self.rqdata.dataCache.hashfn[fn]) |
1256 | sq_hash.append(self.rqdata.runq_hash[task]) | 1233 | sq_hash.append(self.rqdata.runtaskentries[tid].hash) |
1257 | sq_taskname.append(taskname) | 1234 | sq_taskname.append(taskname) |
1258 | sq_task.append(task) | 1235 | sq_task.append(tid) |
1259 | locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data } | 1236 | locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data } |
1260 | try: | 1237 | try: |
1261 | call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)" | 1238 | call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)" |
@@ -1270,13 +1247,13 @@ class RunQueue: | |||
1270 | # Tasks which are both setscene and noexec never care about dependencies | 1247 | # Tasks which are both setscene and noexec never care about dependencies |
1271 | # We therefore find tasks which are setscene and noexec and mark their | 1248 | # We therefore find tasks which are setscene and noexec and mark their |
1272 | # unique dependencies as valid. | 1249 | # unique dependencies as valid. |
1273 | for task in noexec: | 1250 | for tid in noexec: |
1274 | if task not in self.rqdata.runq_setscene: | 1251 | if tid not in self.rqdata.runq_setscene_tids: |
1275 | continue | 1252 | continue |
1276 | for dep in self.rqdata.runq_depends[task]: | 1253 | for dep in self.rqdata.runtaskentries[tid].depends: |
1277 | hasnoexecparents = True | 1254 | hasnoexecparents = True |
1278 | for dep2 in self.rqdata.runq_revdeps[dep]: | 1255 | for dep2 in self.rqdata.runtaskentries[dep].revdeps: |
1279 | if dep2 in self.rqdata.runq_setscene and dep2 in noexec: | 1256 | if dep2 in self.rqdata.runq_setscene_tids and dep2 in noexec: |
1280 | continue | 1257 | continue |
1281 | hasnoexecparents = False | 1258 | hasnoexecparents = False |
1282 | break | 1259 | break |
@@ -1284,30 +1261,30 @@ class RunQueue: | |||
1284 | valid_new.add(dep) | 1261 | valid_new.add(dep) |
1285 | 1262 | ||
1286 | invalidtasks = set() | 1263 | invalidtasks = set() |
1287 | for task in range(len(self.rqdata.runq_fnid)): | 1264 | for tid in self.rqdata.runtaskentries: |
1288 | if task not in valid_new and task not in noexec: | 1265 | if tid not in valid_new and tid not in noexec: |
1289 | invalidtasks.add(task) | 1266 | invalidtasks.add(tid) |
1290 | 1267 | ||
1291 | found = set() | 1268 | found = set() |
1292 | processed = set() | 1269 | processed = set() |
1293 | for task in invalidtasks: | 1270 | for tid in invalidtasks: |
1294 | toprocess = set([task]) | 1271 | toprocess = set([tid]) |
1295 | while toprocess: | 1272 | while toprocess: |
1296 | next = set() | 1273 | next = set() |
1297 | for t in toprocess: | 1274 | for t in toprocess: |
1298 | for dep in self.rqdata.runq_depends[t]: | 1275 | for dep in self.rqdata.runtaskentries[t].depends: |
1299 | if dep in invalidtasks: | 1276 | if dep in invalidtasks: |
1300 | found.add(task) | 1277 | found.add(tid) |
1301 | if dep not in processed: | 1278 | if dep not in processed: |
1302 | processed.add(dep) | 1279 | processed.add(dep) |
1303 | next.add(dep) | 1280 | next.add(dep) |
1304 | toprocess = next | 1281 | toprocess = next |
1305 | if task in found: | 1282 | if tid in found: |
1306 | toprocess = set() | 1283 | toprocess = set() |
1307 | 1284 | ||
1308 | tasklist = [] | 1285 | tasklist = [] |
1309 | for task in invalidtasks.difference(found): | 1286 | for tid in invalidtasks.difference(found): |
1310 | tasklist.append(self.rqdata.get_user_idstring(task)) | 1287 | tasklist.append(tid) |
1311 | 1288 | ||
1312 | if tasklist: | 1289 | if tasklist: |
1313 | bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist)) | 1290 | bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist)) |
@@ -1331,11 +1308,11 @@ class RunQueue: | |||
1331 | return recout | 1308 | return recout |
1332 | 1309 | ||
1333 | 1310 | ||
1334 | for task in invalidtasks: | 1311 | for tid in invalidtasks: |
1335 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] | 1312 | fn = fn_from_tid(tid) |
1336 | pn = self.rqdata.dataCache.pkg_fn[fn] | 1313 | pn = self.rqdata.dataCache.pkg_fn[fn] |
1337 | taskname = self.rqdata.runq_task[task] | 1314 | taskname = taskname_from_tid(tid) |
1338 | h = self.rqdata.runq_hash[task] | 1315 | h = self.rqdata.runtaskentries[tid].hash |
1339 | matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData) | 1316 | matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData) |
1340 | match = None | 1317 | match = None |
1341 | for m in matches: | 1318 | for m in matches: |
@@ -1367,7 +1344,7 @@ class RunQueueExecute: | |||
1367 | 1344 | ||
1368 | self.build_stamps = {} | 1345 | self.build_stamps = {} |
1369 | self.build_stamps2 = [] | 1346 | self.build_stamps2 = [] |
1370 | self.failed_fnids = [] | 1347 | self.failed_fns = [] |
1371 | 1348 | ||
1372 | self.stampcache = {} | 1349 | self.stampcache = {} |
1373 | 1350 | ||
@@ -1401,7 +1378,7 @@ class RunQueueExecute: | |||
1401 | except IOError: | 1378 | except IOError: |
1402 | # worker must have died? | 1379 | # worker must have died? |
1403 | pass | 1380 | pass |
1404 | if len(self.failed_fnids) != 0: | 1381 | if len(self.failed_fns) != 0: |
1405 | self.rq.state = runQueueFailed | 1382 | self.rq.state = runQueueFailed |
1406 | return | 1383 | return |
1407 | 1384 | ||
@@ -1416,7 +1393,7 @@ class RunQueueExecute: | |||
1416 | self.rq.read_workers() | 1393 | self.rq.read_workers() |
1417 | return self.rq.active_fds() | 1394 | return self.rq.active_fds() |
1418 | 1395 | ||
1419 | if len(self.failed_fnids) != 0: | 1396 | if len(self.failed_fns) != 0: |
1420 | self.rq.state = runQueueFailed | 1397 | self.rq.state = runQueueFailed |
1421 | return True | 1398 | return True |
1422 | 1399 | ||
@@ -1430,13 +1407,9 @@ class RunQueueExecute: | |||
1430 | taskdata = {} | 1407 | taskdata = {} |
1431 | taskdeps.add(task) | 1408 | taskdeps.add(task) |
1432 | for dep in taskdeps: | 1409 | for dep in taskdeps: |
1433 | if setscene: | 1410 | fn = fn_from_tid(dep) |
1434 | depid = self.rqdata.runq_setscene[dep] | ||
1435 | else: | ||
1436 | depid = dep | ||
1437 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[depid]] | ||
1438 | pn = self.rqdata.dataCache.pkg_fn[fn] | 1411 | pn = self.rqdata.dataCache.pkg_fn[fn] |
1439 | taskname = self.rqdata.runq_task[depid] | 1412 | taskname = taskname_from_tid(dep) |
1440 | taskdata[dep] = [pn, taskname, fn] | 1413 | taskdata[dep] = [pn, taskname, fn] |
1441 | call = self.rq.depvalidate + "(task, taskdata, notneeded, d)" | 1414 | call = self.rq.depvalidate + "(task, taskdata, notneeded, d)" |
1442 | locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data } | 1415 | locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.expanded_data } |
@@ -1456,30 +1429,30 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1456 | def __init__(self, rq): | 1429 | def __init__(self, rq): |
1457 | RunQueueExecute.__init__(self, rq) | 1430 | RunQueueExecute.__init__(self, rq) |
1458 | 1431 | ||
1459 | self.stats = RunQueueStats(len(self.rqdata.runq_fnid)) | 1432 | self.stats = RunQueueStats(len(self.rqdata.runtaskentries)) |
1460 | 1433 | ||
1461 | self.stampcache = {} | 1434 | self.stampcache = {} |
1462 | 1435 | ||
1463 | initial_covered = self.rq.scenequeue_covered.copy() | 1436 | initial_covered = self.rq.scenequeue_covered.copy() |
1464 | 1437 | ||
1465 | # Mark initial buildable tasks | 1438 | # Mark initial buildable tasks |
1466 | for task in range(self.stats.total): | 1439 | for tid in self.rqdata.runtaskentries: |
1467 | if len(self.rqdata.runq_depends[task]) == 0: | 1440 | if len(self.rqdata.runtaskentries[tid].depends) == 0: |
1468 | self.runq_buildable.add(task) | 1441 | self.runq_buildable.add(tid) |
1469 | if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered): | 1442 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): |
1470 | self.rq.scenequeue_covered.add(task) | 1443 | self.rq.scenequeue_covered.add(tid) |
1471 | 1444 | ||
1472 | found = True | 1445 | found = True |
1473 | while found: | 1446 | while found: |
1474 | found = False | 1447 | found = False |
1475 | for task in range(self.stats.total): | 1448 | for tid in self.rqdata.runtaskentries: |
1476 | if task in self.rq.scenequeue_covered: | 1449 | if tid in self.rq.scenequeue_covered: |
1477 | continue | 1450 | continue |
1478 | logger.debug(1, 'Considering %s (%s): %s' % (task, self.rqdata.get_user_idstring(task), str(self.rqdata.runq_revdeps[task]))) | 1451 | logger.debug(1, 'Considering %s: %s' % (tid, str(self.rqdata.runtaskentries[tid].revdeps))) |
1479 | 1452 | ||
1480 | if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered): | 1453 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): |
1481 | found = True | 1454 | found = True |
1482 | self.rq.scenequeue_covered.add(task) | 1455 | self.rq.scenequeue_covered.add(tid) |
1483 | 1456 | ||
1484 | logger.debug(1, 'Skip list (pre setsceneverify) %s', sorted(self.rq.scenequeue_covered)) | 1457 | logger.debug(1, 'Skip list (pre setsceneverify) %s', sorted(self.rq.scenequeue_covered)) |
1485 | 1458 | ||
@@ -1487,35 +1460,33 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1487 | covered_remove = set() | 1460 | covered_remove = set() |
1488 | if self.rq.setsceneverify: | 1461 | if self.rq.setsceneverify: |
1489 | invalidtasks = [] | 1462 | invalidtasks = [] |
1490 | for task in range(len(self.rqdata.runq_task)): | 1463 | tasknames = {} |
1491 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] | 1464 | fns = {} |
1492 | taskname = self.rqdata.runq_task[task] | 1465 | for tid in self.rqdata.runtaskentries: |
1466 | fn = fn_from_tid(tid) | ||
1467 | taskname = taskname_from_tid(tid) | ||
1493 | taskdep = self.rqdata.dataCache.task_deps[fn] | 1468 | taskdep = self.rqdata.dataCache.task_deps[fn] |
1494 | 1469 | fns[tid] = fn | |
1470 | tasknames[tid] = taskname | ||
1495 | if 'noexec' in taskdep and taskname in taskdep['noexec']: | 1471 | if 'noexec' in taskdep and taskname in taskdep['noexec']: |
1496 | continue | 1472 | continue |
1497 | if self.rq.check_stamp_task(task, taskname + "_setscene", cache=self.stampcache): | 1473 | if self.rq.check_stamp_task(tid, taskname + "_setscene", cache=self.stampcache): |
1498 | logger.debug(2, 'Setscene stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(task)) | 1474 | logger.debug(2, 'Setscene stamp current for task %s', tid) |
1499 | continue | 1475 | continue |
1500 | if self.rq.check_stamp_task(task, taskname, recurse = True, cache=self.stampcache): | 1476 | if self.rq.check_stamp_task(tid, taskname, recurse = True, cache=self.stampcache): |
1501 | logger.debug(2, 'Normal stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(task)) | 1477 | logger.debug(2, 'Normal stamp current for task %s', tid) |
1502 | continue | 1478 | continue |
1503 | invalidtasks.append(task) | 1479 | invalidtasks.append(tid) |
1504 | 1480 | ||
1505 | call = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d, invalidtasks=invalidtasks)" | 1481 | call = self.rq.setsceneverify + "(covered, tasknames, fns, d, invalidtasks=invalidtasks)" |
1506 | call2 = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d)" | 1482 | locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : tasknames, "fns" : fns, "d" : self.cooker.expanded_data, "invalidtasks" : invalidtasks } |
1507 | locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : self.rqdata.runq_task, "fnids" : self.rqdata.runq_fnid, "fns" : self.rqdata.taskData.fn_index, "d" : self.cooker.expanded_data, "invalidtasks" : invalidtasks } | 1483 | covered_remove = bb.utils.better_eval(call, locs) |
1508 | # Backwards compatibility with older versions without invalidtasks | ||
1509 | try: | ||
1510 | covered_remove = bb.utils.better_eval(call, locs) | ||
1511 | except TypeError: | ||
1512 | covered_remove = bb.utils.better_eval(call2, locs) | ||
1513 | 1484 | ||
1514 | def removecoveredtask(task): | 1485 | def removecoveredtask(tid): |
1515 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] | 1486 | fn = fn_from_tid(tid) |
1516 | taskname = self.rqdata.runq_task[task] + '_setscene' | 1487 | taskname = taskname_from_tid(tid) + '_setscene' |
1517 | bb.build.del_stamp(taskname, self.rqdata.dataCache, fn) | 1488 | bb.build.del_stamp(taskname, self.rqdata.dataCache, fn) |
1518 | self.rq.scenequeue_covered.remove(task) | 1489 | self.rq.scenequeue_covered.remove(tid) |
1519 | 1490 | ||
1520 | toremove = covered_remove | 1491 | toremove = covered_remove |
1521 | for task in toremove: | 1492 | for task in toremove: |
@@ -1524,7 +1495,7 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1524 | covered_remove = [] | 1495 | covered_remove = [] |
1525 | for task in toremove: | 1496 | for task in toremove: |
1526 | removecoveredtask(task) | 1497 | removecoveredtask(task) |
1527 | for deptask in self.rqdata.runq_depends[task]: | 1498 | for deptask in self.rqdata.runtaskentries[task].depends: |
1528 | if deptask not in self.rq.scenequeue_covered: | 1499 | if deptask not in self.rq.scenequeue_covered: |
1529 | continue | 1500 | continue |
1530 | if deptask in toremove or deptask in covered_remove or deptask in initial_covered: | 1501 | if deptask in toremove or deptask in covered_remove or deptask in initial_covered: |
@@ -1580,20 +1551,20 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1580 | completed dependencies as buildable | 1551 | completed dependencies as buildable |
1581 | """ | 1552 | """ |
1582 | self.runq_complete.add(task) | 1553 | self.runq_complete.add(task) |
1583 | for revdep in self.rqdata.runq_revdeps[task]: | 1554 | for revdep in self.rqdata.runtaskentries[task].revdeps: |
1584 | if revdep in self.runq_running: | 1555 | if revdep in self.runq_running: |
1585 | continue | 1556 | continue |
1586 | if revdep in self.runq_buildable: | 1557 | if revdep in self.runq_buildable: |
1587 | continue | 1558 | continue |
1588 | alldeps = 1 | 1559 | alldeps = 1 |
1589 | for dep in self.rqdata.runq_depends[revdep]: | 1560 | for dep in self.rqdata.runtaskentries[revdep].depends: |
1590 | if dep not in self.runq_complete: | 1561 | if dep not in self.runq_complete: |
1591 | alldeps = 0 | 1562 | alldeps = 0 |
1592 | if alldeps == 1: | 1563 | if alldeps == 1: |
1593 | self.setbuildable(revdep) | 1564 | self.setbuildable(revdep) |
1594 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]] | 1565 | fn = fn_from_tid(revdep) |
1595 | taskname = self.rqdata.runq_task[revdep] | 1566 | taskname = taskname_from_tid(revdep) |
1596 | logger.debug(1, "Marking task %s (%s, %s) as buildable", revdep, fn, taskname) | 1567 | logger.debug(1, "Marking task %s as buildable", revdep) |
1597 | 1568 | ||
1598 | def task_complete(self, task): | 1569 | def task_complete(self, task): |
1599 | self.stats.taskCompleted() | 1570 | self.stats.taskCompleted() |
@@ -1606,8 +1577,8 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1606 | Updates the state engine with the failure | 1577 | Updates the state engine with the failure |
1607 | """ | 1578 | """ |
1608 | self.stats.taskFailed() | 1579 | self.stats.taskFailed() |
1609 | fnid = self.rqdata.runq_fnid[task] | 1580 | fn = fn_from_tid(task) |
1610 | self.failed_fnids.append(fnid) | 1581 | self.failed_fns.append(fn) |
1611 | bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData) | 1582 | bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData) |
1612 | if self.rqdata.taskData.abort: | 1583 | if self.rqdata.taskData.abort: |
1613 | self.rq.state = runQueueCleanUp | 1584 | self.rq.state = runQueueCleanUp |
@@ -1626,7 +1597,7 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1626 | """ | 1597 | """ |
1627 | 1598 | ||
1628 | self.rq.read_workers() | 1599 | self.rq.read_workers() |
1629 | 1600 | ||
1630 | 1601 | ||
1631 | if self.stats.total == 0: | 1602 | if self.stats.total == 0: |
1632 | # nothing to do | 1603 | # nothing to do |
@@ -1634,18 +1605,17 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1634 | 1605 | ||
1635 | task = self.sched.next() | 1606 | task = self.sched.next() |
1636 | if task is not None: | 1607 | if task is not None: |
1637 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]] | 1608 | fn = fn_from_tid(task) |
1638 | taskname = self.rqdata.runq_task[task] | 1609 | taskname = taskname_from_tid(task) |
1639 | 1610 | ||
1640 | if task in self.rq.scenequeue_covered: | 1611 | if task in self.rq.scenequeue_covered: |
1641 | logger.debug(2, "Setscene covered task %s (%s)", task, | 1612 | logger.debug(2, "Setscene covered task %s", task) |
1642 | self.rqdata.get_user_idstring(task)) | ||
1643 | self.task_skip(task, "covered") | 1613 | self.task_skip(task, "covered") |
1644 | return True | 1614 | return True |
1645 | 1615 | ||
1646 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): | 1616 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): |
1647 | logger.debug(2, "Stamp current task %s (%s)", task, | 1617 | logger.debug(2, "Stamp current task %s", task) |
1648 | self.rqdata.get_user_idstring(task)) | 1618 | |
1649 | self.task_skip(task, "existing") | 1619 | self.task_skip(task, "existing") |
1650 | return True | 1620 | return True |
1651 | 1621 | ||
@@ -1672,7 +1642,7 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1672 | try: | 1642 | try: |
1673 | self.rq.start_fakeworker(self) | 1643 | self.rq.start_fakeworker(self) |
1674 | except OSError as exc: | 1644 | except OSError as exc: |
1675 | logger.critical("Failed to spawn fakeroot worker to run %s:%s: %s" % (fn, taskname, str(exc))) | 1645 | logger.critical("Failed to spawn fakeroot worker to run %s: %s" % (task, str(exc))) |
1676 | self.rq.state = runQueueFailed | 1646 | self.rq.state = runQueueFailed |
1677 | return True | 1647 | return True |
1678 | self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + b"</runtask>") | 1648 | self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, False, self.cooker.collection.get_file_appends(fn), taskdepdata)) + b"</runtask>") |
@@ -1682,7 +1652,7 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1682 | self.rq.worker.stdin.flush() | 1652 | self.rq.worker.stdin.flush() |
1683 | 1653 | ||
1684 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) | 1654 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) |
1685 | self.build_stamps2.append(self.build_stamps[task]) | 1655 | self.build_stamps2.append(self.build_stamps[task]) |
1686 | self.runq_running.add(task) | 1656 | self.runq_running.add(task) |
1687 | self.stats.taskActive() | 1657 | self.stats.taskActive() |
1688 | if self.stats.active < self.number_tasks: | 1658 | if self.stats.active < self.number_tasks: |
@@ -1692,12 +1662,12 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1692 | self.rq.read_workers() | 1662 | self.rq.read_workers() |
1693 | return self.rq.active_fds() | 1663 | return self.rq.active_fds() |
1694 | 1664 | ||
1695 | if len(self.failed_fnids) != 0: | 1665 | if len(self.failed_fns) != 0: |
1696 | self.rq.state = runQueueFailed | 1666 | self.rq.state = runQueueFailed |
1697 | return True | 1667 | return True |
1698 | 1668 | ||
1699 | # Sanity Checks | 1669 | # Sanity Checks |
1700 | for task in range(self.stats.total): | 1670 | for task in self.rqdata.runtaskentries: |
1701 | if task not in self.runq_buildable: | 1671 | if task not in self.runq_buildable: |
1702 | logger.error("Task %s never buildable!", task) | 1672 | logger.error("Task %s never buildable!", task) |
1703 | if task not in self.runq_running: | 1673 | if task not in self.runq_running: |
@@ -1710,15 +1680,15 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
1710 | 1680 | ||
1711 | def build_taskdepdata(self, task): | 1681 | def build_taskdepdata(self, task): |
1712 | taskdepdata = {} | 1682 | taskdepdata = {} |
1713 | next = self.rqdata.runq_depends[task] | 1683 | next = self.rqdata.runtaskentries[task].depends |
1714 | next.add(task) | 1684 | next.add(task) |
1715 | while next: | 1685 | while next: |
1716 | additional = [] | 1686 | additional = [] |
1717 | for revdep in next: | 1687 | for revdep in next: |
1718 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]] | 1688 | fn = fn_from_tid(revdep) |
1719 | pn = self.rqdata.dataCache.pkg_fn[fn] | 1689 | pn = self.rqdata.dataCache.pkg_fn[fn] |
1720 | taskname = self.rqdata.runq_task[revdep] | 1690 | taskname = taskname_from_tid(revdep) |
1721 | deps = self.rqdata.runq_depends[revdep] | 1691 | deps = self.rqdata.runtaskentries[revdep].depends |
1722 | provides = self.rqdata.dataCache.fn_provides[fn] | 1692 | provides = self.rqdata.dataCache.fn_provides[fn] |
1723 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides] | 1693 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides] |
1724 | for revdep2 in deps: | 1694 | for revdep2 in deps: |
@@ -1738,16 +1708,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1738 | self.scenequeue_notneeded = set() | 1708 | self.scenequeue_notneeded = set() |
1739 | 1709 | ||
1740 | # If we don't have any setscene functions, skip this step | 1710 | # If we don't have any setscene functions, skip this step |
1741 | if len(self.rqdata.runq_setscene) == 0: | 1711 | if len(self.rqdata.runq_setscene_tids) == 0: |
1742 | rq.scenequeue_covered = set() | 1712 | rq.scenequeue_covered = set() |
1743 | rq.state = runQueueRunInit | 1713 | rq.state = runQueueRunInit |
1744 | return | 1714 | return |
1745 | 1715 | ||
1746 | self.stats = RunQueueStats(len(self.rqdata.runq_setscene)) | 1716 | self.stats = RunQueueStats(len(self.rqdata.runq_setscene_tids)) |
1747 | 1717 | ||
1748 | sq_revdeps = [] | 1718 | sq_revdeps = {} |
1749 | sq_revdeps_new = [] | 1719 | sq_revdeps_new = {} |
1750 | sq_revdeps_squash = [] | 1720 | sq_revdeps_squash = {} |
1751 | self.sq_harddeps = {} | 1721 | self.sq_harddeps = {} |
1752 | 1722 | ||
1753 | # We need to construct a dependency graph for the setscene functions. Intermediate | 1723 | # We need to construct a dependency graph for the setscene functions. Intermediate |
@@ -1757,18 +1727,21 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1757 | 1727 | ||
1758 | # First process the chains up to the first setscene task. | 1728 | # First process the chains up to the first setscene task. |
1759 | endpoints = {} | 1729 | endpoints = {} |
1760 | for task in range(len(self.rqdata.runq_fnid)): | 1730 | for tid in self.rqdata.runtaskentries: |
1761 | sq_revdeps.append(copy.copy(self.rqdata.runq_revdeps[task])) | 1731 | sq_revdeps[tid] = copy.copy(self.rqdata.runtaskentries[tid].revdeps) |
1762 | sq_revdeps_new.append(set()) | 1732 | sq_revdeps_new[tid] = set() |
1763 | if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene: | 1733 | if (len(sq_revdeps[tid]) == 0) and tid not in self.rqdata.runq_setscene_tids: |
1764 | endpoints[task] = set() | 1734 | #bb.warn("Added endpoint %s" % (tid)) |
1735 | endpoints[tid] = set() | ||
1765 | 1736 | ||
1766 | # Secondly process the chains between setscene tasks. | 1737 | # Secondly process the chains between setscene tasks. |
1767 | for task in self.rqdata.runq_setscene: | 1738 | for tid in self.rqdata.runq_setscene_tids: |
1768 | for dep in self.rqdata.runq_depends[task]: | 1739 | #bb.warn("Added endpoint 2 %s" % (tid)) |
1740 | for dep in self.rqdata.runtaskentries[tid].depends: | ||
1769 | if dep not in endpoints: | 1741 | if dep not in endpoints: |
1770 | endpoints[dep] = set() | 1742 | endpoints[dep] = set() |
1771 | endpoints[dep].add(task) | 1743 | #bb.warn(" Added endpoint 3 %s" % (dep)) |
1744 | endpoints[dep].add(tid) | ||
1772 | 1745 | ||
1773 | def process_endpoints(endpoints): | 1746 | def process_endpoints(endpoints): |
1774 | newendpoints = {} | 1747 | newendpoints = {} |
@@ -1779,15 +1752,15 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1779 | if sq_revdeps_new[point]: | 1752 | if sq_revdeps_new[point]: |
1780 | tasks |= sq_revdeps_new[point] | 1753 | tasks |= sq_revdeps_new[point] |
1781 | sq_revdeps_new[point] = set() | 1754 | sq_revdeps_new[point] = set() |
1782 | if point in self.rqdata.runq_setscene: | 1755 | if point in self.rqdata.runq_setscene_tids: |
1783 | sq_revdeps_new[point] = tasks | 1756 | sq_revdeps_new[point] = tasks |
1784 | tasks = set() | 1757 | tasks = set() |
1785 | for dep in self.rqdata.runq_depends[point]: | 1758 | for dep in self.rqdata.runtaskentries[point].depends: |
1786 | if point in sq_revdeps[dep]: | 1759 | if point in sq_revdeps[dep]: |
1787 | sq_revdeps[dep].remove(point) | 1760 | sq_revdeps[dep].remove(point) |
1788 | if tasks: | 1761 | if tasks: |
1789 | sq_revdeps_new[dep] |= tasks | 1762 | sq_revdeps_new[dep] |= tasks |
1790 | if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in self.rqdata.runq_setscene: | 1763 | if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in self.rqdata.runq_setscene_tids: |
1791 | newendpoints[dep] = task | 1764 | newendpoints[dep] = task |
1792 | if len(newendpoints) != 0: | 1765 | if len(newendpoints) != 0: |
1793 | process_endpoints(newendpoints) | 1766 | process_endpoints(newendpoints) |
@@ -1797,8 +1770,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1797 | # Build a list of setscene tasks which are "unskippable" | 1770 | # Build a list of setscene tasks which are "unskippable" |
1798 | # These are direct endpoints referenced by the build | 1771 | # These are direct endpoints referenced by the build |
1799 | endpoints2 = {} | 1772 | endpoints2 = {} |
1800 | sq_revdeps2 = [] | 1773 | sq_revdeps2 = {} |
1801 | sq_revdeps_new2 = [] | 1774 | sq_revdeps_new2 = {} |
1802 | def process_endpoints2(endpoints): | 1775 | def process_endpoints2(endpoints): |
1803 | newendpoints = {} | 1776 | newendpoints = {} |
1804 | for point, task in endpoints.items(): | 1777 | for point, task in endpoints.items(): |
@@ -1808,84 +1781,85 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1808 | if sq_revdeps_new2[point]: | 1781 | if sq_revdeps_new2[point]: |
1809 | tasks |= sq_revdeps_new2[point] | 1782 | tasks |= sq_revdeps_new2[point] |
1810 | sq_revdeps_new2[point] = set() | 1783 | sq_revdeps_new2[point] = set() |
1811 | if point in self.rqdata.runq_setscene: | 1784 | if point in self.rqdata.runq_setscene_tids: |
1812 | sq_revdeps_new2[point] = tasks | 1785 | sq_revdeps_new2[point] = tasks |
1813 | for dep in self.rqdata.runq_depends[point]: | 1786 | for dep in self.rqdata.runtaskentries[point].depends: |
1814 | if point in sq_revdeps2[dep]: | 1787 | if point in sq_revdeps2[dep]: |
1815 | sq_revdeps2[dep].remove(point) | 1788 | sq_revdeps2[dep].remove(point) |
1816 | if tasks: | 1789 | if tasks: |
1817 | sq_revdeps_new2[dep] |= tasks | 1790 | sq_revdeps_new2[dep] |= tasks |
1818 | if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in self.rqdata.runq_setscene: | 1791 | if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in self.rqdata.runq_setscene_tids: |
1819 | newendpoints[dep] = tasks | 1792 | newendpoints[dep] = tasks |
1820 | if len(newendpoints) != 0: | 1793 | if len(newendpoints) != 0: |
1821 | process_endpoints2(newendpoints) | 1794 | process_endpoints2(newendpoints) |
1822 | for task in range(len(self.rqdata.runq_fnid)): | 1795 | for tid in self.rqdata.runtaskentries: |
1823 | sq_revdeps2.append(copy.copy(self.rqdata.runq_revdeps[task])) | 1796 | sq_revdeps2[tid] = copy.copy(self.rqdata.runtaskentries[tid].revdeps) |
1824 | sq_revdeps_new2.append(set()) | 1797 | sq_revdeps_new2[tid] = set() |
1825 | if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene: | 1798 | if (len(sq_revdeps2[tid]) == 0) and tid not in self.rqdata.runq_setscene_tids: |
1826 | endpoints2[task] = set() | 1799 | endpoints2[tid] = set() |
1827 | process_endpoints2(endpoints2) | 1800 | process_endpoints2(endpoints2) |
1828 | self.unskippable = [] | 1801 | self.unskippable = [] |
1829 | for task in self.rqdata.runq_setscene: | 1802 | for tid in self.rqdata.runq_setscene_tids: |
1830 | if sq_revdeps_new2[task]: | 1803 | if sq_revdeps_new2[tid]: |
1831 | self.unskippable.append(self.rqdata.runq_setscene.index(task)) | 1804 | self.unskippable.append(tid) |
1832 | 1805 | ||
1833 | for task in range(len(self.rqdata.runq_fnid)): | 1806 | for tid in self.rqdata.runtaskentries: |
1834 | if task in self.rqdata.runq_setscene: | 1807 | if tid in self.rqdata.runq_setscene_tids: |
1835 | deps = set() | 1808 | deps = set() |
1836 | for dep in sq_revdeps_new[task]: | 1809 | for dep in sq_revdeps_new[tid]: |
1837 | deps.add(self.rqdata.runq_setscene.index(dep)) | 1810 | deps.add(dep) |
1838 | sq_revdeps_squash.append(deps) | 1811 | sq_revdeps_squash[tid] = deps |
1839 | elif len(sq_revdeps_new[task]) != 0: | 1812 | elif len(sq_revdeps_new[tid]) != 0: |
1840 | bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") | 1813 | bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") |
1841 | 1814 | ||
1842 | # Resolve setscene inter-task dependencies | 1815 | # Resolve setscene inter-task dependencies |
1843 | # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" | 1816 | # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" |
1844 | # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies | 1817 | # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies |
1845 | for task in self.rqdata.runq_setscene: | 1818 | for tid in self.rqdata.runq_setscene_tids: |
1846 | realid = self.rqdata.taskData.gettask_id(self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]], self.rqdata.runq_task[task] + "_setscene", False) | 1819 | realtid = tid + "_setscene" |
1847 | idepends = self.rqdata.taskData.tasks_idepends[realid] | 1820 | idepends = self.rqdata.taskData.taskentries[realtid].idepends |
1848 | for (depid, idependtask) in idepends: | 1821 | for (depname, idependtask) in idepends: |
1849 | if depid not in self.rqdata.taskData.build_targets: | 1822 | |
1823 | if depname not in self.rqdata.taskData.build_targets: | ||
1850 | continue | 1824 | continue |
1851 | 1825 | ||
1852 | depdata = self.rqdata.taskData.build_targets[depid][0] | 1826 | depfn = self.rqdata.taskData.build_targets[depname][0] |
1853 | if depdata is None: | 1827 | if depfn is None: |
1854 | continue | 1828 | continue |
1855 | dep = self.rqdata.taskData.fn_index[depdata] | 1829 | deptid = depfn + ":" + idependtask.replace("_setscene", "") |
1856 | taskid = self.rqdata.get_task_id(self.rqdata.taskData.getfn_id(dep), idependtask.replace("_setscene", "")) | 1830 | if deptid not in self.rqdata.runtaskentries: |
1857 | if taskid is None: | 1831 | bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask)) |
1858 | bb.msg.fatal("RunQueue", "Task %s_setscene depends upon non-existent task %s:%s" % (self.rqdata.get_user_idstring(task), dep, idependtask)) | ||
1859 | 1832 | ||
1860 | if not self.rqdata.runq_setscene.index(taskid) in self.sq_harddeps: | 1833 | if not deptid in self.sq_harddeps: |
1861 | self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)] = set() | 1834 | self.sq_harddeps[deptid] = set() |
1862 | self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)].add(self.rqdata.runq_setscene.index(task)) | 1835 | self.sq_harddeps[deptid].add(tid) |
1863 | 1836 | ||
1864 | sq_revdeps_squash[self.rqdata.runq_setscene.index(task)].add(self.rqdata.runq_setscene.index(taskid)) | 1837 | sq_revdeps_squash[tid].add(deptid) |
1865 | # Have to zero this to avoid circular dependencies | 1838 | # Have to zero this to avoid circular dependencies |
1866 | sq_revdeps_squash[self.rqdata.runq_setscene.index(taskid)] = set() | 1839 | sq_revdeps_squash[deptid] = set() |
1867 | 1840 | ||
1868 | for task in self.sq_harddeps: | 1841 | for task in self.sq_harddeps: |
1869 | for dep in self.sq_harddeps[task]: | 1842 | for dep in self.sq_harddeps[task]: |
1870 | sq_revdeps_squash[dep].add(task) | 1843 | sq_revdeps_squash[dep].add(task) |
1871 | 1844 | ||
1872 | #for task in range(len(sq_revdeps_squash)): | 1845 | #for tid in sq_revdeps_squash: |
1873 | # realtask = self.rqdata.runq_setscene[task] | 1846 | # for dep in sq_revdeps_squash[tid]: |
1874 | # bb.warn("Task %s: %s_setscene is %s " % (task, self.rqdata.get_user_idstring(realtask) , sq_revdeps_squash[task])) | 1847 | # data = data + "\n %s" % dep |
1848 | # bb.warn("Task %s_setscene: is %s " % (tid, data | ||
1875 | 1849 | ||
1876 | self.sq_deps = [] | 1850 | self.sq_deps = {} |
1877 | self.sq_revdeps = sq_revdeps_squash | 1851 | self.sq_revdeps = sq_revdeps_squash |
1878 | self.sq_revdeps2 = copy.deepcopy(self.sq_revdeps) | 1852 | self.sq_revdeps2 = copy.deepcopy(self.sq_revdeps) |
1879 | 1853 | ||
1880 | for task in range(len(self.sq_revdeps)): | 1854 | for tid in self.sq_revdeps: |
1881 | self.sq_deps.append(set()) | 1855 | self.sq_deps[tid] = set() |
1882 | for task in range(len(self.sq_revdeps)): | 1856 | for tid in self.sq_revdeps: |
1883 | for dep in self.sq_revdeps[task]: | 1857 | for dep in self.sq_revdeps[tid]: |
1884 | self.sq_deps[dep].add(task) | 1858 | self.sq_deps[dep].add(tid) |
1885 | 1859 | ||
1886 | for task in range(len(self.sq_revdeps)): | 1860 | for tid in self.sq_revdeps: |
1887 | if len(self.sq_revdeps[task]) == 0: | 1861 | if len(self.sq_revdeps[tid]) == 0: |
1888 | self.runq_buildable.add(task) | 1862 | self.runq_buildable.add(tid) |
1889 | 1863 | ||
1890 | self.outrightfail = [] | 1864 | self.outrightfail = [] |
1891 | if self.rq.hashvalidate: | 1865 | if self.rq.hashvalidate: |
@@ -1896,35 +1870,35 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1896 | sq_task = [] | 1870 | sq_task = [] |
1897 | noexec = [] | 1871 | noexec = [] |
1898 | stamppresent = [] | 1872 | stamppresent = [] |
1899 | for task in range(len(self.sq_revdeps)): | 1873 | for tid in self.sq_revdeps: |
1900 | realtask = self.rqdata.runq_setscene[task] | 1874 | fn = fn_from_tid(tid) |
1901 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]] | 1875 | taskname = taskname_from_tid(tid) |
1902 | taskname = self.rqdata.runq_task[realtask] | 1876 | |
1903 | taskdep = self.rqdata.dataCache.task_deps[fn] | 1877 | taskdep = self.rqdata.dataCache.task_deps[fn] |
1904 | 1878 | ||
1905 | if 'noexec' in taskdep and taskname in taskdep['noexec']: | 1879 | if 'noexec' in taskdep and taskname in taskdep['noexec']: |
1906 | noexec.append(task) | 1880 | noexec.append(tid) |
1907 | self.task_skip(task) | 1881 | self.task_skip(tid) |
1908 | bb.build.make_stamp(taskname + "_setscene", self.rqdata.dataCache, fn) | 1882 | bb.build.make_stamp(taskname + "_setscene", self.rqdata.dataCache, fn) |
1909 | continue | 1883 | continue |
1910 | 1884 | ||
1911 | if self.rq.check_stamp_task(realtask, taskname + "_setscene", cache=self.stampcache): | 1885 | if self.rq.check_stamp_task(tid, taskname + "_setscene", cache=self.stampcache): |
1912 | logger.debug(2, 'Setscene stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(realtask)) | 1886 | logger.debug(2, 'Setscene stamp current for task %s', tid) |
1913 | stamppresent.append(task) | 1887 | stamppresent.append(tid) |
1914 | self.task_skip(task) | 1888 | self.task_skip(tid) |
1915 | continue | 1889 | continue |
1916 | 1890 | ||
1917 | if self.rq.check_stamp_task(realtask, taskname, recurse = True, cache=self.stampcache): | 1891 | if self.rq.check_stamp_task(tid, taskname, recurse = True, cache=self.stampcache): |
1918 | logger.debug(2, 'Normal stamp current for task %s(%s)', task, self.rqdata.get_user_idstring(realtask)) | 1892 | logger.debug(2, 'Normal stamp current for task %s', tid) |
1919 | stamppresent.append(task) | 1893 | stamppresent.append(tid) |
1920 | self.task_skip(task) | 1894 | self.task_skip(tid) |
1921 | continue | 1895 | continue |
1922 | 1896 | ||
1923 | sq_fn.append(fn) | 1897 | sq_fn.append(fn) |
1924 | sq_hashfn.append(self.rqdata.dataCache.hashfn[fn]) | 1898 | sq_hashfn.append(self.rqdata.dataCache.hashfn[fn]) |
1925 | sq_hash.append(self.rqdata.runq_hash[realtask]) | 1899 | sq_hash.append(self.rqdata.runtaskentries[tid].hash) |
1926 | sq_taskname.append(taskname) | 1900 | sq_taskname.append(taskname) |
1927 | sq_task.append(task) | 1901 | sq_task.append(tid) |
1928 | call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)" | 1902 | call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)" |
1929 | locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data } | 1903 | locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d" : self.cooker.expanded_data } |
1930 | valid = bb.utils.better_eval(call, locs) | 1904 | valid = bb.utils.better_eval(call, locs) |
@@ -1933,12 +1907,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1933 | for v in valid: | 1907 | for v in valid: |
1934 | valid_new.append(sq_task[v]) | 1908 | valid_new.append(sq_task[v]) |
1935 | 1909 | ||
1936 | for task in range(len(self.sq_revdeps)): | 1910 | for tid in self.sq_revdeps: |
1937 | if task not in valid_new and task not in noexec: | 1911 | if tid not in valid_new and tid not in noexec: |
1938 | realtask = self.rqdata.runq_setscene[task] | 1912 | logger.debug(2, 'No package found, so skipping setscene task %s', tid) |
1939 | logger.debug(2, 'No package found, so skipping setscene task %s', | 1913 | self.outrightfail.append(tid) |
1940 | self.rqdata.get_user_idstring(realtask)) | ||
1941 | self.outrightfail.append(task) | ||
1942 | 1914 | ||
1943 | logger.info('Executing SetScene Tasks') | 1915 | logger.info('Executing SetScene Tasks') |
1944 | 1916 | ||
@@ -1947,9 +1919,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1947 | def scenequeue_updatecounters(self, task, fail = False): | 1919 | def scenequeue_updatecounters(self, task, fail = False): |
1948 | for dep in self.sq_deps[task]: | 1920 | for dep in self.sq_deps[task]: |
1949 | if fail and task in self.sq_harddeps and dep in self.sq_harddeps[task]: | 1921 | if fail and task in self.sq_harddeps and dep in self.sq_harddeps[task]: |
1950 | realtask = self.rqdata.runq_setscene[task] | 1922 | logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep)) |
1951 | realdep = self.rqdata.runq_setscene[dep] | ||
1952 | logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (self.rqdata.get_user_idstring(realtask), self.rqdata.get_user_idstring(realdep))) | ||
1953 | self.scenequeue_updatecounters(dep, fail) | 1923 | self.scenequeue_updatecounters(dep, fail) |
1954 | continue | 1924 | continue |
1955 | if task not in self.sq_revdeps2[dep]: | 1925 | if task not in self.sq_revdeps2[dep]: |
@@ -1966,10 +1936,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1966 | completed dependencies as buildable | 1936 | completed dependencies as buildable |
1967 | """ | 1937 | """ |
1968 | 1938 | ||
1969 | index = self.rqdata.runq_setscene[task] | 1939 | logger.debug(1, 'Found task %s which could be accelerated', task) |
1970 | logger.debug(1, 'Found task %s which could be accelerated', | ||
1971 | self.rqdata.get_user_idstring(index)) | ||
1972 | |||
1973 | self.scenequeue_covered.add(task) | 1940 | self.scenequeue_covered.add(task) |
1974 | self.scenequeue_updatecounters(task) | 1941 | self.scenequeue_updatecounters(task) |
1975 | 1942 | ||
@@ -1989,7 +1956,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
1989 | self.runq_buildable.add(task) | 1956 | self.runq_buildable.add(task) |
1990 | self.stats.taskCompleted() | 1957 | self.stats.taskCompleted() |
1991 | self.stats.taskSkipped() | 1958 | self.stats.taskSkipped() |
1992 | index = self.rqdata.runq_setscene[task] | ||
1993 | self.scenequeue_notcovered.add(task) | 1959 | self.scenequeue_notcovered.add(task) |
1994 | self.scenequeue_updatecounters(task, True) | 1960 | self.scenequeue_updatecounters(task, True) |
1995 | 1961 | ||
@@ -2010,20 +1976,19 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2010 | task = None | 1976 | task = None |
2011 | if self.stats.active < self.number_tasks: | 1977 | if self.stats.active < self.number_tasks: |
2012 | # Find the next setscene to run | 1978 | # Find the next setscene to run |
2013 | for nexttask in range(self.stats.total): | 1979 | for nexttask in self.rqdata.runq_setscene_tids: |
2014 | if nexttask in self.runq_buildable and nexttask not in self.runq_running: | 1980 | if nexttask in self.runq_buildable and nexttask not in self.runq_running: |
2015 | if nexttask in self.unskippable: | 1981 | if nexttask in self.unskippable: |
2016 | logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask])) | 1982 | logger.debug(2, "Setscene task %s is unskippable" % nexttask) |
2017 | if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True): | 1983 | if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True): |
2018 | realtask = self.rqdata.runq_setscene[nexttask] | 1984 | fn = fn_from_tid(nexttask) |
2019 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]] | ||
2020 | foundtarget = False | 1985 | foundtarget = False |
2021 | for target in self.rqdata.target_pairs: | 1986 | for target in self.rqdata.target_pairs: |
2022 | if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]: | 1987 | if target[0] == fn and target[1] == taskname_from_tid(nexttask): |
2023 | foundtarget = True | 1988 | foundtarget = True |
2024 | break | 1989 | break |
2025 | if not foundtarget: | 1990 | if not foundtarget: |
2026 | logger.debug(2, "Skipping setscene for task %s" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask])) | 1991 | logger.debug(2, "Skipping setscene for task %s" % nexttask) |
2027 | self.task_skip(nexttask) | 1992 | self.task_skip(nexttask) |
2028 | self.scenequeue_notneeded.add(nexttask) | 1993 | self.scenequeue_notneeded.add(nexttask) |
2029 | return True | 1994 | return True |
@@ -2033,25 +1998,21 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2033 | task = nexttask | 1998 | task = nexttask |
2034 | break | 1999 | break |
2035 | if task is not None: | 2000 | if task is not None: |
2036 | realtask = self.rqdata.runq_setscene[task] | 2001 | fn = fn_from_tid(task) |
2037 | fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]] | 2002 | taskname = taskname_from_tid(task) + "_setscene" |
2038 | 2003 | if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache): | |
2039 | taskname = self.rqdata.runq_task[realtask] + "_setscene" | 2004 | logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task) |
2040 | if self.rq.check_stamp_task(realtask, self.rqdata.runq_task[realtask], recurse = True, cache=self.stampcache): | ||
2041 | logger.debug(2, 'Stamp for underlying task %s(%s) is current, so skipping setscene variant', | ||
2042 | task, self.rqdata.get_user_idstring(realtask)) | ||
2043 | self.task_failoutright(task) | 2005 | self.task_failoutright(task) |
2044 | return True | 2006 | return True |
2045 | 2007 | ||
2046 | if self.cooker.configuration.force: | 2008 | if self.cooker.configuration.force: |
2047 | for target in self.rqdata.target_pairs: | 2009 | for target in self.rqdata.target_pairs: |
2048 | if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]: | 2010 | if target[0] == fn and target[1] == taskname_from_tid(task): |
2049 | self.task_failoutright(task) | 2011 | self.task_failoutright(task) |
2050 | return True | 2012 | return True |
2051 | 2013 | ||
2052 | if self.rq.check_stamp_task(realtask, taskname, cache=self.stampcache): | 2014 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): |
2053 | logger.debug(2, 'Setscene stamp current task %s(%s), so skip it and its dependencies', | 2015 | logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task) |
2054 | task, self.rqdata.get_user_idstring(realtask)) | ||
2055 | self.task_skip(task) | 2016 | self.task_skip(task) |
2056 | return True | 2017 | return True |
2057 | 2018 | ||
@@ -2062,10 +2023,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2062 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: | 2023 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: |
2063 | if not self.rq.fakeworker: | 2024 | if not self.rq.fakeworker: |
2064 | self.rq.start_fakeworker(self) | 2025 | self.rq.start_fakeworker(self) |
2065 | self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>") | 2026 | self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>") |
2066 | self.rq.fakeworker.stdin.flush() | 2027 | self.rq.fakeworker.stdin.flush() |
2067 | else: | 2028 | else: |
2068 | self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>") | 2029 | self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>") |
2069 | self.rq.worker.stdin.flush() | 2030 | self.rq.worker.stdin.flush() |
2070 | 2031 | ||
2071 | self.runq_running.add(task) | 2032 | self.runq_running.add(task) |
@@ -2077,17 +2038,17 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2077 | self.rq.read_workers() | 2038 | self.rq.read_workers() |
2078 | return self.rq.active_fds() | 2039 | return self.rq.active_fds() |
2079 | 2040 | ||
2080 | #for task in range(self.stats.total): | 2041 | #for tid in self.sq_revdeps: |
2081 | # if task not in self.runq_running: | 2042 | # if tid not in self.runq_running: |
2082 | # buildable = task in self.runq_buildable | 2043 | # buildable = tid in self.runq_buildable |
2083 | # revdeps = self.sq_revdeps[task] | 2044 | # revdeps = self.sq_revdeps[tid] |
2084 | # bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task]))) | 2045 | # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps))) |
2085 | 2046 | ||
2086 | # Convert scenequeue_covered task numbers into full taskgraph ids | 2047 | # Convert scenequeue_covered task numbers into full taskgraph ids |
2087 | oldcovered = self.scenequeue_covered | 2048 | oldcovered = self.scenequeue_covered |
2088 | self.rq.scenequeue_covered = set() | 2049 | self.rq.scenequeue_covered = set() |
2089 | for task in oldcovered: | 2050 | for task in oldcovered: |
2090 | self.rq.scenequeue_covered.add(self.rqdata.runq_setscene[task]) | 2051 | self.rq.scenequeue_covered.add(task) |
2091 | 2052 | ||
2092 | logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered)) | 2053 | logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered)) |
2093 | 2054 | ||
@@ -2099,8 +2060,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2099 | return True | 2060 | return True |
2100 | 2061 | ||
2101 | def runqueue_process_waitpid(self, task, status): | 2062 | def runqueue_process_waitpid(self, task, status): |
2102 | task = self.rq.rqdata.runq_setscene.index(task) | ||
2103 | |||
2104 | RunQueueExecute.runqueue_process_waitpid(self, task, status) | 2063 | RunQueueExecute.runqueue_process_waitpid(self, task, status) |
2105 | 2064 | ||
2106 | class TaskFailure(Exception): | 2065 | class TaskFailure(Exception): |
@@ -2127,9 +2086,9 @@ class runQueueEvent(bb.event.Event): | |||
2127 | """ | 2086 | """ |
2128 | def __init__(self, task, stats, rq): | 2087 | def __init__(self, task, stats, rq): |
2129 | self.taskid = task | 2088 | self.taskid = task |
2130 | self.taskstring = rq.rqdata.get_user_idstring(task) | 2089 | self.taskstring = task |
2131 | self.taskname = rq.rqdata.get_task_name(task) | 2090 | self.taskname = taskname_from_tid(task) |
2132 | self.taskfile = rq.rqdata.get_task_file(task) | 2091 | self.taskfile = fn_from_tid(task) |
2133 | self.taskhash = rq.rqdata.get_task_hash(task) | 2092 | self.taskhash = rq.rqdata.get_task_hash(task) |
2134 | self.stats = stats.copy() | 2093 | self.stats = stats.copy() |
2135 | bb.event.Event.__init__(self) | 2094 | bb.event.Event.__init__(self) |
@@ -2140,11 +2099,10 @@ class sceneQueueEvent(runQueueEvent): | |||
2140 | """ | 2099 | """ |
2141 | def __init__(self, task, stats, rq, noexec=False): | 2100 | def __init__(self, task, stats, rq, noexec=False): |
2142 | runQueueEvent.__init__(self, task, stats, rq) | 2101 | runQueueEvent.__init__(self, task, stats, rq) |
2143 | realtask = rq.rqdata.runq_setscene[task] | 2102 | self.taskstring = task + "_setscene" |
2144 | self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene") | 2103 | self.taskname = taskname_from_tid(task) + "_setscene" |
2145 | self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene" | 2104 | self.taskfile = fn_from_tid(task) |
2146 | self.taskfile = rq.rqdata.get_task_file(realtask) | 2105 | self.taskhash = rq.rqdata.get_task_hash(task) |
2147 | self.taskhash = rq.rqdata.get_task_hash(realtask) | ||
2148 | 2106 | ||
2149 | class runQueueTaskStarted(runQueueEvent): | 2107 | class runQueueTaskStarted(runQueueEvent): |
2150 | """ | 2108 | """ |