diff options
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 93 |
1 files changed, 46 insertions, 47 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 9a368b8622..2ecfd09469 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -30,7 +30,7 @@ import fcntl | |||
30 | 30 | ||
31 | class TaskFailure(Exception): | 31 | class TaskFailure(Exception): |
32 | """Exception raised when a task in a runqueue fails""" | 32 | """Exception raised when a task in a runqueue fails""" |
33 | def __init__(self, x): | 33 | def __init__(self, x): |
34 | self.args = x | 34 | self.args = x |
35 | 35 | ||
36 | 36 | ||
@@ -60,7 +60,7 @@ class RunQueueStats: | |||
60 | def taskActive(self): | 60 | def taskActive(self): |
61 | self.active = self.active + 1 | 61 | self.active = self.active + 1 |
62 | 62 | ||
63 | # These values indicate the next step due to be run in the | 63 | # These values indicate the next step due to be run in the |
64 | # runQueue state machine | 64 | # runQueue state machine |
65 | runQueuePrepare = 2 | 65 | runQueuePrepare = 2 |
66 | runQueueRunInit = 3 | 66 | runQueueRunInit = 3 |
@@ -76,7 +76,7 @@ class RunQueueScheduler: | |||
76 | """ | 76 | """ |
77 | def __init__(self, runqueue): | 77 | def __init__(self, runqueue): |
78 | """ | 78 | """ |
79 | The default scheduler just returns the first buildable task (the | 79 | The default scheduler just returns the first buildable task (the |
80 | priority map is sorted by task numer) | 80 | priority map is sorted by task numer) |
81 | """ | 81 | """ |
82 | self.rq = runqueue | 82 | self.rq = runqueue |
@@ -123,10 +123,10 @@ class RunQueueSchedulerSpeed(RunQueueScheduler): | |||
123 | 123 | ||
124 | class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): | 124 | class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): |
125 | """ | 125 | """ |
126 | A scheduler optimised to complete .bb files are quickly as possible. The | 126 | A scheduler optimised to complete .bb files are quickly as possible. The |
127 | priority map is sorted by task weight, but then reordered so once a given | 127 | priority map is sorted by task weight, but then reordered so once a given |
128 | .bb file starts to build, its completed as quickly as possible. This works | 128 | .bb file starts to build, its completed as quickly as possible. This works |
129 | well where disk space is at a premium and classes like OE's rm_work are in | 129 | well where disk space is at a premium and classes like OE's rm_work are in |
130 | force. | 130 | force. |
131 | """ | 131 | """ |
132 | def __init__(self, runqueue): | 132 | def __init__(self, runqueue): |
@@ -135,7 +135,7 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): | |||
135 | 135 | ||
136 | #FIXME - whilst this groups all fnids together it does not reorder the | 136 | #FIXME - whilst this groups all fnids together it does not reorder the |
137 | #fnid groups optimally. | 137 | #fnid groups optimally. |
138 | 138 | ||
139 | basemap = deepcopy(self.prio_map) | 139 | basemap = deepcopy(self.prio_map) |
140 | self.prio_map = [] | 140 | self.prio_map = [] |
141 | while (len(basemap) > 0): | 141 | while (len(basemap) > 0): |
@@ -231,7 +231,7 @@ class RunQueue: | |||
231 | if chain1[index] != chain2[index]: | 231 | if chain1[index] != chain2[index]: |
232 | return False | 232 | return False |
233 | return True | 233 | return True |
234 | 234 | ||
235 | def chain_array_contains(chain, chain_array): | 235 | def chain_array_contains(chain, chain_array): |
236 | """ | 236 | """ |
237 | Return True if chain_array contains chain | 237 | Return True if chain_array contains chain |
@@ -286,7 +286,7 @@ class RunQueue: | |||
286 | 286 | ||
287 | def calculate_task_weights(self, endpoints): | 287 | def calculate_task_weights(self, endpoints): |
288 | """ | 288 | """ |
289 | Calculate a number representing the "weight" of each task. Heavier weighted tasks | 289 | Calculate a number representing the "weight" of each task. Heavier weighted tasks |
290 | have more dependencies and hence should be executed sooner for maximum speed. | 290 | have more dependencies and hence should be executed sooner for maximum speed. |
291 | 291 | ||
292 | This function also sanity checks the task list finding tasks that its not | 292 | This function also sanity checks the task list finding tasks that its not |
@@ -318,7 +318,7 @@ class RunQueue: | |||
318 | task_done[revdep] = True | 318 | task_done[revdep] = True |
319 | endpoints = next_points | 319 | endpoints = next_points |
320 | if len(next_points) == 0: | 320 | if len(next_points) == 0: |
321 | break | 321 | break |
322 | 322 | ||
323 | # Circular dependency sanity check | 323 | # Circular dependency sanity check |
324 | problem_tasks = [] | 324 | problem_tasks = [] |
@@ -345,7 +345,7 @@ class RunQueue: | |||
345 | 345 | ||
346 | def prepare_runqueue(self): | 346 | def prepare_runqueue(self): |
347 | """ | 347 | """ |
348 | Turn a set of taskData into a RunQueue and compute data needed | 348 | Turn a set of taskData into a RunQueue and compute data needed |
349 | to optimise the execution order. | 349 | to optimise the execution order. |
350 | """ | 350 | """ |
351 | 351 | ||
@@ -365,12 +365,12 @@ class RunQueue: | |||
365 | # Step A - Work out a list of tasks to run | 365 | # Step A - Work out a list of tasks to run |
366 | # | 366 | # |
367 | # Taskdata gives us a list of possible providers for every build and run | 367 | # Taskdata gives us a list of possible providers for every build and run |
368 | # target ordered by priority. It also gives information on each of those | 368 | # target ordered by priority. It also gives information on each of those |
369 | # providers. | 369 | # providers. |
370 | # | 370 | # |
371 | # To create the actual list of tasks to execute we fix the list of | 371 | # To create the actual list of tasks to execute we fix the list of |
372 | # providers and then resolve the dependencies into task IDs. This | 372 | # providers and then resolve the dependencies into task IDs. This |
373 | # process is repeated for each type of dependency (tdepends, deptask, | 373 | # process is repeated for each type of dependency (tdepends, deptask, |
374 | # rdeptast, recrdeptask, idepends). | 374 | # rdeptast, recrdeptask, idepends). |
375 | 375 | ||
376 | def add_build_dependencies(depids, tasknames, depends): | 376 | def add_build_dependencies(depids, tasknames, depends): |
@@ -411,12 +411,12 @@ class RunQueue: | |||
411 | 411 | ||
412 | if fnid not in taskData.failed_fnids: | 412 | if fnid not in taskData.failed_fnids: |
413 | 413 | ||
414 | # Resolve task internal dependencies | 414 | # Resolve task internal dependencies |
415 | # | 415 | # |
416 | # e.g. addtask before X after Y | 416 | # e.g. addtask before X after Y |
417 | depends = taskData.tasks_tdepends[task] | 417 | depends = taskData.tasks_tdepends[task] |
418 | 418 | ||
419 | # Resolve 'deptask' dependencies | 419 | # Resolve 'deptask' dependencies |
420 | # | 420 | # |
421 | # e.g. do_sometask[deptask] = "do_someothertask" | 421 | # e.g. do_sometask[deptask] = "do_someothertask" |
422 | # (makes sure sometask runs after someothertask of all DEPENDS) | 422 | # (makes sure sometask runs after someothertask of all DEPENDS) |
@@ -424,7 +424,7 @@ class RunQueue: | |||
424 | tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() | 424 | tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() |
425 | add_build_dependencies(taskData.depids[fnid], tasknames, depends) | 425 | add_build_dependencies(taskData.depids[fnid], tasknames, depends) |
426 | 426 | ||
427 | # Resolve 'rdeptask' dependencies | 427 | # Resolve 'rdeptask' dependencies |
428 | # | 428 | # |
429 | # e.g. do_sometask[rdeptask] = "do_someothertask" | 429 | # e.g. do_sometask[rdeptask] = "do_someothertask" |
430 | # (makes sure sometask runs after someothertask of all RDEPENDS) | 430 | # (makes sure sometask runs after someothertask of all RDEPENDS) |
@@ -432,7 +432,7 @@ class RunQueue: | |||
432 | taskname = task_deps['rdeptask'][taskData.tasks_name[task]] | 432 | taskname = task_deps['rdeptask'][taskData.tasks_name[task]] |
433 | add_runtime_dependencies(taskData.rdepids[fnid], [taskname], depends) | 433 | add_runtime_dependencies(taskData.rdepids[fnid], [taskname], depends) |
434 | 434 | ||
435 | # Resolve inter-task dependencies | 435 | # Resolve inter-task dependencies |
436 | # | 436 | # |
437 | # e.g. do_sometask[depends] = "targetname:do_someothertask" | 437 | # e.g. do_sometask[depends] = "targetname:do_someothertask" |
438 | # (makes sure sometask runs after targetname's someothertask) | 438 | # (makes sure sometask runs after targetname's someothertask) |
@@ -467,8 +467,8 @@ class RunQueue: | |||
467 | newdep = [] | 467 | newdep = [] |
468 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends)) | 468 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends)) |
469 | for dep in depends: | 469 | for dep in depends: |
470 | if task != dep: | 470 | if task != dep: |
471 | newdep.append(dep) | 471 | newdep.append(dep) |
472 | depends = newdep | 472 | depends = newdep |
473 | 473 | ||
474 | self.runq_fnid.append(taskData.tasks_fnid[task]) | 474 | self.runq_fnid.append(taskData.tasks_fnid[task]) |
@@ -482,7 +482,7 @@ class RunQueue: | |||
482 | # | 482 | # |
483 | # Build a list of recursive cumulative dependencies for each fnid | 483 | # Build a list of recursive cumulative dependencies for each fnid |
484 | # We do this by fnid, since if A depends on some task in B | 484 | # We do this by fnid, since if A depends on some task in B |
485 | # we're interested in later tasks B's fnid might have but B itself | 485 | # we're interested in later tasks B's fnid might have but B itself |
486 | # doesn't depend on | 486 | # doesn't depend on |
487 | # | 487 | # |
488 | # Algorithm is O(tasks) + O(tasks)*O(fnids) | 488 | # Algorithm is O(tasks) + O(tasks)*O(fnids) |
@@ -513,7 +513,7 @@ class RunQueue: | |||
513 | if len(runq_recrdepends[task]) > 0: | 513 | if len(runq_recrdepends[task]) > 0: |
514 | taskfnid = self.runq_fnid[task] | 514 | taskfnid = self.runq_fnid[task] |
515 | for dep in reccumdepends[taskfnid]: | 515 | for dep in reccumdepends[taskfnid]: |
516 | # Ignore self references | 516 | # Ignore self references |
517 | if dep == task: | 517 | if dep == task: |
518 | continue | 518 | continue |
519 | for taskname in runq_recrdepends[task]: | 519 | for taskname in runq_recrdepends[task]: |
@@ -635,7 +635,7 @@ class RunQueue: | |||
635 | 635 | ||
636 | bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints)) | 636 | bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints)) |
637 | 637 | ||
638 | # Calculate task weights | 638 | # Calculate task weights |
639 | # Check of higher length circular dependencies | 639 | # Check of higher length circular dependencies |
640 | self.runq_weight = self.calculate_task_weights(endpoints) | 640 | self.runq_weight = self.calculate_task_weights(endpoints) |
641 | 641 | ||
@@ -657,7 +657,7 @@ class RunQueue: | |||
657 | for prov in self.dataCache.fn_provides[fn]: | 657 | for prov in self.dataCache.fn_provides[fn]: |
658 | if prov not in prov_list: | 658 | if prov not in prov_list: |
659 | prov_list[prov] = [fn] | 659 | prov_list[prov] = [fn] |
660 | elif fn not in prov_list[prov]: | 660 | elif fn not in prov_list[prov]: |
661 | prov_list[prov].append(fn) | 661 | prov_list[prov].append(fn) |
662 | error = False | 662 | error = False |
663 | for prov in prov_list: | 663 | for prov in prov_list: |
@@ -703,7 +703,7 @@ class RunQueue: | |||
703 | buildable.append(task) | 703 | buildable.append(task) |
704 | 704 | ||
705 | def check_buildable(self, task, buildable): | 705 | def check_buildable(self, task, buildable): |
706 | for revdep in self.runq_revdeps[task]: | 706 | for revdep in self.runq_revdeps[task]: |
707 | alldeps = 1 | 707 | alldeps = 1 |
708 | for dep in self.runq_depends[revdep]: | 708 | for dep in self.runq_depends[revdep]: |
709 | if dep in unchecked: | 709 | if dep in unchecked: |
@@ -811,10 +811,10 @@ class RunQueue: | |||
811 | try: | 811 | try: |
812 | t2 = os.stat(stampfile2)[stat.ST_MTIME] | 812 | t2 = os.stat(stampfile2)[stat.ST_MTIME] |
813 | if t1 < t2: | 813 | if t1 < t2: |
814 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile,stampfile2)) | 814 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile, stampfile2)) |
815 | iscurrent = False | 815 | iscurrent = False |
816 | except: | 816 | except: |
817 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2 ,stampfile)) | 817 | bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2 , stampfile)) |
818 | iscurrent = False | 818 | iscurrent = False |
819 | 819 | ||
820 | return iscurrent | 820 | return iscurrent |
@@ -885,7 +885,7 @@ class RunQueue: | |||
885 | def task_complete(self, task): | 885 | def task_complete(self, task): |
886 | """ | 886 | """ |
887 | Mark a task as completed | 887 | Mark a task as completed |
888 | Look at the reverse dependencies and mark any task with | 888 | Look at the reverse dependencies and mark any task with |
889 | completed dependencies as buildable | 889 | completed dependencies as buildable |
890 | """ | 890 | """ |
891 | self.runq_complete[task] = 1 | 891 | self.runq_complete[task] = 1 |
@@ -1033,10 +1033,10 @@ class RunQueue: | |||
1033 | def finish_runqueue_now(self): | 1033 | def finish_runqueue_now(self): |
1034 | bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active) | 1034 | bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active) |
1035 | for k, v in self.build_pids.iteritems(): | 1035 | for k, v in self.build_pids.iteritems(): |
1036 | try: | 1036 | try: |
1037 | os.kill(-k, signal.SIGINT) | 1037 | os.kill(-k, signal.SIGINT) |
1038 | except: | 1038 | except: |
1039 | pass | 1039 | pass |
1040 | for pipe in self.build_pipes: | 1040 | for pipe in self.build_pipes: |
1041 | self.build_pipes[pipe].read() | 1041 | self.build_pipes[pipe].read() |
1042 | 1042 | ||
@@ -1085,30 +1085,30 @@ class RunQueue: | |||
1085 | """ | 1085 | """ |
1086 | bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:") | 1086 | bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:") |
1087 | for task in range(len(self.runq_task)): | 1087 | for task in range(len(self.runq_task)): |
1088 | bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, | 1088 | bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, |
1089 | taskQueue.fn_index[self.runq_fnid[task]], | 1089 | taskQueue.fn_index[self.runq_fnid[task]], |
1090 | self.runq_task[task], | 1090 | self.runq_task[task], |
1091 | self.runq_weight[task], | 1091 | self.runq_weight[task], |
1092 | self.runq_depends[task], | 1092 | self.runq_depends[task], |
1093 | self.runq_revdeps[task])) | 1093 | self.runq_revdeps[task])) |
1094 | 1094 | ||
1095 | bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:") | 1095 | bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:") |
1096 | for task1 in range(len(self.runq_task)): | 1096 | for task1 in range(len(self.runq_task)): |
1097 | if task1 in self.prio_map: | 1097 | if task1 in self.prio_map: |
1098 | task = self.prio_map[task1] | 1098 | task = self.prio_map[task1] |
1099 | bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, | 1099 | bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, |
1100 | taskQueue.fn_index[self.runq_fnid[task]], | 1100 | taskQueue.fn_index[self.runq_fnid[task]], |
1101 | self.runq_task[task], | 1101 | self.runq_task[task], |
1102 | self.runq_weight[task], | 1102 | self.runq_weight[task], |
1103 | self.runq_depends[task], | 1103 | self.runq_depends[task], |
1104 | self.runq_revdeps[task])) | 1104 | self.runq_revdeps[task])) |
1105 | 1105 | ||
1106 | 1106 | ||
1107 | class TaskFailure(Exception): | 1107 | class TaskFailure(Exception): |
1108 | """ | 1108 | """ |
1109 | Exception raised when a task in a runqueue fails | 1109 | Exception raised when a task in a runqueue fails |
1110 | """ | 1110 | """ |
1111 | def __init__(self, x): | 1111 | def __init__(self, x): |
1112 | self.args = x | 1112 | self.args = x |
1113 | 1113 | ||
1114 | 1114 | ||
@@ -1196,4 +1196,3 @@ class runQueuePipe(): | |||
1196 | if len(self.queue) > 0: | 1196 | if len(self.queue) > 0: |
1197 | print "Warning, worker left partial message" | 1197 | print "Warning, worker left partial message" |
1198 | os.close(self.fd) | 1198 | os.close(self.fd) |
1199 | |||