summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/runqueue.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r--bitbake/lib/bb/runqueue.py93
1 files changed, 46 insertions, 47 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 9a368b8622..2ecfd09469 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -30,7 +30,7 @@ import fcntl
30 30
31class TaskFailure(Exception): 31class TaskFailure(Exception):
32 """Exception raised when a task in a runqueue fails""" 32 """Exception raised when a task in a runqueue fails"""
33 def __init__(self, x): 33 def __init__(self, x):
34 self.args = x 34 self.args = x
35 35
36 36
@@ -60,7 +60,7 @@ class RunQueueStats:
60 def taskActive(self): 60 def taskActive(self):
61 self.active = self.active + 1 61 self.active = self.active + 1
62 62
63# These values indicate the next step due to be run in the 63# These values indicate the next step due to be run in the
64# runQueue state machine 64# runQueue state machine
65runQueuePrepare = 2 65runQueuePrepare = 2
66runQueueRunInit = 3 66runQueueRunInit = 3
@@ -76,7 +76,7 @@ class RunQueueScheduler:
76 """ 76 """
77 def __init__(self, runqueue): 77 def __init__(self, runqueue):
78 """ 78 """
79 The default scheduler just returns the first buildable task (the 79 The default scheduler just returns the first buildable task (the
80 priority map is sorted by task numer) 80 priority map is sorted by task numer)
81 """ 81 """
82 self.rq = runqueue 82 self.rq = runqueue
@@ -123,10 +123,10 @@ class RunQueueSchedulerSpeed(RunQueueScheduler):
123 123
124class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed): 124class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
125 """ 125 """
126 A scheduler optimised to complete .bb files are quickly as possible. The 126 A scheduler optimised to complete .bb files are quickly as possible. The
127 priority map is sorted by task weight, but then reordered so once a given 127 priority map is sorted by task weight, but then reordered so once a given
128 .bb file starts to build, its completed as quickly as possible. This works 128 .bb file starts to build, its completed as quickly as possible. This works
129 well where disk space is at a premium and classes like OE's rm_work are in 129 well where disk space is at a premium and classes like OE's rm_work are in
130 force. 130 force.
131 """ 131 """
132 def __init__(self, runqueue): 132 def __init__(self, runqueue):
@@ -135,7 +135,7 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
135 135
136 #FIXME - whilst this groups all fnids together it does not reorder the 136 #FIXME - whilst this groups all fnids together it does not reorder the
137 #fnid groups optimally. 137 #fnid groups optimally.
138 138
139 basemap = deepcopy(self.prio_map) 139 basemap = deepcopy(self.prio_map)
140 self.prio_map = [] 140 self.prio_map = []
141 while (len(basemap) > 0): 141 while (len(basemap) > 0):
@@ -231,7 +231,7 @@ class RunQueue:
231 if chain1[index] != chain2[index]: 231 if chain1[index] != chain2[index]:
232 return False 232 return False
233 return True 233 return True
234 234
235 def chain_array_contains(chain, chain_array): 235 def chain_array_contains(chain, chain_array):
236 """ 236 """
237 Return True if chain_array contains chain 237 Return True if chain_array contains chain
@@ -286,7 +286,7 @@ class RunQueue:
286 286
287 def calculate_task_weights(self, endpoints): 287 def calculate_task_weights(self, endpoints):
288 """ 288 """
289 Calculate a number representing the "weight" of each task. Heavier weighted tasks 289 Calculate a number representing the "weight" of each task. Heavier weighted tasks
290 have more dependencies and hence should be executed sooner for maximum speed. 290 have more dependencies and hence should be executed sooner for maximum speed.
291 291
292 This function also sanity checks the task list finding tasks that its not 292 This function also sanity checks the task list finding tasks that its not
@@ -318,7 +318,7 @@ class RunQueue:
318 task_done[revdep] = True 318 task_done[revdep] = True
319 endpoints = next_points 319 endpoints = next_points
320 if len(next_points) == 0: 320 if len(next_points) == 0:
321 break 321 break
322 322
323 # Circular dependency sanity check 323 # Circular dependency sanity check
324 problem_tasks = [] 324 problem_tasks = []
@@ -345,7 +345,7 @@ class RunQueue:
345 345
346 def prepare_runqueue(self): 346 def prepare_runqueue(self):
347 """ 347 """
348 Turn a set of taskData into a RunQueue and compute data needed 348 Turn a set of taskData into a RunQueue and compute data needed
349 to optimise the execution order. 349 to optimise the execution order.
350 """ 350 """
351 351
@@ -365,12 +365,12 @@ class RunQueue:
365 # Step A - Work out a list of tasks to run 365 # Step A - Work out a list of tasks to run
366 # 366 #
367 # Taskdata gives us a list of possible providers for every build and run 367 # Taskdata gives us a list of possible providers for every build and run
368 # target ordered by priority. It also gives information on each of those 368 # target ordered by priority. It also gives information on each of those
369 # providers. 369 # providers.
370 # 370 #
371 # To create the actual list of tasks to execute we fix the list of 371 # To create the actual list of tasks to execute we fix the list of
372 # providers and then resolve the dependencies into task IDs. This 372 # providers and then resolve the dependencies into task IDs. This
373 # process is repeated for each type of dependency (tdepends, deptask, 373 # process is repeated for each type of dependency (tdepends, deptask,
374 # rdeptast, recrdeptask, idepends). 374 # rdeptast, recrdeptask, idepends).
375 375
376 def add_build_dependencies(depids, tasknames, depends): 376 def add_build_dependencies(depids, tasknames, depends):
@@ -411,12 +411,12 @@ class RunQueue:
411 411
412 if fnid not in taskData.failed_fnids: 412 if fnid not in taskData.failed_fnids:
413 413
414 # Resolve task internal dependencies 414 # Resolve task internal dependencies
415 # 415 #
416 # e.g. addtask before X after Y 416 # e.g. addtask before X after Y
417 depends = taskData.tasks_tdepends[task] 417 depends = taskData.tasks_tdepends[task]
418 418
419 # Resolve 'deptask' dependencies 419 # Resolve 'deptask' dependencies
420 # 420 #
421 # e.g. do_sometask[deptask] = "do_someothertask" 421 # e.g. do_sometask[deptask] = "do_someothertask"
422 # (makes sure sometask runs after someothertask of all DEPENDS) 422 # (makes sure sometask runs after someothertask of all DEPENDS)
@@ -424,7 +424,7 @@ class RunQueue:
424 tasknames = task_deps['deptask'][taskData.tasks_name[task]].split() 424 tasknames = task_deps['deptask'][taskData.tasks_name[task]].split()
425 add_build_dependencies(taskData.depids[fnid], tasknames, depends) 425 add_build_dependencies(taskData.depids[fnid], tasknames, depends)
426 426
427 # Resolve 'rdeptask' dependencies 427 # Resolve 'rdeptask' dependencies
428 # 428 #
429 # e.g. do_sometask[rdeptask] = "do_someothertask" 429 # e.g. do_sometask[rdeptask] = "do_someothertask"
430 # (makes sure sometask runs after someothertask of all RDEPENDS) 430 # (makes sure sometask runs after someothertask of all RDEPENDS)
@@ -432,7 +432,7 @@ class RunQueue:
432 taskname = task_deps['rdeptask'][taskData.tasks_name[task]] 432 taskname = task_deps['rdeptask'][taskData.tasks_name[task]]
433 add_runtime_dependencies(taskData.rdepids[fnid], [taskname], depends) 433 add_runtime_dependencies(taskData.rdepids[fnid], [taskname], depends)
434 434
435 # Resolve inter-task dependencies 435 # Resolve inter-task dependencies
436 # 436 #
437 # e.g. do_sometask[depends] = "targetname:do_someothertask" 437 # e.g. do_sometask[depends] = "targetname:do_someothertask"
438 # (makes sure sometask runs after targetname's someothertask) 438 # (makes sure sometask runs after targetname's someothertask)
@@ -467,8 +467,8 @@ class RunQueue:
467 newdep = [] 467 newdep = []
468 bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends)) 468 bb.msg.debug(2, bb.msg.domain.RunQueue, "Task %s (%s %s) contains self reference! %s" % (task, taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], depends))
469 for dep in depends: 469 for dep in depends:
470 if task != dep: 470 if task != dep:
471 newdep.append(dep) 471 newdep.append(dep)
472 depends = newdep 472 depends = newdep
473 473
474 self.runq_fnid.append(taskData.tasks_fnid[task]) 474 self.runq_fnid.append(taskData.tasks_fnid[task])
@@ -482,7 +482,7 @@ class RunQueue:
482 # 482 #
483 # Build a list of recursive cumulative dependencies for each fnid 483 # Build a list of recursive cumulative dependencies for each fnid
484 # We do this by fnid, since if A depends on some task in B 484 # We do this by fnid, since if A depends on some task in B
485 # we're interested in later tasks B's fnid might have but B itself 485 # we're interested in later tasks B's fnid might have but B itself
486 # doesn't depend on 486 # doesn't depend on
487 # 487 #
488 # Algorithm is O(tasks) + O(tasks)*O(fnids) 488 # Algorithm is O(tasks) + O(tasks)*O(fnids)
@@ -513,7 +513,7 @@ class RunQueue:
513 if len(runq_recrdepends[task]) > 0: 513 if len(runq_recrdepends[task]) > 0:
514 taskfnid = self.runq_fnid[task] 514 taskfnid = self.runq_fnid[task]
515 for dep in reccumdepends[taskfnid]: 515 for dep in reccumdepends[taskfnid]:
516 # Ignore self references 516 # Ignore self references
517 if dep == task: 517 if dep == task:
518 continue 518 continue
519 for taskname in runq_recrdepends[task]: 519 for taskname in runq_recrdepends[task]:
@@ -635,7 +635,7 @@ class RunQueue:
635 635
636 bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints)) 636 bb.msg.note(2, bb.msg.domain.RunQueue, "Compute totals (have %s endpoint(s))" % len(endpoints))
637 637
638 # Calculate task weights 638 # Calculate task weights
639 # Check of higher length circular dependencies 639 # Check of higher length circular dependencies
640 self.runq_weight = self.calculate_task_weights(endpoints) 640 self.runq_weight = self.calculate_task_weights(endpoints)
641 641
@@ -657,7 +657,7 @@ class RunQueue:
657 for prov in self.dataCache.fn_provides[fn]: 657 for prov in self.dataCache.fn_provides[fn]:
658 if prov not in prov_list: 658 if prov not in prov_list:
659 prov_list[prov] = [fn] 659 prov_list[prov] = [fn]
660 elif fn not in prov_list[prov]: 660 elif fn not in prov_list[prov]:
661 prov_list[prov].append(fn) 661 prov_list[prov].append(fn)
662 error = False 662 error = False
663 for prov in prov_list: 663 for prov in prov_list:
@@ -703,7 +703,7 @@ class RunQueue:
703 buildable.append(task) 703 buildable.append(task)
704 704
705 def check_buildable(self, task, buildable): 705 def check_buildable(self, task, buildable):
706 for revdep in self.runq_revdeps[task]: 706 for revdep in self.runq_revdeps[task]:
707 alldeps = 1 707 alldeps = 1
708 for dep in self.runq_depends[revdep]: 708 for dep in self.runq_depends[revdep]:
709 if dep in unchecked: 709 if dep in unchecked:
@@ -811,10 +811,10 @@ class RunQueue:
811 try: 811 try:
812 t2 = os.stat(stampfile2)[stat.ST_MTIME] 812 t2 = os.stat(stampfile2)[stat.ST_MTIME]
813 if t1 < t2: 813 if t1 < t2:
814 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile,stampfile2)) 814 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile, stampfile2))
815 iscurrent = False 815 iscurrent = False
816 except: 816 except:
817 bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2 ,stampfile)) 817 bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2 , stampfile))
818 iscurrent = False 818 iscurrent = False
819 819
820 return iscurrent 820 return iscurrent
@@ -885,7 +885,7 @@ class RunQueue:
885 def task_complete(self, task): 885 def task_complete(self, task):
886 """ 886 """
887 Mark a task as completed 887 Mark a task as completed
888 Look at the reverse dependencies and mark any task with 888 Look at the reverse dependencies and mark any task with
889 completed dependencies as buildable 889 completed dependencies as buildable
890 """ 890 """
891 self.runq_complete[task] = 1 891 self.runq_complete[task] = 1
@@ -1033,10 +1033,10 @@ class RunQueue:
1033 def finish_runqueue_now(self): 1033 def finish_runqueue_now(self):
1034 bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active) 1034 bb.msg.note(1, bb.msg.domain.RunQueue, "Sending SIGINT to remaining %s tasks" % self.stats.active)
1035 for k, v in self.build_pids.iteritems(): 1035 for k, v in self.build_pids.iteritems():
1036 try: 1036 try:
1037 os.kill(-k, signal.SIGINT) 1037 os.kill(-k, signal.SIGINT)
1038 except: 1038 except:
1039 pass 1039 pass
1040 for pipe in self.build_pipes: 1040 for pipe in self.build_pipes:
1041 self.build_pipes[pipe].read() 1041 self.build_pipes[pipe].read()
1042 1042
@@ -1085,30 +1085,30 @@ class RunQueue:
1085 """ 1085 """
1086 bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:") 1086 bb.msg.debug(3, bb.msg.domain.RunQueue, "run_tasks:")
1087 for task in range(len(self.runq_task)): 1087 for task in range(len(self.runq_task)):
1088 bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, 1088 bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task,
1089 taskQueue.fn_index[self.runq_fnid[task]], 1089 taskQueue.fn_index[self.runq_fnid[task]],
1090 self.runq_task[task], 1090 self.runq_task[task],
1091 self.runq_weight[task], 1091 self.runq_weight[task],
1092 self.runq_depends[task], 1092 self.runq_depends[task],
1093 self.runq_revdeps[task])) 1093 self.runq_revdeps[task]))
1094 1094
1095 bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:") 1095 bb.msg.debug(3, bb.msg.domain.RunQueue, "sorted_tasks:")
1096 for task1 in range(len(self.runq_task)): 1096 for task1 in range(len(self.runq_task)):
1097 if task1 in self.prio_map: 1097 if task1 in self.prio_map:
1098 task = self.prio_map[task1] 1098 task = self.prio_map[task1]
1099 bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task, 1099 bb.msg.debug(3, bb.msg.domain.RunQueue, " (%s)%s - %s: %s Deps %s RevDeps %s" % (task,
1100 taskQueue.fn_index[self.runq_fnid[task]], 1100 taskQueue.fn_index[self.runq_fnid[task]],
1101 self.runq_task[task], 1101 self.runq_task[task],
1102 self.runq_weight[task], 1102 self.runq_weight[task],
1103 self.runq_depends[task], 1103 self.runq_depends[task],
1104 self.runq_revdeps[task])) 1104 self.runq_revdeps[task]))
1105 1105
1106 1106
1107class TaskFailure(Exception): 1107class TaskFailure(Exception):
1108 """ 1108 """
1109 Exception raised when a task in a runqueue fails 1109 Exception raised when a task in a runqueue fails
1110 """ 1110 """
1111 def __init__(self, x): 1111 def __init__(self, x):
1112 self.args = x 1112 self.args = x
1113 1113
1114 1114
@@ -1196,4 +1196,3 @@ class runQueuePipe():
1196 if len(self.queue) > 0: 1196 if len(self.queue) > 0:
1197 print "Warning, worker left partial message" 1197 print "Warning, worker left partial message"
1198 os.close(self.fd) 1198 os.close(self.fd)
1199