summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/runqueue.py
diff options
context:
space:
mode:
authorRichard Purdie <richard@openedhand.com>2008-03-03 22:01:45 +0000
committerRichard Purdie <richard@openedhand.com>2008-03-03 22:01:45 +0000
commitab191d21e2e5e1609206146d238af6ec0b3f0554 (patch)
tree728fa74dbf00f6b11964aa53b8427a0d221d6e91 /bitbake/lib/bb/runqueue.py
parente88b4753781d54dc2625c3260c611d30ad76dbed (diff)
downloadpoky-ab191d21e2e5e1609206146d238af6ec0b3f0554.tar.gz
bitbake: Update to bitbake 1.8 branch head
git-svn-id: https://svn.o-hand.com/repos/poky/trunk@3892 311d38ba-8fff-0310-9ca6-ca027cbcb966
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r--bitbake/lib/bb/runqueue.py142
1 files changed, 116 insertions, 26 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 9d72d92fac..2765343a3e 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -26,6 +26,7 @@ from bb import msg, data, event, mkdirhier, utils
26from sets import Set 26from sets import Set
27import bb, os, sys 27import bb, os, sys
28import signal 28import signal
29import stat
29 30
30class TaskFailure(Exception): 31class TaskFailure(Exception):
31 """Exception raised when a task in a runqueue fails""" 32 """Exception raised when a task in a runqueue fails"""
@@ -45,11 +46,11 @@ class RunQueueStats:
45 def taskFailed(self): 46 def taskFailed(self):
46 self.failed = self.failed + 1 47 self.failed = self.failed + 1
47 48
48 def taskCompleted(self): 49 def taskCompleted(self, number = 1):
49 self.completed = self.completed + 1 50 self.completed = self.completed + number
50 51
51 def taskSkipped(self): 52 def taskSkipped(self, number = 1):
52 self.skipped = self.skipped + 1 53 self.skipped = self.skipped + number
53 54
54class RunQueueScheduler: 55class RunQueueScheduler:
55 """ 56 """
@@ -144,8 +145,11 @@ class RunQueue:
144 self.taskData = taskData 145 self.taskData = taskData
145 self.targets = targets 146 self.targets = targets
146 147
147 self.number_tasks = int(bb.data.getVar("BB_NUMBER_THREADS", cfgData) or 1) 148 self.cfgdata = cfgData
148 self.multi_provider_whitelist = (bb.data.getVar("MULTI_PROVIDER_WHITELIST", cfgData) or "").split() 149 self.number_tasks = int(bb.data.getVar("BB_NUMBER_THREADS", cfgData, 1) or 1)
150 self.multi_provider_whitelist = (bb.data.getVar("MULTI_PROVIDER_WHITELIST", cfgData, 1) or "").split()
151 self.scheduler = bb.data.getVar("BB_SCHEDULER", cfgData, 1) or "speed"
152 self.stamppolicy = bb.data.getVar("BB_STAMP_POLICY", cfgData, 1) or "perfile"
149 153
150 def reset_runqueue(self): 154 def reset_runqueue(self):
151 155
@@ -512,6 +516,7 @@ class RunQueue:
512 for depend in depends: 516 for depend in depends:
513 mark_active(depend, depth+1) 517 mark_active(depend, depth+1)
514 518
519 self.target_pairs = []
515 for target in self.targets: 520 for target in self.targets:
516 targetid = taskData.getbuild_id(target[0]) 521 targetid = taskData.getbuild_id(target[0])
517 522
@@ -522,10 +527,11 @@ class RunQueue:
522 continue 527 continue
523 528
524 fnid = taskData.build_targets[targetid][0] 529 fnid = taskData.build_targets[targetid][0]
530 fn = taskData.fn_index[fnid]
531 self.target_pairs.append((fn, target[1]))
525 532
526 # Remove stamps for targets if force mode active 533 # Remove stamps for targets if force mode active
527 if self.cooker.configuration.force: 534 if self.cooker.configuration.force:
528 fn = taskData.fn_index[fnid]
529 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (target[1], fn)) 535 bb.msg.note(2, bb.msg.domain.RunQueue, "Remove stamp %s, %s" % (target[1], fn))
530 bb.build.del_stamp(target[1], self.dataCache, fn) 536 bb.build.del_stamp(target[1], self.dataCache, fn)
531 537
@@ -608,10 +614,11 @@ class RunQueue:
608 self.runq_weight = self.calculate_task_weights(endpoints) 614 self.runq_weight = self.calculate_task_weights(endpoints)
609 615
610 # Decide what order to execute the tasks in, pick a scheduler 616 # Decide what order to execute the tasks in, pick a scheduler
611 # FIXME - Allow user selection
612 #self.sched = RunQueueScheduler(self) 617 #self.sched = RunQueueScheduler(self)
613 self.sched = RunQueueSchedulerSpeed(self) 618 if self.scheduler == "completion":
614 #self.sched = RunQueueSchedulerCompletion(self) 619 self.sched = RunQueueSchedulerCompletion(self)
620 else:
621 self.sched = RunQueueSchedulerSpeed(self)
615 622
616 # Sanity Check - Check for multiple tasks building the same provider 623 # Sanity Check - Check for multiple tasks building the same provider
617 prov_list = {} 624 prov_list = {}
@@ -636,6 +643,93 @@ class RunQueue:
636 643
637 #self.dump_data(taskData) 644 #self.dump_data(taskData)
638 645
646 def check_stamps(self):
647 unchecked = {}
648 current = []
649 notcurrent = []
650 buildable = []
651
652 if self.stamppolicy == "perfile":
653 fulldeptree = False
654 else:
655 fulldeptree = True
656
657 for task in range(len(self.runq_fnid)):
658 unchecked[task] = ""
659 if len(self.runq_depends[task]) == 0:
660 buildable.append(task)
661
662 for task in range(len(self.runq_fnid)):
663 if task not in unchecked:
664 continue
665 fn = self.taskData.fn_index[self.runq_fnid[task]]
666 taskname = self.runq_task[task]
667 stampfile = "%s.%s" % (self.dataCache.stamp[fn], taskname)
668 # If the stamp is missing its not current
669 if not os.access(stampfile, os.F_OK):
670 del unchecked[task]
671 notcurrent.append(task)
672 continue
673 # If its a 'nostamp' task, it's not current
674 taskdep = self.dataCache.task_deps[fn]
675 if 'nostamp' in taskdep and task in taskdep['nostamp']:
676 del unchecked[task]
677 notcurrent.append(task)
678 continue
679
680 while (len(buildable) > 0):
681 nextbuildable = []
682 for task in buildable:
683 if task in unchecked:
684 fn = self.taskData.fn_index[self.runq_fnid[task]]
685 taskname = self.runq_task[task]
686 stampfile = "%s.%s" % (self.dataCache.stamp[fn], taskname)
687 iscurrent = True
688
689 t1 = os.stat(stampfile)[stat.ST_MTIME]
690 for dep in self.runq_depends[task]:
691 if iscurrent:
692 fn2 = self.taskData.fn_index[self.runq_fnid[dep]]
693 taskname2 = self.runq_task[dep]
694 stampfile2 = "%s.%s" % (self.dataCache.stamp[fn2], taskname2)
695 if fulldeptree or fn == fn2:
696 if dep in notcurrent:
697 iscurrent = False
698 else:
699 t2 = os.stat(stampfile2)[stat.ST_MTIME]
700 if t1 < t2:
701 iscurrent = False
702 del unchecked[task]
703 if iscurrent:
704 current.append(task)
705 else:
706 notcurrent.append(task)
707
708 for revdep in self.runq_revdeps[task]:
709 alldeps = 1
710 for dep in self.runq_depends[revdep]:
711 if dep in unchecked:
712 alldeps = 0
713 if alldeps == 1:
714 if revdep in unchecked:
715 nextbuildable.append(revdep)
716
717 buildable = nextbuildable
718
719 #for task in range(len(self.runq_fnid)):
720 # fn = self.taskData.fn_index[self.runq_fnid[task]]
721 # taskname = self.runq_task[task]
722 # print "%s %s.%s" % (task, taskname, fn)
723
724 #print "Unchecked: %s" % unchecked
725 #print "Current: %s" % current
726 #print "Not current: %s" % notcurrent
727
728 if len(unchecked) > 0:
729 bb.fatal("check_stamps fatal internal error")
730 return current
731
732
639 def execute_runqueue(self): 733 def execute_runqueue(self):
640 """ 734 """
641 Run the tasks in a queue prepared by prepare_runqueue 735 Run the tasks in a queue prepared by prepare_runqueue
@@ -721,18 +815,13 @@ class RunQueue:
721 def sigint_handler(signum, frame): 815 def sigint_handler(signum, frame):
722 raise KeyboardInterrupt 816 raise KeyboardInterrupt
723 817
724 # RP - this code allows tasks to run out of the correct order - disabled, FIXME 818 event.fire(bb.event.StampUpdate(self.target_pairs, self.dataCache.stamp, self.cfgdata))
725 # Find any tasks with current stamps and remove them from the queue 819
726 #for task1 in range(len(self.runq_fnid)): 820 # Find out which tasks have current stamps which we can skip when the
727 # task = self.prio_map[task1] 821 # time comes
728 # fn = self.taskData.fn_index[self.runq_fnid[task]] 822 currentstamps = self.check_stamps()
729 # taskname = self.runq_task[task] 823 self.stats.taskSkipped(len(currentstamps))
730 # if bb.build.stamp_is_current(taskname, self.dataCache, fn): 824 self.stats.taskCompleted(len(currentstamps))
731 # bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task)))
732 # self.runq_running[task] = 1
733 # self.task_complete(task)
734 # self.stats.taskCompleted()
735 # self.stats.taskSkipped()
736 825
737 while True: 826 while True:
738 task = self.sched.next() 827 task = self.sched.next()
@@ -740,12 +829,13 @@ class RunQueue:
740 fn = self.taskData.fn_index[self.runq_fnid[task]] 829 fn = self.taskData.fn_index[self.runq_fnid[task]]
741 830
742 taskname = self.runq_task[task] 831 taskname = self.runq_task[task]
743 if bb.build.stamp_is_current(taskname, self.dataCache, fn): 832 if task in currentstamps:
833 #if bb.build.stamp_is_current(taskname, self.dataCache, fn):
744 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task))) 834 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp current task %s (%s)" % (task, self.get_user_idstring(task)))
745 self.runq_running[task] = 1 835 self.runq_running[task] = 1
746 self.task_complete(task) 836 self.task_complete(task)
747 self.stats.taskCompleted() 837 #self.stats.taskCompleted()
748 self.stats.taskSkipped() 838 #self.stats.taskSkipped()
749 continue 839 continue
750 840
751 bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task))) 841 bb.msg.note(1, bb.msg.domain.RunQueue, "Running task %d of %d (ID: %s, %s)" % (self.stats.completed + self.active_builds + 1, len(self.runq_fnid), task, self.get_user_idstring(task)))
@@ -764,7 +854,7 @@ class RunQueue:
764 os.dup2(newsi, sys.stdin.fileno()) 854 os.dup2(newsi, sys.stdin.fileno())
765 self.cooker.configuration.cmd = taskname[3:] 855 self.cooker.configuration.cmd = taskname[3:]
766 try: 856 try:
767 self.cooker.tryBuild(fn, False) 857 self.cooker.tryBuild(fn)
768 except bb.build.EventException: 858 except bb.build.EventException:
769 bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed") 859 bb.msg.error(bb.msg.domain.Build, "Build of " + fn + " " + taskname + " failed")
770 sys.exit(1) 860 sys.exit(1)