summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <rpurdie@linux.intel.com>2010-08-19 11:36:29 +0100
committerRichard Purdie <rpurdie@linux.intel.com>2010-08-19 20:06:25 +0100
commit787c1cf81195e97dda5fdab3a0a5d8bda97f6770 (patch)
treef6a3eb03758b2c9ad38f51ebe0f1f11115122c51
parent41a364f59f70f3c643924eff67ca63ab667b63ae (diff)
downloadpoky-787c1cf81195e97dda5fdab3a0a5d8bda97f6770.tar.gz
bitbake: Initial scenequeue implementation (needs major fixes)
bitbake: scenequeue: Skip setscene if the underlying task already ran bitbake/setscene: Make sure uneeded dependencies are removed recursively Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
-rw-r--r--bitbake/lib/bb/runqueue.py286
1 files changed, 262 insertions, 24 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 488aa04d06..9127f248d5 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -27,6 +27,7 @@ from bb import msg, data, event
27import signal 27import signal
28import stat 28import stat
29import fcntl 29import fcntl
30import copy
30 31
31class RunQueueStats: 32class RunQueueStats:
32 """ 33 """
@@ -57,12 +58,14 @@ class RunQueueStats:
57# These values indicate the next step due to be run in the 58# These values indicate the next step due to be run in the
58# runQueue state machine 59# runQueue state machine
59runQueuePrepare = 2 60runQueuePrepare = 2
60runQueueRunInit = 3 61runQueueSceneInit = 3
61runQueueRunning = 4 62runQueueSceneRun = 4
62runQueueFailed = 6 63runQueueRunInit = 5
63runQueueCleanUp = 7 64runQueueRunning = 6
64runQueueComplete = 8 65runQueueFailed = 7
65runQueueChildProcess = 9 66runQueueCleanUp = 8
67runQueueComplete = 9
68runQueueChildProcess = 10
66 69
67class RunQueueScheduler(object): 70class RunQueueScheduler(object):
68 """ 71 """
@@ -672,6 +675,16 @@ class RunQueueData:
672 675
673 #self.dump_data(taskData) 676 #self.dump_data(taskData)
674 677
678 # Interate over the task list looking for tasks with a 'setscene' function
679
680 self.runq_setscene = []
681 for task in range(len(self.runq_fnid)):
682 setscene = taskData.gettask_id(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task] + "_setscene", False)
683 if not setscene:
684 continue
685 bb.note("Found setscene for %s %s" % (self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task]))
686 self.runq_setscene.append(task)
687
675 def dump_data(self, taskQueue): 688 def dump_data(self, taskQueue):
676 """ 689 """
677 Dump some debug information on the internal data structures 690 Dump some debug information on the internal data structures
@@ -802,6 +815,13 @@ class RunQueue:
802 return current 815 return current
803 816
804 def check_stamp_task(self, task, taskname = None): 817 def check_stamp_task(self, task, taskname = None):
818 def get_timestamp(f):
819 try:
820 if not os.access(f, os.F_OK):
821 return None
822 return os.stat(f)[stat.ST_MTIME]
823 except:
824 return None
805 825
806 if self.stamppolicy == "perfile": 826 if self.stamppolicy == "perfile":
807 fulldeptree = False 827 fulldeptree = False
@@ -825,23 +845,24 @@ class RunQueue:
825 bb.msg.debug(2, bb.msg.domain.RunQueue, "%s.%s is nostamp\n" % (fn, taskname)) 845 bb.msg.debug(2, bb.msg.domain.RunQueue, "%s.%s is nostamp\n" % (fn, taskname))
826 return False 846 return False
827 847
848 if taskname.endswith("_setscene"):
849 return True
850
828 iscurrent = True 851 iscurrent = True
829 t1 = os.stat(stampfile)[stat.ST_MTIME] 852 t1 = get_timestamp(stampfile)
830 for dep in self.rqdata.runq_depends[task]: 853 for dep in self.rqdata.runq_depends[task]:
831 if iscurrent: 854 if iscurrent:
832 fn2 = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[dep]] 855 fn2 = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[dep]]
833 taskname2 = self.rqdata.runq_task[dep] 856 taskname2 = self.rqdata.runq_task[dep]
834 stampfile2 = "%s.%s" % (self.rqdata.dataCache.stamp[fn2], taskname2) 857 stampfile2 = "%s.%s" % (self.rqdata.dataCache.stamp[fn2], taskname2)
858 t2 = get_timestamp(stampfile2)
859 t3 = get_timestamp(stampfile2 + "_setscene")
860 if t3 and t3 > t2:
861 continue
835 if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist): 862 if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist):
836 try: 863 if not t2 or t1 < t2:
837 t2 = os.stat(stampfile2)[stat.ST_MTIME] 864 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s (or does not exist)" % (stampfile, stampfile2))
838 if t1 < t2:
839 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stampfile %s < %s" % (stampfile, stampfile2))
840 iscurrent = False
841 except:
842 bb.msg.debug(2, bb.msg.domain.RunQueue, "Exception reading %s for %s" % (stampfile2, stampfile))
843 iscurrent = False 865 iscurrent = False
844
845 return iscurrent 866 return iscurrent
846 867
847 def execute_runqueue(self): 868 def execute_runqueue(self):
@@ -855,7 +876,13 @@ class RunQueue:
855 876
856 if self.state is runQueuePrepare: 877 if self.state is runQueuePrepare:
857 self.rqdata.prepare() 878 self.rqdata.prepare()
858 self.state = runQueueRunInit 879 self.state = runQueueSceneInit
880
881 if self.state is runQueueSceneInit:
882 self.rqexe = RunQueueExecuteScenequeue(self)
883
884 if self.state is runQueueSceneRun:
885 self.rqexe.execute()
859 886
860 if self.state is runQueueRunInit: 887 if self.state is runQueueRunInit:
861 bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue") 888 bb.msg.note(1, bb.msg.domain.RunQueue, "Executing runqueue")
@@ -948,14 +975,10 @@ class RunQueueExecute:
948 for pipe in self.build_pipes: 975 for pipe in self.build_pipes:
949 self.build_pipes[pipe].read() 976 self.build_pipes[pipe].read()
950 977
951 try: 978 if self.stats.active > 0:
952 while self.stats.active > 0: 979 bb.event.fire(runQueueExitWait(self.stats.active), self.cfgData)
953 bb.event.fire(runQueueExitWait(self.stats.active), self.cfgData) 980 self.runqueue_process_waitpid()
954 if self.runqueue_process_waitpid() is None: 981 return
955 return
956 except:
957 self.finish_now()
958 raise
959 982
960 if len(self.failed_fnids) != 0: 983 if len(self.failed_fnids) != 0:
961 self.rq.state = runQueueFailed 984 self.rq.state = runQueueFailed
@@ -1034,6 +1057,23 @@ class RunQueueExecuteTasks(RunQueueExecute):
1034 self.runq_buildable.append(1) 1057 self.runq_buildable.append(1)
1035 else: 1058 else:
1036 self.runq_buildable.append(0) 1059 self.runq_buildable.append(0)
1060 if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered):
1061 self.rq.scenequeue_covered.add(task)
1062
1063 found = True
1064 while found:
1065 found = False
1066 for task in range(self.stats.total):
1067 if task in self.rq.scenequeue_covered:
1068 continue
1069 if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered):
1070 self.rq.scenequeue_covered.add(task)
1071 found = True
1072
1073 bb.note("Full skip list %s" % self.rq.scenequeue_covered)
1074
1075 for task in self.rq.scenequeue_covered:
1076 self.task_skip(task)
1037 1077
1038 event.fire(bb.event.StampUpdate(self.rqdata.target_pairs, self.rqdata.dataCache.stamp), self.cfgData) 1078 event.fire(bb.event.StampUpdate(self.rqdata.target_pairs, self.rqdata.dataCache.stamp), self.cfgData)
1039 1079
@@ -1145,6 +1185,204 @@ class RunQueueExecuteTasks(RunQueueExecute):
1145 self.rq.state = runQueueComplete 1185 self.rq.state = runQueueComplete
1146 return 1186 return
1147 1187
1188class RunQueueExecuteScenequeue(RunQueueExecute):
1189 def __init__(self, rq):
1190 RunQueueExecute.__init__(self, rq)
1191
1192 self.scenequeue_covered = set()
1193 self.scenequeue_notcovered = set()
1194
1195 # If we don't have any setscene functions, skip this step
1196 if len(self.rqdata.runq_setscene) == 0:
1197 rq.scenequeue_covered = set()
1198 rq.state = runQueueRunInit
1199 return
1200
1201 self.stats = RunQueueStats(len(self.rqdata.runq_setscene))
1202
1203 endpoints = {}
1204 sq_revdeps = []
1205 sq_revdeps_new = []
1206 sq_revdeps_squash = []
1207
1208 # We need to construct a dependency graph for the setscene functions. Intermediate
1209 # dependencies between the setscene tasks only complicate the code. This code
1210 # therefore aims to collapse the huge runqueue dependency tree into a smaller one
1211 # only containing the setscene functions.
1212
1213 for task in range(self.stats.total):
1214 self.runq_running.append(0)
1215 self.runq_complete.append(0)
1216 self.runq_buildable.append(0)
1217
1218 for task in range(len(self.rqdata.runq_fnid)):
1219 sq_revdeps.append(copy.copy(self.rqdata.runq_revdeps[task]))
1220 sq_revdeps_new.append(set())
1221 if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene:
1222 endpoints[task] = None
1223
1224 for task in self.rqdata.runq_setscene:
1225 for dep in self.rqdata.runq_depends[task]:
1226 endpoints[dep] = task
1227
1228 def process_endpoints(endpoints):
1229 newendpoints = {}
1230 for point, task in endpoints.items():
1231 tasks = set()
1232 if task:
1233 tasks.add(task)
1234 if sq_revdeps_new[point]:
1235 tasks |= sq_revdeps_new[point]
1236 sq_revdeps_new[point] = set()
1237 for dep in self.rqdata.runq_depends[point]:
1238 if point in sq_revdeps[dep]:
1239 sq_revdeps[dep].remove(point)
1240 if tasks:
1241 sq_revdeps_new[dep] |= tasks
1242 if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in self.rqdata.runq_setscene:
1243 newendpoints[dep] = task
1244 if len(newendpoints) != 0:
1245 process_endpoints(newendpoints)
1246
1247 process_endpoints(endpoints)
1248
1249 for task in range(len(self.rqdata.runq_fnid)):
1250 if task in self.rqdata.runq_setscene:
1251 deps = set()
1252 for dep in sq_revdeps_new[task]:
1253 deps.add(self.rqdata.runq_setscene.index(dep))
1254 sq_revdeps_squash.append(deps)
1255 elif len(sq_revdeps_new[task]) != 0:
1256 bb.msg.fatal(bb.msg.domain.RunQueue, "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
1257
1258 #for task in range(len(sq_revdeps_squash)):
1259 # print "Task %s: %s.%s is %s " % (task, self.taskData.fn_index[self.runq_fnid[self.runq_setscene[task]]], self.runq_task[self.runq_setscene[task]] + "_setscene", sq_revdeps_squash[task])
1260
1261 self.sq_deps = []
1262 self.sq_revdeps = sq_revdeps_squash
1263 self.sq_revdeps2 = copy.deepcopy(self.sq_revdeps)
1264
1265 for task in range(len(self.sq_revdeps)):
1266 self.sq_deps.append(set())
1267 for task in range(len(self.sq_revdeps)):
1268 for dep in self.sq_revdeps[task]:
1269 self.sq_deps[dep].add(task)
1270
1271 for task in range(len(self.sq_revdeps)):
1272 if len(self.sq_revdeps[task]) == 0:
1273 self.runq_buildable[task] = 1
1274
1275 bb.msg.note(1, bb.msg.domain.RunQueue, "Executing setscene Tasks")
1276
1277 self.rq.state = runQueueSceneRun
1278
1279 def scenequeue_updatecounters(self, task):
1280 for dep in self.sq_deps[task]:
1281 self.sq_revdeps2[dep].remove(task)
1282 if len(self.sq_revdeps2[dep]) == 0:
1283 self.runq_buildable[dep] = 1
1284
1285 def task_complete(self, task):
1286 """
1287 Mark a task as completed
1288 Look at the reverse dependencies and mark any task with
1289 completed dependencies as buildable
1290 """
1291
1292 index = self.rqdata.runq_setscene[task]
1293 bb.msg.note(1, bb.msg.domain.RunQueue, "Found task %s could be accelerated" % self.rqdata.get_user_idstring(index))
1294
1295 self.scenequeue_covered.add(task)
1296 self.scenequeue_updatecounters(task)
1297
1298 def task_fail(self, task, result):
1299 self.stats.taskFailed()
1300 index = self.rqdata.runq_setscene[task]
1301 bb.event.fire(runQueueTaskFailed(task, self.stats, self), self.cfgData)
1302 self.scenequeue_notcovered.add(task)
1303 self.scenequeue_updatecounters(task)
1304
1305 def task_failoutright(self, task):
1306 self.runq_running[task] = 1
1307 self.runq_buildable[task] = 1
1308 self.stats.taskCompleted()
1309 self.stats.taskSkipped()
1310 index = self.rqdata.runq_setscene[task]
1311 self.scenequeue_notcovered.add(task)
1312 self.scenequeue_updatecounters(task)
1313
1314 def task_skip(self, task):
1315 self.runq_running[task] = 1
1316 self.runq_buildable[task] = 1
1317 self.task_complete(task)
1318 self.stats.taskCompleted()
1319 self.stats.taskSkipped()
1320
1321 def execute(self):
1322 """
1323 Run the tasks in a queue prepared by prepare_runqueue
1324 """
1325
1326 task = None
1327 if self.stats.active < self.number_tasks:
1328 # Find the next setscene to run
1329 for nexttask in range(self.stats.total):
1330 if self.runq_buildable[nexttask] == 1 and self.runq_running[nexttask] != 1:
1331 #bb.note("Comparing %s to %s" % (self.sq_revdeps[nexttask], self.scenequeue_covered))
1332 #if len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered):
1333 # bb.note("Skipping task %s" % nexttask)
1334 # self.scenequeue_skip(nexttask)
1335 # return True
1336 task = nexttask
1337 break
1338 if task is not None:
1339 realtask = self.rqdata.runq_setscene[task]
1340 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]]
1341
1342 taskname = self.rqdata.runq_task[realtask] + "_setscene"
1343 if self.rq.check_stamp_task(realtask, self.rqdata.runq_task[realtask]):
1344 bb.msg.debug(2, bb.msg.domain.RunQueue, "Stamp for underlying task %s (%s) is current so skipping setscene varient" % (task, self.rqdata.get_user_idstring(task)))
1345 self.task_failoutright(task)
1346 return True
1347
1348 if self.cooker.configuration.force:
1349 for target in self.target_pairs:
1350 if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]:
1351 self.task_failoutright(task)
1352 return True
1353
1354 if self.rq.check_stamp_task(realtask, taskname):
1355 bb.msg.debug(2, bb.msg.domain.RunQueue, "Setscene stamp current task %s (%s) so skip it and its dependencies" % (task, self.rqdata.get_user_idstring(realtask)))
1356 self.task_skip(task)
1357 return True
1358
1359 pid, pipein, pipeout = self.fork_off_task(fn, realtask, taskname)
1360
1361 self.build_pids[pid] = task
1362 self.build_pipes[pid] = runQueuePipe(pipein, pipeout, self.cfgData)
1363 self.runq_running[task] = 1
1364 self.stats.taskActive()
1365 if self.stats.active < self.number_tasks:
1366 return True
1367
1368 for pipe in self.build_pipes:
1369 self.build_pipes[pipe].read()
1370
1371 if self.stats.active > 0:
1372 if self.runqueue_process_waitpid() is None:
1373 return True
1374 return True
1375
1376 # Convert scenequeue_covered task numbers into full taskgraph ids
1377 oldcovered = self.scenequeue_covered
1378 self.rq.scenequeue_covered = set()
1379 for task in oldcovered:
1380 self.rq.scenequeue_covered.add(self.rqdata.runq_setscene[task])
1381
1382 bb.note("We can skip tasks %s" % self.rq.scenequeue_covered)
1383
1384 self.rq.state = runQueueRunInit
1385 return True
1148 1386
1149 1387
1150class TaskFailure(Exception): 1388class TaskFailure(Exception):