summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2012-11-16 15:30:52 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-11-20 15:31:56 +0000
commit328f74a5565856919a3a906fbd0d396d5f88d896 (patch)
treea14c81ec38e9531928dfe269ed5f43dd702e2cdb
parent5de7744a49bc0523fd65356638457bc51e1b262f (diff)
downloadpoky-328f74a5565856919a3a906fbd0d396d5f88d896.tar.gz
bitbake: runqueue: Allow partial setscene task coverage
When the setscene code was originally written it was thought that we'd allow "partial" coverage. For example, if we just want to build the target "bash:do_populate_sysroot" and its available from sstate, it makes no sense to install gcc-cross's sstate package as its simply not needed. Due to various other issues in the codebase, this functionality was disabled/removed to allow the setscene code and sstate to stabilise and allow us to concentrate on other problems. The time has now come to enable "partial" coverage. There are two major changes in this patch: a) Creation of an unskippable list. This lists direct dependencies of build targets and hence things that cannot be skipped. b) Addition of a handler which looks at a given setscene target and what depends on it and then decides whether its necessary. (Bitbake rev: 2a937cd6a6c3110030b40bc4d85e349b85cb4db7) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/runqueue.py83
1 files changed, 72 insertions, 11 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 75797e2520..0c7dfec2b6 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -785,6 +785,7 @@ class RunQueue:
785 self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile" 785 self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile"
786 self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None 786 self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None
787 self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION", True) or None 787 self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION", True) or None
788 self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None
788 789
789 self.state = runQueuePrepare 790 self.state = runQueuePrepare
790 791
@@ -1161,6 +1162,26 @@ class RunQueueExecute:
1161 1162
1162 return pid, pipein, pipeout 1163 return pid, pipein, pipeout
1163 1164
1165 def check_dependencies(self, task, taskdeps, setscene = False):
1166 if not self.rq.depvalidate:
1167 return False
1168
1169 taskdata = {}
1170 taskdeps.add(task)
1171 for dep in taskdeps:
1172 if setscene:
1173 depid = self.rqdata.runq_setscene[dep]
1174 else:
1175 depid = dep
1176 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[depid]]
1177 pn = self.rqdata.dataCache.pkg_fn[fn]
1178 taskname = self.rqdata.runq_task[depid]
1179 taskdata[dep] = [pn, taskname, fn]
1180 call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
1181 locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" : self.cooker.configuration.data }
1182 valid = bb.utils.better_eval(call, locs)
1183 return valid
1184
1164class RunQueueExecuteDummy(RunQueueExecute): 1185class RunQueueExecuteDummy(RunQueueExecute):
1165 def __init__(self, rq): 1186 def __init__(self, rq):
1166 self.rq = rq 1187 self.rq = rq
@@ -1198,16 +1219,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
1198 logger.debug(1, 'Considering %s (%s): %s' % (task, self.rqdata.get_user_idstring(task), str(self.rqdata.runq_revdeps[task]))) 1219 logger.debug(1, 'Considering %s (%s): %s' % (task, self.rqdata.get_user_idstring(task), str(self.rqdata.runq_revdeps[task])))
1199 1220
1200 if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered) and task not in self.rq.scenequeue_notcovered: 1221 if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered) and task not in self.rq.scenequeue_notcovered:
1201 ok = True 1222 found = True
1202 for revdep in self.rqdata.runq_revdeps[task]: 1223 self.rq.scenequeue_covered.add(task)
1203 if self.rqdata.runq_fnid[task] != self.rqdata.runq_fnid[revdep]:
1204 logger.debug(1, 'Found "bad" dep %s (%s) for %s (%s)' % (revdep, self.rqdata.get_user_idstring(revdep), task, self.rqdata.get_user_idstring(task)))
1205
1206 ok = False
1207 break
1208 if ok:
1209 found = True
1210 self.rq.scenequeue_covered.add(task)
1211 1224
1212 logger.debug(1, 'Skip list (pre setsceneverify) %s', sorted(self.rq.scenequeue_covered)) 1225 logger.debug(1, 'Skip list (pre setsceneverify) %s', sorted(self.rq.scenequeue_covered))
1213 1226
@@ -1408,6 +1421,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1408 1421
1409 self.scenequeue_covered = set() 1422 self.scenequeue_covered = set()
1410 self.scenequeue_notcovered = set() 1423 self.scenequeue_notcovered = set()
1424 self.scenequeue_notneeded = set()
1411 1425
1412 # If we don't have any setscene functions, skip this step 1426 # If we don't have any setscene functions, skip this step
1413 if len(self.rqdata.runq_setscene) == 0: 1427 if len(self.rqdata.runq_setscene) == 0:
@@ -1417,7 +1431,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1417 1431
1418 self.stats = RunQueueStats(len(self.rqdata.runq_setscene)) 1432 self.stats = RunQueueStats(len(self.rqdata.runq_setscene))
1419 1433
1420 endpoints = {}
1421 sq_revdeps = [] 1434 sq_revdeps = []
1422 sq_revdeps_new = [] 1435 sq_revdeps_new = []
1423 sq_revdeps_squash = [] 1436 sq_revdeps_squash = []
@@ -1432,12 +1445,15 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1432 self.runq_complete.append(0) 1445 self.runq_complete.append(0)
1433 self.runq_buildable.append(0) 1446 self.runq_buildable.append(0)
1434 1447
1448 # First process the chains up to the first setscene task.
1449 endpoints = {}
1435 for task in xrange(len(self.rqdata.runq_fnid)): 1450 for task in xrange(len(self.rqdata.runq_fnid)):
1436 sq_revdeps.append(copy.copy(self.rqdata.runq_revdeps[task])) 1451 sq_revdeps.append(copy.copy(self.rqdata.runq_revdeps[task]))
1437 sq_revdeps_new.append(set()) 1452 sq_revdeps_new.append(set())
1438 if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene: 1453 if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene:
1439 endpoints[task] = set() 1454 endpoints[task] = set()
1440 1455
1456 # Secondly process the chains between setscene tasks.
1441 for task in self.rqdata.runq_setscene: 1457 for task in self.rqdata.runq_setscene:
1442 for dep in self.rqdata.runq_depends[task]: 1458 for dep in self.rqdata.runq_depends[task]:
1443 if dep not in endpoints: 1459 if dep not in endpoints:
@@ -1453,6 +1469,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1453 if sq_revdeps_new[point]: 1469 if sq_revdeps_new[point]:
1454 tasks |= sq_revdeps_new[point] 1470 tasks |= sq_revdeps_new[point]
1455 sq_revdeps_new[point] = set() 1471 sq_revdeps_new[point] = set()
1472 if point in self.rqdata.runq_setscene:
1473 sq_revdeps_new[point] = tasks
1456 for dep in self.rqdata.runq_depends[point]: 1474 for dep in self.rqdata.runq_depends[point]:
1457 if point in sq_revdeps[dep]: 1475 if point in sq_revdeps[dep]:
1458 sq_revdeps[dep].remove(point) 1476 sq_revdeps[dep].remove(point)
@@ -1465,6 +1483,42 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1465 1483
1466 process_endpoints(endpoints) 1484 process_endpoints(endpoints)
1467 1485
1486 # Build a list of setscene tasks which as "unskippable"
1487 # These are direct endpoints referenced by the build
1488 endpoints2 = {}
1489 sq_revdeps2 = []
1490 sq_revdeps_new2 = []
1491 def process_endpoints2(endpoints):
1492 newendpoints = {}
1493 for point, task in endpoints.items():
1494 tasks = set([point])
1495 if task:
1496 tasks |= task
1497 if sq_revdeps_new2[point]:
1498 tasks |= sq_revdeps_new2[point]
1499 sq_revdeps_new2[point] = set()
1500 if point in self.rqdata.runq_setscene:
1501 sq_revdeps_new2[point] = tasks
1502 for dep in self.rqdata.runq_depends[point]:
1503 if point in sq_revdeps2[dep]:
1504 sq_revdeps2[dep].remove(point)
1505 if tasks:
1506 sq_revdeps_new2[dep] |= tasks
1507 if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in self.rqdata.runq_setscene:
1508 newendpoints[dep] = tasks
1509 if len(newendpoints) != 0:
1510 process_endpoints2(newendpoints)
1511 for task in xrange(len(self.rqdata.runq_fnid)):
1512 sq_revdeps2.append(copy.copy(self.rqdata.runq_revdeps[task]))
1513 sq_revdeps_new2.append(set())
1514 if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene:
1515 endpoints2[task] = set()
1516 process_endpoints2(endpoints2)
1517 self.unskippable = []
1518 for task in self.rqdata.runq_setscene:
1519 if sq_revdeps_new2[task]:
1520 self.unskippable.append(self.rqdata.runq_setscene.index(task))
1521
1468 for task in xrange(len(self.rqdata.runq_fnid)): 1522 for task in xrange(len(self.rqdata.runq_fnid)):
1469 if task in self.rqdata.runq_setscene: 1523 if task in self.rqdata.runq_setscene:
1470 deps = set() 1524 deps = set()
@@ -1625,6 +1679,13 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1625 # Find the next setscene to run 1679 # Find the next setscene to run
1626 for nexttask in xrange(self.stats.total): 1680 for nexttask in xrange(self.stats.total):
1627 if self.runq_buildable[nexttask] == 1 and self.runq_running[nexttask] != 1: 1681 if self.runq_buildable[nexttask] == 1 and self.runq_running[nexttask] != 1:
1682 if nexttask in self.unskippable:
1683 logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask]))
1684 if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True):
1685 logger.debug(2, "Skipping setscene for task %s" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask]))
1686 self.task_skip(nexttask)
1687 self.scenequeue_notneeded.add(nexttask)
1688 return True
1628 task = nexttask 1689 task = nexttask
1629 break 1690 break
1630 if task is not None: 1691 if task is not None: