summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-06-23 22:59:11 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-08 09:57:27 +0100
commitf3b62c1c2e47968084342d1fe2a9af596a3ba93c (patch)
treeef08f8f6d64ec42fc3fbe3ef172aa0ff72dc9833
parent76feac37ceb68a850394e51dce8e36f1ec2c895d (diff)
downloadpoky-f3b62c1c2e47968084342d1fe2a9af596a3ba93c.tar.gz
bitbake: runqueue: add ability to enforce that tasks are setscened
Add the ability to enter a mode where only a specified whitelist of tasks can be executed outright; everything else must be successfully provided in the form of a setscene task (or covered by a setscene task). Any setscene failure outside of the whitelist will cause the build to fail immediately instead of running the real task, and any real tasks that would execute outside of the whitelist cause an immediate build failure when it comes to executing the runqueue as well. The mode is enabled by setting BB_SETSCENE_ENFORCE="1", and the whitelist is specified through BB_SETSCENE_ENFORCE_WHITELIST, consisting of pn:taskname pairs. A single % character can be substituted for the pn value to match any target explicitly specified on the bitbake command line. Wildcards * and ? can also be used as per standard unix file name matching for both pn and taskname. Part of the implementation of [YOCTO #9367]. (Bitbake rev: 624722c067a7fdd0c0f5d8be611e1f6666ecc4a2) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/runqueue.py81
1 files changed, 80 insertions, 1 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index c9b6b847db..b62a28a2be 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -240,6 +240,7 @@ class RunQueueData:
240 240
241 self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or "" 241 self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
242 self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split() 242 self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
243 self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
243 244
244 self.reset() 245 self.reset()
245 246
@@ -1596,8 +1597,51 @@ class RunQueueExecuteTasks(RunQueueExecute):
1596 Run the tasks in a queue prepared by rqdata.prepare() 1597 Run the tasks in a queue prepared by rqdata.prepare()
1597 """ 1598 """
1598 1599
1599 self.rq.read_workers() 1600 if self.rqdata.setscenewhitelist:
1601 # Check tasks that are going to run against the whitelist
1602 def check_norun_task(tid, showerror=False):
1603 fn = fn_from_tid(tid)
1604 taskname = taskname_from_tid(tid)
1605 # Ignore covered tasks
1606 if tid in self.rq.scenequeue_covered:
1607 return False
1608 # Ignore stamped tasks
1609 if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache):
1610 return False
1611 # Ignore noexec tasks
1612 taskdep = self.rqdata.dataCache.task_deps[fn]
1613 if 'noexec' in taskdep and taskname in taskdep['noexec']:
1614 return False
1600 1615
1616 pn = self.rqdata.dataCache.pkg_fn[fn]
1617 if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
1618 if showerror:
1619 if tid in self.rqdata.runq_setscene_tids:
1620 logger.error('Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname))
1621 else:
1622 logger.error('Task %s.%s attempted to execute unexpectedly' % (pn, taskname))
1623 return True
1624 return False
1625 # Look to see if any tasks that we think shouldn't run are going to
1626 unexpected = False
1627 for tid in self.rqdata.runtaskentries:
1628 if check_norun_task(tid):
1629 unexpected = True
1630 break
1631 if unexpected:
1632 # Run through the tasks in the rough order they'd have executed and print errors
1633 # (since the order can be useful - usually missing sstate for the last few tasks
1634 # is the cause of the problem)
1635 task = self.sched.next()
1636 while task is not None:
1637 check_norun_task(task, showerror=True)
1638 self.task_skip(task, 'Setscene enforcement check')
1639 task = self.sched.next()
1640
1641 self.rq.state = runQueueCleanUp
1642 return True
1643
1644 self.rq.read_workers()
1601 1645
1602 if self.stats.total == 0: 1646 if self.stats.total == 0:
1603 # nothing to do 1647 # nothing to do
@@ -1940,6 +1984,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1940 self.scenequeue_covered.add(task) 1984 self.scenequeue_covered.add(task)
1941 self.scenequeue_updatecounters(task) 1985 self.scenequeue_updatecounters(task)
1942 1986
1987 def check_taskfail(self, task):
1988 if self.rqdata.setscenewhitelist:
1989 realtask = task.split('_setscene')[0]
1990 fn = fn_from_tid(realtask)
1991 taskname = taskname_from_tid(realtask)
1992 pn = self.rqdata.dataCache.pkg_fn[fn]
1993 if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
1994 logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
1995 self.rq.state = runQueueCleanUp
1996
1943 def task_complete(self, task): 1997 def task_complete(self, task):
1944 self.stats.taskCompleted() 1998 self.stats.taskCompleted()
1945 bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData) 1999 bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
@@ -1950,6 +2004,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1950 bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData) 2004 bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
1951 self.scenequeue_notcovered.add(task) 2005 self.scenequeue_notcovered.add(task)
1952 self.scenequeue_updatecounters(task, True) 2006 self.scenequeue_updatecounters(task, True)
2007 self.check_taskfail(task)
1953 2008
1954 def task_failoutright(self, task): 2009 def task_failoutright(self, task):
1955 self.runq_running.add(task) 2010 self.runq_running.add(task)
@@ -2231,3 +2286,27 @@ class runQueuePipe():
2231 if len(self.queue) > 0: 2286 if len(self.queue) > 0:
2232 print("Warning, worker left partial message: %s" % self.queue) 2287 print("Warning, worker left partial message: %s" % self.queue)
2233 self.input.close() 2288 self.input.close()
2289
2290def get_setscene_enforce_whitelist(d):
2291 if d.getVar('BB_SETSCENE_ENFORCE', True) != '1':
2292 return None
2293 whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST", True) or "").split()
2294 outlist = []
2295 for item in whitelist[:]:
2296 if item.startswith('%:'):
2297 for target in sys.argv[1:]:
2298 if not target.startswith('-'):
2299 outlist.append(target.split(':')[0] + ':' + item.split(':')[1])
2300 else:
2301 outlist.append(item)
2302 return outlist
2303
2304def check_setscene_enforce_whitelist(pn, taskname, whitelist):
2305 import fnmatch
2306 if whitelist:
2307 item = '%s:%s' % (pn, taskname)
2308 for whitelist_item in whitelist:
2309 if fnmatch.fnmatch(item, whitelist_item):
2310 return True
2311 return False
2312 return True