summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-06 16:16:54 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-15 10:28:12 +0100
commit2d84e3682c252eff425a1e73ea51f4a1a118e97f (patch)
treecd7a4add18a8df464df82347e98fc89978359aed
parentcf829a5f660392bc83a9175ab4900b170496f349 (diff)
downloadpoky-2d84e3682c252eff425a1e73ea51f4a1a118e97f.tar.gz
bitbake: runqueue: Alter setscenewhitelist handling
Since there is now parallel execution of setscene and normal tasks, the way setscenewhitelist handling worked can't function the way it did. Paul and I never liked its error output anyway. This code tries a different approach, checking the task at execution time but printing the uncovered task list. This code may need improvement after real world usage but can work with the new task flows. (Bitbake rev: a08d8ba5f5194a09391b1904ee31c04c5f0b1e28) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/runqueue.py79
1 files changed, 30 insertions, 49 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index aafb6ffa58..d995e4c04a 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -25,6 +25,7 @@ import subprocess
25import pickle 25import pickle
26from multiprocessing import Process 26from multiprocessing import Process
27import shlex 27import shlex
28import pprint
28 29
29bblogger = logging.getLogger("BitBake") 30bblogger = logging.getLogger("BitBake")
30logger = logging.getLogger("BitBake.RunQueue") 31logger = logging.getLogger("BitBake.RunQueue")
@@ -1681,49 +1682,6 @@ class RunQueue:
1681 output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb) 1682 output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb)
1682 bb.plain("\nTask %s:%s couldn't be used from the cache because:\n We need hash %s, closest matching task was %s\n " % (pn, taskname, h, prevh) + '\n '.join(output)) 1683 bb.plain("\nTask %s:%s couldn't be used from the cache because:\n We need hash %s, closest matching task was %s\n " % (pn, taskname, h, prevh) + '\n '.join(output))
1683 1684
1684def process_setscene_whitelist(rq, rqdata, stampcache, sched, rqex):
1685 # Check tasks that are going to run against the whitelist
1686 def check_norun_task(tid, showerror=False):
1687 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
1688 # Ignore covered tasks
1689 if tid in rqex.tasks_covered:
1690 return False
1691 # Ignore stamped tasks
1692 if rq.check_stamp_task(tid, taskname, cache=stampcache):
1693 return False
1694 # Ignore noexec tasks
1695 taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
1696 if 'noexec' in taskdep and taskname in taskdep['noexec']:
1697 return False
1698
1699 pn = rqdata.dataCaches[mc].pkg_fn[taskfn]
1700 if not check_setscene_enforce_whitelist(pn, taskname, rqdata.setscenewhitelist):
1701 if showerror:
1702 if tid in rqdata.runq_setscene_tids:
1703 logger.error('Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname))
1704 else:
1705 logger.error('Task %s.%s attempted to execute unexpectedly' % (pn, taskname))
1706 return True
1707 return False
1708 # Look to see if any tasks that we think shouldn't run are going to
1709 unexpected = False
1710 for tid in rqdata.runtaskentries:
1711 if check_norun_task(tid):
1712 unexpected = True
1713 break
1714 if unexpected:
1715 # Run through the tasks in the rough order they'd have executed and print errors
1716 # (since the order can be useful - usually missing sstate for the last few tasks
1717 # is the cause of the problem)
1718 task = sched.next()
1719 while task is not None:
1720 check_norun_task(task, showerror=True)
1721 rqex.task_skip(task, 'Setscene enforcement check')
1722 task = sched.next()
1723
1724 rq.state = runQueueCleanUp
1725 return True
1726
1727 1685
1728class RunQueueExecute: 1686class RunQueueExecute:
1729 1687
@@ -1944,12 +1902,6 @@ class RunQueueExecute:
1944 Run the tasks in a queue prepared by rqdata.prepare() 1902 Run the tasks in a queue prepared by rqdata.prepare()
1945 """ 1903 """
1946 1904
1947 if self.rqdata.setscenewhitelist is not None and not self.rqdata.setscenewhitelist_checked:
1948 self.rqdata.setscenewhitelist_checked = True
1949
1950 if process_setscenewhitelist(self.rq, self.rqdata, self.stampcache, self.sched, self):
1951 return True
1952
1953 if self.cooker.configuration.setsceneonly: 1905 if self.cooker.configuration.setsceneonly:
1954 return True 1906 return True
1955 1907
@@ -1963,6 +1915,11 @@ class RunQueueExecute:
1963 if task is not None: 1915 if task is not None:
1964 (mc, fn, taskname, taskfn) = split_tid_mcfn(task) 1916 (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
1965 1917
1918 if self.rqdata.setscenewhitelist is not None:
1919 if self.check_setscenewhitelist(task):
1920 self.task_fail(task, "setscene whitelist")
1921 return True
1922
1966 if task in self.tasks_covered: 1923 if task in self.tasks_covered:
1967 logger.debug(2, "Setscene covered task %s", task) 1924 logger.debug(2, "Setscene covered task %s", task)
1968 self.task_skip(task, "covered") 1925 self.task_skip(task, "covered")
@@ -2348,6 +2305,30 @@ class RunQueueExecute:
2348 #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) 2305 #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n"))
2349 return taskdepdata 2306 return taskdepdata
2350 2307
2308 def check_setscenewhitelist(self, tid):
2309 # Check task that is going to run against the whitelist
2310 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
2311 # Ignore covered tasks
2312 if tid in self.tasks_covered:
2313 return False
2314 # Ignore stamped tasks
2315 if self.rq.check_stamp_task(tid, taskname, cache=self.stampcache):
2316 return False
2317 # Ignore noexec tasks
2318 taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
2319 if 'noexec' in taskdep and taskname in taskdep['noexec']:
2320 return False
2321
2322 pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
2323 if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
2324 if tid in self.rqdata.runq_setscene_tids:
2325 msg = 'Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)
2326 else:
2327 msg = 'Task %s.%s attempted to execute unexpectedly' % (pn, taskname)
2328 logger.error(msg + '\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered))
2329 return True
2330 return False
2331
2351class SQData(object): 2332class SQData(object):
2352 def __init__(self): 2333 def __init__(self):
2353 # SceneQueue dependencies 2334 # SceneQueue dependencies