summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-06 16:29:29 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-15 10:28:12 +0100
commit39ef064da5998b1bf1d47d10a9622e9fe4107796 (patch)
treeb8b34477e2c0f34fc9e42da6a56dda53fe4df23a
parent2d84e3682c252eff425a1e73ea51f4a1a118e97f (diff)
downloadpoky-39ef064da5998b1bf1d47d10a9622e9fe4107796.tar.gz
bitbake: runqueue: Complete the merge of scenequeue and normal task execution
This combines the scqenequeue and normal task execution into one function and simplifies the state engine accordingly. This is the final set of cleanup to fully merge things without adding the extra noise to the previous commits. (Bitbake rev: 56f3396d8c7cfbebd175877c9d773e4e35f8dea1) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/runqueue.py226
1 files changed, 109 insertions, 117 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index d995e4c04a..db0d852c54 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -103,8 +103,6 @@ class RunQueueStats:
103# runQueue state machine 103# runQueue state machine
104runQueuePrepare = 2 104runQueuePrepare = 2
105runQueueSceneInit = 3 105runQueueSceneInit = 3
106runQueueSceneRun = 4
107runQueueRunInit = 5
108runQueueRunning = 6 106runQueueRunning = 6
109runQueueFailed = 7 107runQueueFailed = 7
110runQueueCleanUp = 8 108runQueueCleanUp = 8
@@ -1203,7 +1201,7 @@ class RunQueue:
1203 # Invoked at regular time intervals via the bitbake heartbeat event 1201 # Invoked at regular time intervals via the bitbake heartbeat event
1204 # while the build is running. We generate a unique name for the handler 1202 # while the build is running. We generate a unique name for the handler
1205 # here, just in case that there ever is more than one RunQueue instance, 1203 # here, just in case that there ever is more than one RunQueue instance,
1206 # start the handler when reaching runQueueSceneRun, and stop it when 1204 # start the handler when reaching runQueueSceneInit, and stop it when
1207 # done with the build. 1205 # done with the build.
1208 self.dm = monitordisk.diskMonitor(cfgData) 1206 self.dm = monitordisk.diskMonitor(cfgData)
1209 self.dm_event_handler_name = '_bb_diskmonitor_' + str(id(self)) 1207 self.dm_event_handler_name = '_bb_diskmonitor_' + str(id(self))
@@ -1433,7 +1431,7 @@ class RunQueue:
1433 1431
1434 if not self.dm_event_handler_registered: 1432 if not self.dm_event_handler_registered:
1435 res = bb.event.register(self.dm_event_handler_name, 1433 res = bb.event.register(self.dm_event_handler_name,
1436 lambda x: self.dm.check(self) if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp] else False, 1434 lambda x: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
1437 ('bb.event.HeartbeatEvent',)) 1435 ('bb.event.HeartbeatEvent',))
1438 self.dm_event_handler_registered = True 1436 self.dm_event_handler_registered = True
1439 1437
@@ -1465,8 +1463,6 @@ class RunQueue:
1465 self.state = runQueueRunning 1463 self.state = runQueueRunning
1466 1464
1467 if self.state is runQueueRunning: 1465 if self.state is runQueueRunning:
1468 retval = self.rqexe.sq_execute()
1469 # FIXME revtal
1470 retval = self.rqexe.execute() 1466 retval = self.rqexe.execute()
1471 1467
1472 if self.state is runQueueCleanUp: 1468 if self.state is runQueueCleanUp:
@@ -1899,19 +1895,118 @@ class RunQueueExecute:
1899 1895
1900 def execute(self): 1896 def execute(self):
1901 """ 1897 """
1902 Run the tasks in a queue prepared by rqdata.prepare() 1898 Run the tasks in a queue prepared by prepare_runqueue
1903 """ 1899 """
1904 1900
1905 if self.cooker.configuration.setsceneonly:
1906 return True
1907
1908 self.rq.read_workers() 1901 self.rq.read_workers()
1909 1902
1910 if self.stats.total == 0: 1903 task = None
1911 # nothing to do 1904 if not self.sqdone and self.can_start_task():
1912 self.rq.state = runQueueCleanUp 1905 # Find the next setscene to run
1906 for nexttask in self.rqdata.runq_setscene_tids:
1907 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
1908 if nexttask in self.sqdata.unskippable:
1909 logger.debug(2, "Setscene task %s is unskippable" % nexttask)
1910 if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
1911 fn = fn_from_tid(nexttask)
1912 foundtarget = False
1913 1913
1914 task = self.sched.next() 1914 if nexttask in self.rqdata.target_tids:
1915 foundtarget = True
1916 if not foundtarget:
1917 logger.debug(2, "Skipping setscene for task %s" % nexttask)
1918 self.sq_task_skip(nexttask)
1919 self.scenequeue_notneeded.add(nexttask)
1920 return True
1921 if nexttask in self.sqdata.outrightfail:
1922 self.sq_task_failoutright(nexttask)
1923 return True
1924 task = nexttask
1925 break
1926 if task is not None:
1927 (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
1928 taskname = taskname + "_setscene"
1929 if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache):
1930 logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task)
1931 self.sq_task_failoutright(task)
1932 return True
1933
1934 if self.cooker.configuration.force:
1935 if task in self.rqdata.target_tids:
1936 self.sq_task_failoutright(task)
1937 return True
1938
1939 if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
1940 logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task)
1941 self.sq_task_skip(task)
1942 return True
1943
1944 if self.cooker.configuration.skipsetscene:
1945 logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task)
1946 self.sq_task_failoutright(task)
1947 return True
1948
1949 startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq)
1950 bb.event.fire(startevent, self.cfgData)
1951
1952 taskdepdata = self.sq_build_taskdepdata(task)
1953
1954 taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
1955 taskhash = self.rqdata.get_task_hash(task)
1956 unihash = self.rqdata.get_task_unihash(task)
1957 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
1958 if not mc in self.rq.fakeworker:
1959 self.rq.start_fakeworker(self, mc)
1960 self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
1961 self.rq.fakeworker[mc].process.stdin.flush()
1962 else:
1963 self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
1964 self.rq.worker[mc].process.stdin.flush()
1965
1966 self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
1967 self.build_stamps2.append(self.build_stamps[task])
1968 self.sq_running.add(task)
1969 self.sq_live.add(task)
1970 self.sq_stats.taskActive()
1971 if self.can_start_task():
1972 return True
1973
1974 if not self.sq_live and not self.sqdone:
1975 logger.info("Setscene tasks completed")
1976 logger.debug(1, 'We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
1977
1978 completeevent = sceneQueueComplete(self.sq_stats, self.rq)
1979 bb.event.fire(completeevent, self.cfgData)
1980
1981 err = False
1982 for x in self.rqdata.runtaskentries:
1983 if x not in self.tasks_covered and x not in self.tasks_notcovered:
1984 logger.error("Task %s was never moved from the setscene queue" % x)
1985 err = True
1986 if x not in self.tasks_scenequeue_done:
1987 logger.error("Task %s was never processed by the setscene code" % x)
1988 err = True
1989 if len(self.rqdata.runtaskentries[x].depends) == 0 and x not in self.runq_buildable:
1990 logger.error("Task %s was never marked as buildable by the setscene code" % x)
1991 err = True
1992 if err:
1993 self.rq.state = runQueueFailed
1994 return True
1995
1996 if self.cooker.configuration.setsceneonly:
1997 self.rq.state = runQueueComplete
1998 return True
1999 self.sqdone = True
2000
2001 if self.stats.total == 0:
2002 # nothing to do
2003 self.rq.state = runQueueComplete
2004 return True
2005
2006 if self.cooker.configuration.setsceneonly:
2007 task = None
2008 else:
2009 task = self.sched.next()
1915 if task is not None: 2010 if task is not None:
1916 (mc, fn, taskname, taskfn) = split_tid_mcfn(task) 2011 (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
1917 2012
@@ -2164,109 +2259,6 @@ class RunQueueExecute:
2164 self.sq_stats.taskSkipped() 2259 self.sq_stats.taskSkipped()
2165 self.sq_stats.taskCompleted() 2260 self.sq_stats.taskCompleted()
2166 2261
2167 def sq_execute(self):
2168 """
2169 Run the tasks in a queue prepared by prepare_runqueue
2170 """
2171
2172 if self.sqdone:
2173 return True
2174
2175 self.rq.read_workers()
2176
2177 task = None
2178 if self.can_start_task():
2179 # Find the next setscene to run
2180 for nexttask in self.rqdata.runq_setscene_tids:
2181 if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
2182 if nexttask in self.sqdata.unskippable:
2183 logger.debug(2, "Setscene task %s is unskippable" % nexttask)
2184 if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
2185 fn = fn_from_tid(nexttask)
2186 foundtarget = False
2187
2188 if nexttask in self.rqdata.target_tids:
2189 foundtarget = True
2190 if not foundtarget:
2191 logger.debug(2, "Skipping setscene for task %s" % nexttask)
2192 self.sq_task_skip(nexttask)
2193 self.scenequeue_notneeded.add(nexttask)
2194 return True
2195 if nexttask in self.sqdata.outrightfail:
2196 self.sq_task_failoutright(nexttask)
2197 return True
2198 task = nexttask
2199 break
2200 if task is not None:
2201 (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
2202 taskname = taskname + "_setscene"
2203 if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache):
2204 logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task)
2205 self.sq_task_failoutright(task)
2206 return True
2207
2208 if self.cooker.configuration.force:
2209 if task in self.rqdata.target_tids:
2210 self.sq_task_failoutright(task)
2211 return True
2212
2213 if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
2214 logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task)
2215 self.sq_task_skip(task)
2216 return True
2217
2218 if self.cooker.configuration.skipsetscene:
2219 logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task)
2220 self.sq_task_failoutright(task)
2221 return True
2222
2223 startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq)
2224 bb.event.fire(startevent, self.cfgData)
2225
2226 taskdepdata = self.sq_build_taskdepdata(task)
2227
2228 taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
2229 taskhash = self.rqdata.get_task_hash(task)
2230 unihash = self.rqdata.get_task_unihash(task)
2231 if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
2232 if not mc in self.rq.fakeworker:
2233 self.rq.start_fakeworker(self, mc)
2234 self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
2235 self.rq.fakeworker[mc].process.stdin.flush()
2236 else:
2237 self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
2238 self.rq.worker[mc].process.stdin.flush()
2239
2240 self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
2241 self.build_stamps2.append(self.build_stamps[task])
2242 self.sq_running.add(task)
2243 self.sq_live.add(task)
2244 self.sq_stats.taskActive()
2245 if self.can_start_task():
2246 return True
2247
2248 if self.stats.active > 0 or self.sq_stats.active > 0:
2249 self.rq.read_workers()
2250 return self.rq.active_fds()
2251
2252 #for tid in self.sqdata.sq_revdeps:
2253 # if tid not in self.sq_running:
2254 # buildable = tid in self.sq_buildable
2255 # revdeps = self.sqdata.sq_revdeps[tid]
2256 # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps)))
2257
2258 logger.debug(1, 'We can skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
2259
2260 completeevent = sceneQueueComplete(self.sq_stats, self.rq)
2261 bb.event.fire(completeevent, self.cfgData)
2262
2263 if self.cooker.configuration.setsceneonly:
2264 self.rq.state = runQueueComplete
2265
2266 self.sqdone = True
2267
2268 return True
2269
2270 def sq_build_taskdepdata(self, task): 2262 def sq_build_taskdepdata(self, task):
2271 def getsetscenedeps(tid): 2263 def getsetscenedeps(tid):
2272 deps = set() 2264 deps = set()