diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-07-03 16:22:15 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-07-15 10:28:12 +0100 |
commit | 57c527d630ce8230fd17f3cdb33d388ced34c63b (patch) | |
tree | 0787a5538b6ff8a2d8586cb06de4f08a99f5752a /bitbake/lib/bb/runqueue.py | |
parent | cfb7312b72b7bfaec05ca6b6ed0735473b2724a7 (diff) | |
download | poky-57c527d630ce8230fd17f3cdb33d388ced34c63b.tar.gz |
bitbake: runqueue: Merge scenequeue and real task queue code together
Merge the unique functions from the Tasks and Scenequeue Tasks classes
into the common base class.
(Bitbake rev: 7539fe22bc831bb835901e3aca77985ab4ebc4c7)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 516 |
1 files changed, 258 insertions, 258 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 3cc804de45..80cc60ed7f 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -1839,66 +1839,6 @@ class RunQueueExecute: | |||
1839 | can_start = active < self.number_tasks | 1839 | can_start = active < self.number_tasks |
1840 | return can_start | 1840 | return can_start |
1841 | 1841 | ||
1842 | class RunQueueExecuteDummy(RunQueueExecute): | ||
1843 | def __init__(self, rq): | ||
1844 | self.rq = rq | ||
1845 | self.stats = RunQueueStats(0) | ||
1846 | |||
1847 | def finish(self): | ||
1848 | self.rq.state = runQueueComplete | ||
1849 | return | ||
1850 | |||
1851 | class RunQueueExecuteTasks(RunQueueExecute): | ||
1852 | def __init__(self, rq): | ||
1853 | RunQueueExecute.__init__(self, rq) | ||
1854 | |||
1855 | self.stampcache = {} | ||
1856 | |||
1857 | # Mark initial buildable tasks | ||
1858 | for tid in self.rqdata.runtaskentries: | ||
1859 | if len(self.rqdata.runtaskentries[tid].depends) == 0: | ||
1860 | self.runq_buildable.add(tid) | ||
1861 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): | ||
1862 | self.rq.scenequeue_covered.add(tid) | ||
1863 | |||
1864 | found = True | ||
1865 | while found: | ||
1866 | found = False | ||
1867 | for tid in self.rqdata.runtaskentries: | ||
1868 | if tid in self.rq.scenequeue_covered: | ||
1869 | continue | ||
1870 | logger.debug(1, 'Considering %s: %s' % (tid, str(self.rqdata.runtaskentries[tid].revdeps))) | ||
1871 | |||
1872 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): | ||
1873 | if tid in self.rq.scenequeue_notcovered: | ||
1874 | continue | ||
1875 | found = True | ||
1876 | self.rq.scenequeue_covered.add(tid) | ||
1877 | |||
1878 | logger.debug(1, 'Skip list %s', sorted(self.rq.scenequeue_covered)) | ||
1879 | |||
1880 | for task in self.rq.scenequeue_notcovered: | ||
1881 | logger.debug(1, 'Not skipping task %s', task) | ||
1882 | |||
1883 | for mc in self.rqdata.dataCaches: | ||
1884 | target_pairs = [] | ||
1885 | for tid in self.rqdata.target_tids: | ||
1886 | (tidmc, fn, taskname, _) = split_tid_mcfn(tid) | ||
1887 | if tidmc == mc: | ||
1888 | target_pairs.append((fn, taskname)) | ||
1889 | |||
1890 | event.fire(bb.event.StampUpdate(target_pairs, self.rqdata.dataCaches[mc].stamp), self.cfgData) | ||
1891 | |||
1892 | schedulers = self.get_schedulers() | ||
1893 | for scheduler in schedulers: | ||
1894 | if self.scheduler == scheduler.name: | ||
1895 | self.sched = scheduler(self, self.rqdata) | ||
1896 | logger.debug(1, "Using runqueue scheduler '%s'", scheduler.name) | ||
1897 | break | ||
1898 | else: | ||
1899 | bb.fatal("Invalid scheduler '%s'. Available schedulers: %s" % | ||
1900 | (self.scheduler, ", ".join(obj.name for obj in schedulers))) | ||
1901 | |||
1902 | def get_schedulers(self): | 1842 | def get_schedulers(self): |
1903 | schedulers = set(obj for obj in globals().values() | 1843 | schedulers = set(obj for obj in globals().values() |
1904 | if type(obj) is type and | 1844 | if type(obj) is type and |
@@ -2100,6 +2040,204 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
2100 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) | 2040 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) |
2101 | return taskdepdata | 2041 | return taskdepdata |
2102 | 2042 | ||
2043 | def scenequeue_updatecounters(self, task, fail = False): | ||
2044 | for dep in self.sqdata.sq_deps[task]: | ||
2045 | if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]: | ||
2046 | logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep)) | ||
2047 | self.scenequeue_updatecounters(dep, fail) | ||
2048 | continue | ||
2049 | if task not in self.sqdata.sq_revdeps2[dep]: | ||
2050 | # May already have been removed by the fail case above | ||
2051 | continue | ||
2052 | self.sqdata.sq_revdeps2[dep].remove(task) | ||
2053 | if len(self.sqdata.sq_revdeps2[dep]) == 0: | ||
2054 | self.sq_buildable.add(dep) | ||
2055 | |||
2056 | def sq_task_completeoutright(self, task): | ||
2057 | """ | ||
2058 | Mark a task as completed | ||
2059 | Look at the reverse dependencies and mark any task with | ||
2060 | completed dependencies as buildable | ||
2061 | """ | ||
2062 | |||
2063 | logger.debug(1, 'Found task %s which could be accelerated', task) | ||
2064 | self.scenequeue_covered.add(task) | ||
2065 | self.scenequeue_updatecounters(task) | ||
2066 | |||
2067 | def sq_check_taskfail(self, task): | ||
2068 | if self.rqdata.setscenewhitelist is not None: | ||
2069 | realtask = task.split('_setscene')[0] | ||
2070 | (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask) | ||
2071 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
2072 | if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist): | ||
2073 | logger.error('Task %s.%s failed' % (pn, taskname + "_setscene")) | ||
2074 | self.rq.state = runQueueCleanUp | ||
2075 | |||
2076 | def sq_task_complete(self, task): | ||
2077 | self.sq_stats.taskCompleted() | ||
2078 | bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData) | ||
2079 | self.sq_task_completeoutright(task) | ||
2080 | |||
2081 | def sq_task_fail(self, task, result): | ||
2082 | self.sq_stats.taskFailed() | ||
2083 | bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData) | ||
2084 | self.scenequeue_notcovered.add(task) | ||
2085 | self.scenequeue_updatecounters(task, True) | ||
2086 | self.sq_check_taskfail(task) | ||
2087 | |||
2088 | def sq_task_failoutright(self, task): | ||
2089 | self.sq_running.add(task) | ||
2090 | self.sq_buildable.add(task) | ||
2091 | self.sq_stats.taskSkipped() | ||
2092 | self.sq_stats.taskCompleted() | ||
2093 | self.scenequeue_notcovered.add(task) | ||
2094 | self.scenequeue_updatecounters(task, True) | ||
2095 | |||
2096 | def sq_task_skip(self, task): | ||
2097 | self.sq_running.add(task) | ||
2098 | self.sq_buildable.add(task) | ||
2099 | self.sq_task_completeoutright(task) | ||
2100 | self.sq_stats.taskSkipped() | ||
2101 | self.sq_stats.taskCompleted() | ||
2102 | |||
2103 | def sq_execute(self): | ||
2104 | """ | ||
2105 | Run the tasks in a queue prepared by prepare_runqueue | ||
2106 | """ | ||
2107 | |||
2108 | self.rq.read_workers() | ||
2109 | |||
2110 | task = None | ||
2111 | if self.can_start_task(): | ||
2112 | # Find the next setscene to run | ||
2113 | for nexttask in self.rqdata.runq_setscene_tids: | ||
2114 | if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): | ||
2115 | if nexttask in self.sqdata.unskippable: | ||
2116 | logger.debug(2, "Setscene task %s is unskippable" % nexttask) | ||
2117 | if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): | ||
2118 | fn = fn_from_tid(nexttask) | ||
2119 | foundtarget = False | ||
2120 | |||
2121 | if nexttask in self.rqdata.target_tids: | ||
2122 | foundtarget = True | ||
2123 | if not foundtarget: | ||
2124 | logger.debug(2, "Skipping setscene for task %s" % nexttask) | ||
2125 | self.sq_task_skip(nexttask) | ||
2126 | self.scenequeue_notneeded.add(nexttask) | ||
2127 | return True | ||
2128 | if nexttask in self.sqdata.outrightfail: | ||
2129 | self.sq_task_failoutright(nexttask) | ||
2130 | return True | ||
2131 | task = nexttask | ||
2132 | break | ||
2133 | if task is not None: | ||
2134 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) | ||
2135 | taskname = taskname + "_setscene" | ||
2136 | if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache): | ||
2137 | logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task) | ||
2138 | self.sq_task_failoutright(task) | ||
2139 | return True | ||
2140 | |||
2141 | if self.cooker.configuration.force: | ||
2142 | if task in self.rqdata.target_tids: | ||
2143 | self.sq_task_failoutright(task) | ||
2144 | return True | ||
2145 | |||
2146 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): | ||
2147 | logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task) | ||
2148 | self.sq_task_skip(task) | ||
2149 | return True | ||
2150 | |||
2151 | if self.cooker.configuration.skipsetscene: | ||
2152 | logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task) | ||
2153 | self.sq_task_failoutright(task) | ||
2154 | return True | ||
2155 | |||
2156 | startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq) | ||
2157 | bb.event.fire(startevent, self.cfgData) | ||
2158 | |||
2159 | taskdepdata = self.sq_build_taskdepdata(task) | ||
2160 | |||
2161 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] | ||
2162 | taskhash = self.rqdata.get_task_hash(task) | ||
2163 | unihash = self.rqdata.get_task_unihash(task) | ||
2164 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: | ||
2165 | if not mc in self.rq.fakeworker: | ||
2166 | self.rq.start_fakeworker(self, mc) | ||
2167 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | ||
2168 | self.rq.fakeworker[mc].process.stdin.flush() | ||
2169 | else: | ||
2170 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | ||
2171 | self.rq.worker[mc].process.stdin.flush() | ||
2172 | |||
2173 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) | ||
2174 | self.build_stamps2.append(self.build_stamps[task]) | ||
2175 | self.sq_running.add(task) | ||
2176 | self.sq_live.add(task) | ||
2177 | self.sq_stats.taskActive() | ||
2178 | if self.can_start_task(): | ||
2179 | return True | ||
2180 | |||
2181 | if self.sq_stats.active > 0: | ||
2182 | self.rq.read_workers() | ||
2183 | return self.rq.active_fds() | ||
2184 | |||
2185 | #for tid in self.sqdata.sq_revdeps: | ||
2186 | # if tid not in self.sq_running: | ||
2187 | # buildable = tid in self.sq_buildable | ||
2188 | # revdeps = self.sqdata.sq_revdeps[tid] | ||
2189 | # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps))) | ||
2190 | |||
2191 | self.rq.scenequeue_covered = self.scenequeue_covered | ||
2192 | self.rq.scenequeue_notcovered = self.scenequeue_notcovered | ||
2193 | |||
2194 | logger.debug(1, 'We can skip tasks %s', "\n".join(sorted(self.rq.scenequeue_covered))) | ||
2195 | |||
2196 | self.rq.state = runQueueRunInit | ||
2197 | |||
2198 | completeevent = sceneQueueComplete(self.sq_stats, self.rq) | ||
2199 | bb.event.fire(completeevent, self.cfgData) | ||
2200 | |||
2201 | return True | ||
2202 | |||
2203 | def sq_build_taskdepdata(self, task): | ||
2204 | def getsetscenedeps(tid): | ||
2205 | deps = set() | ||
2206 | (mc, fn, taskname, _) = split_tid_mcfn(tid) | ||
2207 | realtid = tid + "_setscene" | ||
2208 | idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends | ||
2209 | for (depname, idependtask) in idepends: | ||
2210 | if depname not in self.rqdata.taskData[mc].build_targets: | ||
2211 | continue | ||
2212 | |||
2213 | depfn = self.rqdata.taskData[mc].build_targets[depname][0] | ||
2214 | if depfn is None: | ||
2215 | continue | ||
2216 | deptid = depfn + ":" + idependtask.replace("_setscene", "") | ||
2217 | deps.add(deptid) | ||
2218 | return deps | ||
2219 | |||
2220 | taskdepdata = {} | ||
2221 | next = getsetscenedeps(task) | ||
2222 | next.add(task) | ||
2223 | while next: | ||
2224 | additional = [] | ||
2225 | for revdep in next: | ||
2226 | (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep) | ||
2227 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
2228 | deps = getsetscenedeps(revdep) | ||
2229 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] | ||
2230 | taskhash = self.rqdata.runtaskentries[revdep].hash | ||
2231 | unihash = self.rqdata.runtaskentries[revdep].unihash | ||
2232 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash] | ||
2233 | for revdep2 in deps: | ||
2234 | if revdep2 not in taskdepdata: | ||
2235 | additional.append(revdep2) | ||
2236 | next = additional | ||
2237 | |||
2238 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) | ||
2239 | return taskdepdata | ||
2240 | |||
2103 | class SQData(object): | 2241 | class SQData(object): |
2104 | def __init__(self): | 2242 | def __init__(self): |
2105 | # SceneQueue dependencies | 2243 | # SceneQueue dependencies |
@@ -2346,6 +2484,66 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): | |||
2346 | sqdata.outrightfail.append(tid) | 2484 | sqdata.outrightfail.append(tid) |
2347 | 2485 | ||
2348 | 2486 | ||
2487 | class RunQueueExecuteDummy(RunQueueExecute): | ||
2488 | def __init__(self, rq): | ||
2489 | self.rq = rq | ||
2490 | self.stats = RunQueueStats(0) | ||
2491 | |||
2492 | def finish(self): | ||
2493 | self.rq.state = runQueueComplete | ||
2494 | return | ||
2495 | |||
2496 | class RunQueueExecuteTasks(RunQueueExecute): | ||
2497 | def __init__(self, rq): | ||
2498 | RunQueueExecute.__init__(self, rq) | ||
2499 | |||
2500 | self.stampcache = {} | ||
2501 | |||
2502 | # Mark initial buildable tasks | ||
2503 | for tid in self.rqdata.runtaskentries: | ||
2504 | if len(self.rqdata.runtaskentries[tid].depends) == 0: | ||
2505 | self.runq_buildable.add(tid) | ||
2506 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): | ||
2507 | self.rq.scenequeue_covered.add(tid) | ||
2508 | |||
2509 | found = True | ||
2510 | while found: | ||
2511 | found = False | ||
2512 | for tid in self.rqdata.runtaskentries: | ||
2513 | if tid in self.rq.scenequeue_covered: | ||
2514 | continue | ||
2515 | logger.debug(1, 'Considering %s: %s' % (tid, str(self.rqdata.runtaskentries[tid].revdeps))) | ||
2516 | |||
2517 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): | ||
2518 | if tid in self.rq.scenequeue_notcovered: | ||
2519 | continue | ||
2520 | found = True | ||
2521 | self.rq.scenequeue_covered.add(tid) | ||
2522 | |||
2523 | logger.debug(1, 'Skip list %s', sorted(self.rq.scenequeue_covered)) | ||
2524 | |||
2525 | for task in self.rq.scenequeue_notcovered: | ||
2526 | logger.debug(1, 'Not skipping task %s', task) | ||
2527 | |||
2528 | for mc in self.rqdata.dataCaches: | ||
2529 | target_pairs = [] | ||
2530 | for tid in self.rqdata.target_tids: | ||
2531 | (tidmc, fn, taskname, _) = split_tid_mcfn(tid) | ||
2532 | if tidmc == mc: | ||
2533 | target_pairs.append((fn, taskname)) | ||
2534 | |||
2535 | event.fire(bb.event.StampUpdate(target_pairs, self.rqdata.dataCaches[mc].stamp), self.cfgData) | ||
2536 | |||
2537 | schedulers = self.get_schedulers() | ||
2538 | for scheduler in schedulers: | ||
2539 | if self.scheduler == scheduler.name: | ||
2540 | self.sched = scheduler(self, self.rqdata) | ||
2541 | logger.debug(1, "Using runqueue scheduler '%s'", scheduler.name) | ||
2542 | break | ||
2543 | else: | ||
2544 | bb.fatal("Invalid scheduler '%s'. Available schedulers: %s" % | ||
2545 | (self.scheduler, ", ".join(obj.name for obj in schedulers))) | ||
2546 | |||
2349 | class RunQueueExecuteScenequeue(RunQueueExecute): | 2547 | class RunQueueExecuteScenequeue(RunQueueExecute): |
2350 | def __init__(self, rq): | 2548 | def __init__(self, rq): |
2351 | RunQueueExecute.__init__(self, rq) | 2549 | RunQueueExecute.__init__(self, rq) |
@@ -2368,204 +2566,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2368 | 2566 | ||
2369 | self.rq.state = runQueueSceneRun | 2567 | self.rq.state = runQueueSceneRun |
2370 | 2568 | ||
2371 | def scenequeue_updatecounters(self, task, fail = False): | ||
2372 | for dep in self.sqdata.sq_deps[task]: | ||
2373 | if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]: | ||
2374 | logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep)) | ||
2375 | self.scenequeue_updatecounters(dep, fail) | ||
2376 | continue | ||
2377 | if task not in self.sqdata.sq_revdeps2[dep]: | ||
2378 | # May already have been removed by the fail case above | ||
2379 | continue | ||
2380 | self.sqdata.sq_revdeps2[dep].remove(task) | ||
2381 | if len(self.sqdata.sq_revdeps2[dep]) == 0: | ||
2382 | self.sq_buildable.add(dep) | ||
2383 | |||
2384 | def sq_task_completeoutright(self, task): | ||
2385 | """ | ||
2386 | Mark a task as completed | ||
2387 | Look at the reverse dependencies and mark any task with | ||
2388 | completed dependencies as buildable | ||
2389 | """ | ||
2390 | |||
2391 | logger.debug(1, 'Found task %s which could be accelerated', task) | ||
2392 | self.scenequeue_covered.add(task) | ||
2393 | self.scenequeue_updatecounters(task) | ||
2394 | |||
2395 | def sq_check_taskfail(self, task): | ||
2396 | if self.rqdata.setscenewhitelist is not None: | ||
2397 | realtask = task.split('_setscene')[0] | ||
2398 | (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask) | ||
2399 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
2400 | if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist): | ||
2401 | logger.error('Task %s.%s failed' % (pn, taskname + "_setscene")) | ||
2402 | self.rq.state = runQueueCleanUp | ||
2403 | |||
2404 | def sq_task_complete(self, task): | ||
2405 | self.sq_stats.taskCompleted() | ||
2406 | bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData) | ||
2407 | self.sq_task_completeoutright(task) | ||
2408 | |||
2409 | def sq_task_fail(self, task, result): | ||
2410 | self.sq_stats.taskFailed() | ||
2411 | bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData) | ||
2412 | self.scenequeue_notcovered.add(task) | ||
2413 | self.scenequeue_updatecounters(task, True) | ||
2414 | self.sq_check_taskfail(task) | ||
2415 | |||
2416 | def sq_task_failoutright(self, task): | ||
2417 | self.sq_running.add(task) | ||
2418 | self.sq_buildable.add(task) | ||
2419 | self.sq_stats.taskSkipped() | ||
2420 | self.sq_stats.taskCompleted() | ||
2421 | self.scenequeue_notcovered.add(task) | ||
2422 | self.scenequeue_updatecounters(task, True) | ||
2423 | |||
2424 | def sq_task_skip(self, task): | ||
2425 | self.sq_running.add(task) | ||
2426 | self.sq_buildable.add(task) | ||
2427 | self.sq_task_completeoutright(task) | ||
2428 | self.sq_stats.taskSkipped() | ||
2429 | self.sq_stats.taskCompleted() | ||
2430 | |||
2431 | def sq_execute(self): | ||
2432 | """ | ||
2433 | Run the tasks in a queue prepared by prepare_runqueue | ||
2434 | """ | ||
2435 | |||
2436 | self.rq.read_workers() | ||
2437 | |||
2438 | task = None | ||
2439 | if self.can_start_task(): | ||
2440 | # Find the next setscene to run | ||
2441 | for nexttask in self.rqdata.runq_setscene_tids: | ||
2442 | if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): | ||
2443 | if nexttask in self.sqdata.unskippable: | ||
2444 | logger.debug(2, "Setscene task %s is unskippable" % nexttask) | ||
2445 | if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): | ||
2446 | fn = fn_from_tid(nexttask) | ||
2447 | foundtarget = False | ||
2448 | |||
2449 | if nexttask in self.rqdata.target_tids: | ||
2450 | foundtarget = True | ||
2451 | if not foundtarget: | ||
2452 | logger.debug(2, "Skipping setscene for task %s" % nexttask) | ||
2453 | self.sq_task_skip(nexttask) | ||
2454 | self.scenequeue_notneeded.add(nexttask) | ||
2455 | return True | ||
2456 | if nexttask in self.sqdata.outrightfail: | ||
2457 | self.sq_task_failoutright(nexttask) | ||
2458 | return True | ||
2459 | task = nexttask | ||
2460 | break | ||
2461 | if task is not None: | ||
2462 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) | ||
2463 | taskname = taskname + "_setscene" | ||
2464 | if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache): | ||
2465 | logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task) | ||
2466 | self.sq_task_failoutright(task) | ||
2467 | return True | ||
2468 | |||
2469 | if self.cooker.configuration.force: | ||
2470 | if task in self.rqdata.target_tids: | ||
2471 | self.sq_task_failoutright(task) | ||
2472 | return True | ||
2473 | |||
2474 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): | ||
2475 | logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task) | ||
2476 | self.sq_task_skip(task) | ||
2477 | return True | ||
2478 | |||
2479 | if self.cooker.configuration.skipsetscene: | ||
2480 | logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task) | ||
2481 | self.sq_task_failoutright(task) | ||
2482 | return True | ||
2483 | |||
2484 | startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq) | ||
2485 | bb.event.fire(startevent, self.cfgData) | ||
2486 | |||
2487 | taskdepdata = self.sq_build_taskdepdata(task) | ||
2488 | |||
2489 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] | ||
2490 | taskhash = self.rqdata.get_task_hash(task) | ||
2491 | unihash = self.rqdata.get_task_unihash(task) | ||
2492 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: | ||
2493 | if not mc in self.rq.fakeworker: | ||
2494 | self.rq.start_fakeworker(self, mc) | ||
2495 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | ||
2496 | self.rq.fakeworker[mc].process.stdin.flush() | ||
2497 | else: | ||
2498 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | ||
2499 | self.rq.worker[mc].process.stdin.flush() | ||
2500 | |||
2501 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) | ||
2502 | self.build_stamps2.append(self.build_stamps[task]) | ||
2503 | self.sq_running.add(task) | ||
2504 | self.sq_live.add(task) | ||
2505 | self.sq_stats.taskActive() | ||
2506 | if self.can_start_task(): | ||
2507 | return True | ||
2508 | |||
2509 | if self.sq_stats.active > 0: | ||
2510 | self.rq.read_workers() | ||
2511 | return self.rq.active_fds() | ||
2512 | |||
2513 | #for tid in self.sqdata.sq_revdeps: | ||
2514 | # if tid not in self.sq_running: | ||
2515 | # buildable = tid in self.sq_buildable | ||
2516 | # revdeps = self.sqdata.sq_revdeps[tid] | ||
2517 | # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps))) | ||
2518 | |||
2519 | self.rq.scenequeue_covered = self.scenequeue_covered | ||
2520 | self.rq.scenequeue_notcovered = self.scenequeue_notcovered | ||
2521 | |||
2522 | logger.debug(1, 'We can skip tasks %s', "\n".join(sorted(self.rq.scenequeue_covered))) | ||
2523 | |||
2524 | self.rq.state = runQueueRunInit | ||
2525 | |||
2526 | completeevent = sceneQueueComplete(self.sq_stats, self.rq) | ||
2527 | bb.event.fire(completeevent, self.cfgData) | ||
2528 | |||
2529 | return True | ||
2530 | |||
2531 | def sq_build_taskdepdata(self, task): | ||
2532 | def getsetscenedeps(tid): | ||
2533 | deps = set() | ||
2534 | (mc, fn, taskname, _) = split_tid_mcfn(tid) | ||
2535 | realtid = tid + "_setscene" | ||
2536 | idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends | ||
2537 | for (depname, idependtask) in idepends: | ||
2538 | if depname not in self.rqdata.taskData[mc].build_targets: | ||
2539 | continue | ||
2540 | |||
2541 | depfn = self.rqdata.taskData[mc].build_targets[depname][0] | ||
2542 | if depfn is None: | ||
2543 | continue | ||
2544 | deptid = depfn + ":" + idependtask.replace("_setscene", "") | ||
2545 | deps.add(deptid) | ||
2546 | return deps | ||
2547 | |||
2548 | taskdepdata = {} | ||
2549 | next = getsetscenedeps(task) | ||
2550 | next.add(task) | ||
2551 | while next: | ||
2552 | additional = [] | ||
2553 | for revdep in next: | ||
2554 | (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep) | ||
2555 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
2556 | deps = getsetscenedeps(revdep) | ||
2557 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] | ||
2558 | taskhash = self.rqdata.runtaskentries[revdep].hash | ||
2559 | unihash = self.rqdata.runtaskentries[revdep].unihash | ||
2560 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash] | ||
2561 | for revdep2 in deps: | ||
2562 | if revdep2 not in taskdepdata: | ||
2563 | additional.append(revdep2) | ||
2564 | next = additional | ||
2565 | |||
2566 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) | ||
2567 | return taskdepdata | ||
2568 | |||
2569 | class TaskFailure(Exception): | 2569 | class TaskFailure(Exception): |
2570 | """ | 2570 | """ |
2571 | Exception raised when a task in a runqueue fails | 2571 | Exception raised when a task in a runqueue fails |