diff options
| -rw-r--r-- | bitbake/lib/bb/runqueue.py | 516 |
1 files changed, 258 insertions, 258 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 3cc804de45..80cc60ed7f 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
| @@ -1839,66 +1839,6 @@ class RunQueueExecute: | |||
| 1839 | can_start = active < self.number_tasks | 1839 | can_start = active < self.number_tasks |
| 1840 | return can_start | 1840 | return can_start |
| 1841 | 1841 | ||
| 1842 | class RunQueueExecuteDummy(RunQueueExecute): | ||
| 1843 | def __init__(self, rq): | ||
| 1844 | self.rq = rq | ||
| 1845 | self.stats = RunQueueStats(0) | ||
| 1846 | |||
| 1847 | def finish(self): | ||
| 1848 | self.rq.state = runQueueComplete | ||
| 1849 | return | ||
| 1850 | |||
| 1851 | class RunQueueExecuteTasks(RunQueueExecute): | ||
| 1852 | def __init__(self, rq): | ||
| 1853 | RunQueueExecute.__init__(self, rq) | ||
| 1854 | |||
| 1855 | self.stampcache = {} | ||
| 1856 | |||
| 1857 | # Mark initial buildable tasks | ||
| 1858 | for tid in self.rqdata.runtaskentries: | ||
| 1859 | if len(self.rqdata.runtaskentries[tid].depends) == 0: | ||
| 1860 | self.runq_buildable.add(tid) | ||
| 1861 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): | ||
| 1862 | self.rq.scenequeue_covered.add(tid) | ||
| 1863 | |||
| 1864 | found = True | ||
| 1865 | while found: | ||
| 1866 | found = False | ||
| 1867 | for tid in self.rqdata.runtaskentries: | ||
| 1868 | if tid in self.rq.scenequeue_covered: | ||
| 1869 | continue | ||
| 1870 | logger.debug(1, 'Considering %s: %s' % (tid, str(self.rqdata.runtaskentries[tid].revdeps))) | ||
| 1871 | |||
| 1872 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): | ||
| 1873 | if tid in self.rq.scenequeue_notcovered: | ||
| 1874 | continue | ||
| 1875 | found = True | ||
| 1876 | self.rq.scenequeue_covered.add(tid) | ||
| 1877 | |||
| 1878 | logger.debug(1, 'Skip list %s', sorted(self.rq.scenequeue_covered)) | ||
| 1879 | |||
| 1880 | for task in self.rq.scenequeue_notcovered: | ||
| 1881 | logger.debug(1, 'Not skipping task %s', task) | ||
| 1882 | |||
| 1883 | for mc in self.rqdata.dataCaches: | ||
| 1884 | target_pairs = [] | ||
| 1885 | for tid in self.rqdata.target_tids: | ||
| 1886 | (tidmc, fn, taskname, _) = split_tid_mcfn(tid) | ||
| 1887 | if tidmc == mc: | ||
| 1888 | target_pairs.append((fn, taskname)) | ||
| 1889 | |||
| 1890 | event.fire(bb.event.StampUpdate(target_pairs, self.rqdata.dataCaches[mc].stamp), self.cfgData) | ||
| 1891 | |||
| 1892 | schedulers = self.get_schedulers() | ||
| 1893 | for scheduler in schedulers: | ||
| 1894 | if self.scheduler == scheduler.name: | ||
| 1895 | self.sched = scheduler(self, self.rqdata) | ||
| 1896 | logger.debug(1, "Using runqueue scheduler '%s'", scheduler.name) | ||
| 1897 | break | ||
| 1898 | else: | ||
| 1899 | bb.fatal("Invalid scheduler '%s'. Available schedulers: %s" % | ||
| 1900 | (self.scheduler, ", ".join(obj.name for obj in schedulers))) | ||
| 1901 | |||
| 1902 | def get_schedulers(self): | 1842 | def get_schedulers(self): |
| 1903 | schedulers = set(obj for obj in globals().values() | 1843 | schedulers = set(obj for obj in globals().values() |
| 1904 | if type(obj) is type and | 1844 | if type(obj) is type and |
| @@ -2100,6 +2040,204 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
| 2100 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) | 2040 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) |
| 2101 | return taskdepdata | 2041 | return taskdepdata |
| 2102 | 2042 | ||
| 2043 | def scenequeue_updatecounters(self, task, fail = False): | ||
| 2044 | for dep in self.sqdata.sq_deps[task]: | ||
| 2045 | if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]: | ||
| 2046 | logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep)) | ||
| 2047 | self.scenequeue_updatecounters(dep, fail) | ||
| 2048 | continue | ||
| 2049 | if task not in self.sqdata.sq_revdeps2[dep]: | ||
| 2050 | # May already have been removed by the fail case above | ||
| 2051 | continue | ||
| 2052 | self.sqdata.sq_revdeps2[dep].remove(task) | ||
| 2053 | if len(self.sqdata.sq_revdeps2[dep]) == 0: | ||
| 2054 | self.sq_buildable.add(dep) | ||
| 2055 | |||
| 2056 | def sq_task_completeoutright(self, task): | ||
| 2057 | """ | ||
| 2058 | Mark a task as completed | ||
| 2059 | Look at the reverse dependencies and mark any task with | ||
| 2060 | completed dependencies as buildable | ||
| 2061 | """ | ||
| 2062 | |||
| 2063 | logger.debug(1, 'Found task %s which could be accelerated', task) | ||
| 2064 | self.scenequeue_covered.add(task) | ||
| 2065 | self.scenequeue_updatecounters(task) | ||
| 2066 | |||
| 2067 | def sq_check_taskfail(self, task): | ||
| 2068 | if self.rqdata.setscenewhitelist is not None: | ||
| 2069 | realtask = task.split('_setscene')[0] | ||
| 2070 | (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask) | ||
| 2071 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
| 2072 | if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist): | ||
| 2073 | logger.error('Task %s.%s failed' % (pn, taskname + "_setscene")) | ||
| 2074 | self.rq.state = runQueueCleanUp | ||
| 2075 | |||
| 2076 | def sq_task_complete(self, task): | ||
| 2077 | self.sq_stats.taskCompleted() | ||
| 2078 | bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData) | ||
| 2079 | self.sq_task_completeoutright(task) | ||
| 2080 | |||
| 2081 | def sq_task_fail(self, task, result): | ||
| 2082 | self.sq_stats.taskFailed() | ||
| 2083 | bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData) | ||
| 2084 | self.scenequeue_notcovered.add(task) | ||
| 2085 | self.scenequeue_updatecounters(task, True) | ||
| 2086 | self.sq_check_taskfail(task) | ||
| 2087 | |||
| 2088 | def sq_task_failoutright(self, task): | ||
| 2089 | self.sq_running.add(task) | ||
| 2090 | self.sq_buildable.add(task) | ||
| 2091 | self.sq_stats.taskSkipped() | ||
| 2092 | self.sq_stats.taskCompleted() | ||
| 2093 | self.scenequeue_notcovered.add(task) | ||
| 2094 | self.scenequeue_updatecounters(task, True) | ||
| 2095 | |||
| 2096 | def sq_task_skip(self, task): | ||
| 2097 | self.sq_running.add(task) | ||
| 2098 | self.sq_buildable.add(task) | ||
| 2099 | self.sq_task_completeoutright(task) | ||
| 2100 | self.sq_stats.taskSkipped() | ||
| 2101 | self.sq_stats.taskCompleted() | ||
| 2102 | |||
| 2103 | def sq_execute(self): | ||
| 2104 | """ | ||
| 2105 | Run the tasks in a queue prepared by prepare_runqueue | ||
| 2106 | """ | ||
| 2107 | |||
| 2108 | self.rq.read_workers() | ||
| 2109 | |||
| 2110 | task = None | ||
| 2111 | if self.can_start_task(): | ||
| 2112 | # Find the next setscene to run | ||
| 2113 | for nexttask in self.rqdata.runq_setscene_tids: | ||
| 2114 | if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): | ||
| 2115 | if nexttask in self.sqdata.unskippable: | ||
| 2116 | logger.debug(2, "Setscene task %s is unskippable" % nexttask) | ||
| 2117 | if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): | ||
| 2118 | fn = fn_from_tid(nexttask) | ||
| 2119 | foundtarget = False | ||
| 2120 | |||
| 2121 | if nexttask in self.rqdata.target_tids: | ||
| 2122 | foundtarget = True | ||
| 2123 | if not foundtarget: | ||
| 2124 | logger.debug(2, "Skipping setscene for task %s" % nexttask) | ||
| 2125 | self.sq_task_skip(nexttask) | ||
| 2126 | self.scenequeue_notneeded.add(nexttask) | ||
| 2127 | return True | ||
| 2128 | if nexttask in self.sqdata.outrightfail: | ||
| 2129 | self.sq_task_failoutright(nexttask) | ||
| 2130 | return True | ||
| 2131 | task = nexttask | ||
| 2132 | break | ||
| 2133 | if task is not None: | ||
| 2134 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) | ||
| 2135 | taskname = taskname + "_setscene" | ||
| 2136 | if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache): | ||
| 2137 | logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task) | ||
| 2138 | self.sq_task_failoutright(task) | ||
| 2139 | return True | ||
| 2140 | |||
| 2141 | if self.cooker.configuration.force: | ||
| 2142 | if task in self.rqdata.target_tids: | ||
| 2143 | self.sq_task_failoutright(task) | ||
| 2144 | return True | ||
| 2145 | |||
| 2146 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): | ||
| 2147 | logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task) | ||
| 2148 | self.sq_task_skip(task) | ||
| 2149 | return True | ||
| 2150 | |||
| 2151 | if self.cooker.configuration.skipsetscene: | ||
| 2152 | logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task) | ||
| 2153 | self.sq_task_failoutright(task) | ||
| 2154 | return True | ||
| 2155 | |||
| 2156 | startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq) | ||
| 2157 | bb.event.fire(startevent, self.cfgData) | ||
| 2158 | |||
| 2159 | taskdepdata = self.sq_build_taskdepdata(task) | ||
| 2160 | |||
| 2161 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] | ||
| 2162 | taskhash = self.rqdata.get_task_hash(task) | ||
| 2163 | unihash = self.rqdata.get_task_unihash(task) | ||
| 2164 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: | ||
| 2165 | if not mc in self.rq.fakeworker: | ||
| 2166 | self.rq.start_fakeworker(self, mc) | ||
| 2167 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | ||
| 2168 | self.rq.fakeworker[mc].process.stdin.flush() | ||
| 2169 | else: | ||
| 2170 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | ||
| 2171 | self.rq.worker[mc].process.stdin.flush() | ||
| 2172 | |||
| 2173 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) | ||
| 2174 | self.build_stamps2.append(self.build_stamps[task]) | ||
| 2175 | self.sq_running.add(task) | ||
| 2176 | self.sq_live.add(task) | ||
| 2177 | self.sq_stats.taskActive() | ||
| 2178 | if self.can_start_task(): | ||
| 2179 | return True | ||
| 2180 | |||
| 2181 | if self.sq_stats.active > 0: | ||
| 2182 | self.rq.read_workers() | ||
| 2183 | return self.rq.active_fds() | ||
| 2184 | |||
| 2185 | #for tid in self.sqdata.sq_revdeps: | ||
| 2186 | # if tid not in self.sq_running: | ||
| 2187 | # buildable = tid in self.sq_buildable | ||
| 2188 | # revdeps = self.sqdata.sq_revdeps[tid] | ||
| 2189 | # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps))) | ||
| 2190 | |||
| 2191 | self.rq.scenequeue_covered = self.scenequeue_covered | ||
| 2192 | self.rq.scenequeue_notcovered = self.scenequeue_notcovered | ||
| 2193 | |||
| 2194 | logger.debug(1, 'We can skip tasks %s', "\n".join(sorted(self.rq.scenequeue_covered))) | ||
| 2195 | |||
| 2196 | self.rq.state = runQueueRunInit | ||
| 2197 | |||
| 2198 | completeevent = sceneQueueComplete(self.sq_stats, self.rq) | ||
| 2199 | bb.event.fire(completeevent, self.cfgData) | ||
| 2200 | |||
| 2201 | return True | ||
| 2202 | |||
| 2203 | def sq_build_taskdepdata(self, task): | ||
| 2204 | def getsetscenedeps(tid): | ||
| 2205 | deps = set() | ||
| 2206 | (mc, fn, taskname, _) = split_tid_mcfn(tid) | ||
| 2207 | realtid = tid + "_setscene" | ||
| 2208 | idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends | ||
| 2209 | for (depname, idependtask) in idepends: | ||
| 2210 | if depname not in self.rqdata.taskData[mc].build_targets: | ||
| 2211 | continue | ||
| 2212 | |||
| 2213 | depfn = self.rqdata.taskData[mc].build_targets[depname][0] | ||
| 2214 | if depfn is None: | ||
| 2215 | continue | ||
| 2216 | deptid = depfn + ":" + idependtask.replace("_setscene", "") | ||
| 2217 | deps.add(deptid) | ||
| 2218 | return deps | ||
| 2219 | |||
| 2220 | taskdepdata = {} | ||
| 2221 | next = getsetscenedeps(task) | ||
| 2222 | next.add(task) | ||
| 2223 | while next: | ||
| 2224 | additional = [] | ||
| 2225 | for revdep in next: | ||
| 2226 | (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep) | ||
| 2227 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
| 2228 | deps = getsetscenedeps(revdep) | ||
| 2229 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] | ||
| 2230 | taskhash = self.rqdata.runtaskentries[revdep].hash | ||
| 2231 | unihash = self.rqdata.runtaskentries[revdep].unihash | ||
| 2232 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash] | ||
| 2233 | for revdep2 in deps: | ||
| 2234 | if revdep2 not in taskdepdata: | ||
| 2235 | additional.append(revdep2) | ||
| 2236 | next = additional | ||
| 2237 | |||
| 2238 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) | ||
| 2239 | return taskdepdata | ||
| 2240 | |||
| 2103 | class SQData(object): | 2241 | class SQData(object): |
| 2104 | def __init__(self): | 2242 | def __init__(self): |
| 2105 | # SceneQueue dependencies | 2243 | # SceneQueue dependencies |
| @@ -2346,6 +2484,66 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): | |||
| 2346 | sqdata.outrightfail.append(tid) | 2484 | sqdata.outrightfail.append(tid) |
| 2347 | 2485 | ||
| 2348 | 2486 | ||
| 2487 | class RunQueueExecuteDummy(RunQueueExecute): | ||
| 2488 | def __init__(self, rq): | ||
| 2489 | self.rq = rq | ||
| 2490 | self.stats = RunQueueStats(0) | ||
| 2491 | |||
| 2492 | def finish(self): | ||
| 2493 | self.rq.state = runQueueComplete | ||
| 2494 | return | ||
| 2495 | |||
| 2496 | class RunQueueExecuteTasks(RunQueueExecute): | ||
| 2497 | def __init__(self, rq): | ||
| 2498 | RunQueueExecute.__init__(self, rq) | ||
| 2499 | |||
| 2500 | self.stampcache = {} | ||
| 2501 | |||
| 2502 | # Mark initial buildable tasks | ||
| 2503 | for tid in self.rqdata.runtaskentries: | ||
| 2504 | if len(self.rqdata.runtaskentries[tid].depends) == 0: | ||
| 2505 | self.runq_buildable.add(tid) | ||
| 2506 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): | ||
| 2507 | self.rq.scenequeue_covered.add(tid) | ||
| 2508 | |||
| 2509 | found = True | ||
| 2510 | while found: | ||
| 2511 | found = False | ||
| 2512 | for tid in self.rqdata.runtaskentries: | ||
| 2513 | if tid in self.rq.scenequeue_covered: | ||
| 2514 | continue | ||
| 2515 | logger.debug(1, 'Considering %s: %s' % (tid, str(self.rqdata.runtaskentries[tid].revdeps))) | ||
| 2516 | |||
| 2517 | if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered): | ||
| 2518 | if tid in self.rq.scenequeue_notcovered: | ||
| 2519 | continue | ||
| 2520 | found = True | ||
| 2521 | self.rq.scenequeue_covered.add(tid) | ||
| 2522 | |||
| 2523 | logger.debug(1, 'Skip list %s', sorted(self.rq.scenequeue_covered)) | ||
| 2524 | |||
| 2525 | for task in self.rq.scenequeue_notcovered: | ||
| 2526 | logger.debug(1, 'Not skipping task %s', task) | ||
| 2527 | |||
| 2528 | for mc in self.rqdata.dataCaches: | ||
| 2529 | target_pairs = [] | ||
| 2530 | for tid in self.rqdata.target_tids: | ||
| 2531 | (tidmc, fn, taskname, _) = split_tid_mcfn(tid) | ||
| 2532 | if tidmc == mc: | ||
| 2533 | target_pairs.append((fn, taskname)) | ||
| 2534 | |||
| 2535 | event.fire(bb.event.StampUpdate(target_pairs, self.rqdata.dataCaches[mc].stamp), self.cfgData) | ||
| 2536 | |||
| 2537 | schedulers = self.get_schedulers() | ||
| 2538 | for scheduler in schedulers: | ||
| 2539 | if self.scheduler == scheduler.name: | ||
| 2540 | self.sched = scheduler(self, self.rqdata) | ||
| 2541 | logger.debug(1, "Using runqueue scheduler '%s'", scheduler.name) | ||
| 2542 | break | ||
| 2543 | else: | ||
| 2544 | bb.fatal("Invalid scheduler '%s'. Available schedulers: %s" % | ||
| 2545 | (self.scheduler, ", ".join(obj.name for obj in schedulers))) | ||
| 2546 | |||
| 2349 | class RunQueueExecuteScenequeue(RunQueueExecute): | 2547 | class RunQueueExecuteScenequeue(RunQueueExecute): |
| 2350 | def __init__(self, rq): | 2548 | def __init__(self, rq): |
| 2351 | RunQueueExecute.__init__(self, rq) | 2549 | RunQueueExecute.__init__(self, rq) |
| @@ -2368,204 +2566,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
| 2368 | 2566 | ||
| 2369 | self.rq.state = runQueueSceneRun | 2567 | self.rq.state = runQueueSceneRun |
| 2370 | 2568 | ||
| 2371 | def scenequeue_updatecounters(self, task, fail = False): | ||
| 2372 | for dep in self.sqdata.sq_deps[task]: | ||
| 2373 | if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]: | ||
| 2374 | logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep)) | ||
| 2375 | self.scenequeue_updatecounters(dep, fail) | ||
| 2376 | continue | ||
| 2377 | if task not in self.sqdata.sq_revdeps2[dep]: | ||
| 2378 | # May already have been removed by the fail case above | ||
| 2379 | continue | ||
| 2380 | self.sqdata.sq_revdeps2[dep].remove(task) | ||
| 2381 | if len(self.sqdata.sq_revdeps2[dep]) == 0: | ||
| 2382 | self.sq_buildable.add(dep) | ||
| 2383 | |||
| 2384 | def sq_task_completeoutright(self, task): | ||
| 2385 | """ | ||
| 2386 | Mark a task as completed | ||
| 2387 | Look at the reverse dependencies and mark any task with | ||
| 2388 | completed dependencies as buildable | ||
| 2389 | """ | ||
| 2390 | |||
| 2391 | logger.debug(1, 'Found task %s which could be accelerated', task) | ||
| 2392 | self.scenequeue_covered.add(task) | ||
| 2393 | self.scenequeue_updatecounters(task) | ||
| 2394 | |||
| 2395 | def sq_check_taskfail(self, task): | ||
| 2396 | if self.rqdata.setscenewhitelist is not None: | ||
| 2397 | realtask = task.split('_setscene')[0] | ||
| 2398 | (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask) | ||
| 2399 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
| 2400 | if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist): | ||
| 2401 | logger.error('Task %s.%s failed' % (pn, taskname + "_setscene")) | ||
| 2402 | self.rq.state = runQueueCleanUp | ||
| 2403 | |||
| 2404 | def sq_task_complete(self, task): | ||
| 2405 | self.sq_stats.taskCompleted() | ||
| 2406 | bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData) | ||
| 2407 | self.sq_task_completeoutright(task) | ||
| 2408 | |||
| 2409 | def sq_task_fail(self, task, result): | ||
| 2410 | self.sq_stats.taskFailed() | ||
| 2411 | bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData) | ||
| 2412 | self.scenequeue_notcovered.add(task) | ||
| 2413 | self.scenequeue_updatecounters(task, True) | ||
| 2414 | self.sq_check_taskfail(task) | ||
| 2415 | |||
| 2416 | def sq_task_failoutright(self, task): | ||
| 2417 | self.sq_running.add(task) | ||
| 2418 | self.sq_buildable.add(task) | ||
| 2419 | self.sq_stats.taskSkipped() | ||
| 2420 | self.sq_stats.taskCompleted() | ||
| 2421 | self.scenequeue_notcovered.add(task) | ||
| 2422 | self.scenequeue_updatecounters(task, True) | ||
| 2423 | |||
| 2424 | def sq_task_skip(self, task): | ||
| 2425 | self.sq_running.add(task) | ||
| 2426 | self.sq_buildable.add(task) | ||
| 2427 | self.sq_task_completeoutright(task) | ||
| 2428 | self.sq_stats.taskSkipped() | ||
| 2429 | self.sq_stats.taskCompleted() | ||
| 2430 | |||
| 2431 | def sq_execute(self): | ||
| 2432 | """ | ||
| 2433 | Run the tasks in a queue prepared by prepare_runqueue | ||
| 2434 | """ | ||
| 2435 | |||
| 2436 | self.rq.read_workers() | ||
| 2437 | |||
| 2438 | task = None | ||
| 2439 | if self.can_start_task(): | ||
| 2440 | # Find the next setscene to run | ||
| 2441 | for nexttask in self.rqdata.runq_setscene_tids: | ||
| 2442 | if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): | ||
| 2443 | if nexttask in self.sqdata.unskippable: | ||
| 2444 | logger.debug(2, "Setscene task %s is unskippable" % nexttask) | ||
| 2445 | if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]): | ||
| 2446 | fn = fn_from_tid(nexttask) | ||
| 2447 | foundtarget = False | ||
| 2448 | |||
| 2449 | if nexttask in self.rqdata.target_tids: | ||
| 2450 | foundtarget = True | ||
| 2451 | if not foundtarget: | ||
| 2452 | logger.debug(2, "Skipping setscene for task %s" % nexttask) | ||
| 2453 | self.sq_task_skip(nexttask) | ||
| 2454 | self.scenequeue_notneeded.add(nexttask) | ||
| 2455 | return True | ||
| 2456 | if nexttask in self.sqdata.outrightfail: | ||
| 2457 | self.sq_task_failoutright(nexttask) | ||
| 2458 | return True | ||
| 2459 | task = nexttask | ||
| 2460 | break | ||
| 2461 | if task is not None: | ||
| 2462 | (mc, fn, taskname, taskfn) = split_tid_mcfn(task) | ||
| 2463 | taskname = taskname + "_setscene" | ||
| 2464 | if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache): | ||
| 2465 | logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task) | ||
| 2466 | self.sq_task_failoutright(task) | ||
| 2467 | return True | ||
| 2468 | |||
| 2469 | if self.cooker.configuration.force: | ||
| 2470 | if task in self.rqdata.target_tids: | ||
| 2471 | self.sq_task_failoutright(task) | ||
| 2472 | return True | ||
| 2473 | |||
| 2474 | if self.rq.check_stamp_task(task, taskname, cache=self.stampcache): | ||
| 2475 | logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task) | ||
| 2476 | self.sq_task_skip(task) | ||
| 2477 | return True | ||
| 2478 | |||
| 2479 | if self.cooker.configuration.skipsetscene: | ||
| 2480 | logger.debug(2, 'No setscene tasks should be executed. Skipping %s', task) | ||
| 2481 | self.sq_task_failoutright(task) | ||
| 2482 | return True | ||
| 2483 | |||
| 2484 | startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq) | ||
| 2485 | bb.event.fire(startevent, self.cfgData) | ||
| 2486 | |||
| 2487 | taskdepdata = self.sq_build_taskdepdata(task) | ||
| 2488 | |||
| 2489 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] | ||
| 2490 | taskhash = self.rqdata.get_task_hash(task) | ||
| 2491 | unihash = self.rqdata.get_task_unihash(task) | ||
| 2492 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run: | ||
| 2493 | if not mc in self.rq.fakeworker: | ||
| 2494 | self.rq.start_fakeworker(self, mc) | ||
| 2495 | self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | ||
| 2496 | self.rq.fakeworker[mc].process.stdin.flush() | ||
| 2497 | else: | ||
| 2498 | self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collection.get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>") | ||
| 2499 | self.rq.worker[mc].process.stdin.flush() | ||
| 2500 | |||
| 2501 | self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) | ||
| 2502 | self.build_stamps2.append(self.build_stamps[task]) | ||
| 2503 | self.sq_running.add(task) | ||
| 2504 | self.sq_live.add(task) | ||
| 2505 | self.sq_stats.taskActive() | ||
| 2506 | if self.can_start_task(): | ||
| 2507 | return True | ||
| 2508 | |||
| 2509 | if self.sq_stats.active > 0: | ||
| 2510 | self.rq.read_workers() | ||
| 2511 | return self.rq.active_fds() | ||
| 2512 | |||
| 2513 | #for tid in self.sqdata.sq_revdeps: | ||
| 2514 | # if tid not in self.sq_running: | ||
| 2515 | # buildable = tid in self.sq_buildable | ||
| 2516 | # revdeps = self.sqdata.sq_revdeps[tid] | ||
| 2517 | # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps))) | ||
| 2518 | |||
| 2519 | self.rq.scenequeue_covered = self.scenequeue_covered | ||
| 2520 | self.rq.scenequeue_notcovered = self.scenequeue_notcovered | ||
| 2521 | |||
| 2522 | logger.debug(1, 'We can skip tasks %s', "\n".join(sorted(self.rq.scenequeue_covered))) | ||
| 2523 | |||
| 2524 | self.rq.state = runQueueRunInit | ||
| 2525 | |||
| 2526 | completeevent = sceneQueueComplete(self.sq_stats, self.rq) | ||
| 2527 | bb.event.fire(completeevent, self.cfgData) | ||
| 2528 | |||
| 2529 | return True | ||
| 2530 | |||
| 2531 | def sq_build_taskdepdata(self, task): | ||
| 2532 | def getsetscenedeps(tid): | ||
| 2533 | deps = set() | ||
| 2534 | (mc, fn, taskname, _) = split_tid_mcfn(tid) | ||
| 2535 | realtid = tid + "_setscene" | ||
| 2536 | idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends | ||
| 2537 | for (depname, idependtask) in idepends: | ||
| 2538 | if depname not in self.rqdata.taskData[mc].build_targets: | ||
| 2539 | continue | ||
| 2540 | |||
| 2541 | depfn = self.rqdata.taskData[mc].build_targets[depname][0] | ||
| 2542 | if depfn is None: | ||
| 2543 | continue | ||
| 2544 | deptid = depfn + ":" + idependtask.replace("_setscene", "") | ||
| 2545 | deps.add(deptid) | ||
| 2546 | return deps | ||
| 2547 | |||
| 2548 | taskdepdata = {} | ||
| 2549 | next = getsetscenedeps(task) | ||
| 2550 | next.add(task) | ||
| 2551 | while next: | ||
| 2552 | additional = [] | ||
| 2553 | for revdep in next: | ||
| 2554 | (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep) | ||
| 2555 | pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] | ||
| 2556 | deps = getsetscenedeps(revdep) | ||
| 2557 | provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] | ||
| 2558 | taskhash = self.rqdata.runtaskentries[revdep].hash | ||
| 2559 | unihash = self.rqdata.runtaskentries[revdep].unihash | ||
| 2560 | taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash] | ||
| 2561 | for revdep2 in deps: | ||
| 2562 | if revdep2 not in taskdepdata: | ||
| 2563 | additional.append(revdep2) | ||
| 2564 | next = additional | ||
| 2565 | |||
| 2566 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) | ||
| 2567 | return taskdepdata | ||
| 2568 | |||
| 2569 | class TaskFailure(Exception): | 2569 | class TaskFailure(Exception): |
| 2570 | """ | 2570 | """ |
| 2571 | Exception raised when a task in a runqueue fails | 2571 | Exception raised when a task in a runqueue fails |
