diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-07-03 12:06:36 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-07-15 10:28:12 +0100 |
commit | a4dae1741b125669d50f3b60cfab3a5dd3acee31 (patch) | |
tree | 339c35d639dd77e02bc052b7a1bf7ffc9cb98eb5 | |
parent | 76f64f94b355ea438b89bda969b604436c0ea71b (diff) | |
download | poky-a4dae1741b125669d50f3b60cfab3a5dd3acee31.tar.gz |
bitbake: runqueue: Move scenequeue data generation to a separate function
Move the bulk of the scenequeue data generation to its own function
allowing for refactoring of the code.
Create the start of an object to represent this data.
(Bitbake rev: 68326e0426f25a1bbfd5ae3aa278656a3744053e)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | bitbake/lib/bb/runqueue.py | 468 |
1 files changed, 244 insertions, 224 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py index 6ac9fb1678..c9335195cb 100644 --- a/bitbake/lib/bb/runqueue.py +++ b/bitbake/lib/bb/runqueue.py | |||
@@ -2080,267 +2080,287 @@ class RunQueueExecuteTasks(RunQueueExecute): | |||
2080 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) | 2080 | #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n")) |
2081 | return taskdepdata | 2081 | return taskdepdata |
2082 | 2082 | ||
2083 | class RunQueueExecuteScenequeue(RunQueueExecute): | 2083 | class SQData(object): |
2084 | def __init__(self, rq): | 2084 | def __init__(self): |
2085 | RunQueueExecute.__init__(self, rq) | 2085 | # SceneQueue dependencies |
2086 | 2086 | self.sq_deps = {} | |
2087 | self.scenequeue_covered = set() | 2087 | # SceneQueue reverse dependencies |
2088 | self.scenequeue_notcovered = set() | 2088 | self.sq_revdeps = {} |
2089 | self.scenequeue_notneeded = set() | 2089 | # Copy of reverse dependencies used by sq processing code |
2090 | 2090 | self.sq_revdeps2 = {} | |
2091 | # If we don't have any setscene functions, skip this step | 2091 | # Injected inter-setscene task dependencies |
2092 | if len(self.rqdata.runq_setscene_tids) == 0: | ||
2093 | rq.scenequeue_covered = set() | ||
2094 | rq.scenequeue_notcovered = set() | ||
2095 | rq.state = runQueueRunInit | ||
2096 | return | ||
2097 | |||
2098 | self.stats = RunQueueStats(len(self.rqdata.runq_setscene_tids)) | ||
2099 | |||
2100 | sq_revdeps = {} | ||
2101 | sq_revdeps_new = {} | ||
2102 | sq_revdeps_squash = {} | ||
2103 | self.sq_harddeps = {} | 2092 | self.sq_harddeps = {} |
2093 | # Cache of stamp files so duplicates can't run in parallel | ||
2104 | self.stamps = {} | 2094 | self.stamps = {} |
2095 | # Setscene tasks directly depended upon by the build | ||
2096 | self.unskippable = [] | ||
2097 | # List of setscene tasks which aren't present | ||
2098 | self.outrightfail = [] | ||
2099 | # A list of normal tasks a setscene task covers | ||
2100 | self.sq_covered_tasks = {} | ||
2101 | |||
2102 | def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq): | ||
2103 | |||
2104 | sq_revdeps = {} | ||
2105 | sq_revdeps_new = {} | ||
2106 | sq_revdeps_squash = {} | ||
2107 | |||
2108 | # We need to construct a dependency graph for the setscene functions. Intermediate | ||
2109 | # dependencies between the setscene tasks only complicate the code. This code | ||
2110 | # therefore aims to collapse the huge runqueue dependency tree into a smaller one | ||
2111 | # only containing the setscene functions. | ||
2112 | |||
2113 | rqdata.init_progress_reporter.next_stage() | ||
2114 | |||
2115 | # First process the chains up to the first setscene task. | ||
2116 | endpoints = {} | ||
2117 | for tid in rqdata.runtaskentries: | ||
2118 | sq_revdeps[tid] = copy.copy(rqdata.runtaskentries[tid].revdeps) | ||
2119 | sq_revdeps_new[tid] = set() | ||
2120 | if (len(sq_revdeps[tid]) == 0) and tid not in rqdata.runq_setscene_tids: | ||
2121 | #bb.warn("Added endpoint %s" % (tid)) | ||
2122 | endpoints[tid] = set() | ||
2123 | |||
2124 | rqdata.init_progress_reporter.next_stage() | ||
2125 | |||
2126 | # Secondly process the chains between setscene tasks. | ||
2127 | for tid in rqdata.runq_setscene_tids: | ||
2128 | #bb.warn("Added endpoint 2 %s" % (tid)) | ||
2129 | for dep in rqdata.runtaskentries[tid].depends: | ||
2130 | if tid in sq_revdeps[dep]: | ||
2131 | sq_revdeps[dep].remove(tid) | ||
2132 | if dep not in endpoints: | ||
2133 | endpoints[dep] = set() | ||
2134 | #bb.warn(" Added endpoint 3 %s" % (dep)) | ||
2135 | endpoints[dep].add(tid) | ||
2136 | |||
2137 | rqdata.init_progress_reporter.next_stage() | ||
2138 | |||
2139 | def process_endpoints(endpoints): | ||
2140 | newendpoints = {} | ||
2141 | for point, task in endpoints.items(): | ||
2142 | tasks = set() | ||
2143 | if task: | ||
2144 | tasks |= task | ||
2145 | if sq_revdeps_new[point]: | ||
2146 | tasks |= sq_revdeps_new[point] | ||
2147 | sq_revdeps_new[point] = set() | ||
2148 | if point in rqdata.runq_setscene_tids: | ||
2149 | sq_revdeps_new[point] = tasks | ||
2150 | tasks = set() | ||
2151 | continue | ||
2152 | for dep in rqdata.runtaskentries[point].depends: | ||
2153 | if point in sq_revdeps[dep]: | ||
2154 | sq_revdeps[dep].remove(point) | ||
2155 | if tasks: | ||
2156 | sq_revdeps_new[dep] |= tasks | ||
2157 | if len(sq_revdeps[dep]) == 0 and dep not in rqdata.runq_setscene_tids: | ||
2158 | newendpoints[dep] = task | ||
2159 | if len(newendpoints) != 0: | ||
2160 | process_endpoints(newendpoints) | ||
2161 | |||
2162 | process_endpoints(endpoints) | ||
2163 | |||
2164 | rqdata.init_progress_reporter.next_stage() | ||
2165 | |||
2166 | # Build a list of setscene tasks which are "unskippable" | ||
2167 | # These are direct endpoints referenced by the build | ||
2168 | endpoints2 = {} | ||
2169 | sq_revdeps2 = {} | ||
2170 | sq_revdeps_new2 = {} | ||
2171 | def process_endpoints2(endpoints): | ||
2172 | newendpoints = {} | ||
2173 | for point, task in endpoints.items(): | ||
2174 | tasks = set([point]) | ||
2175 | if task: | ||
2176 | tasks |= task | ||
2177 | if sq_revdeps_new2[point]: | ||
2178 | tasks |= sq_revdeps_new2[point] | ||
2179 | sq_revdeps_new2[point] = set() | ||
2180 | if point in rqdata.runq_setscene_tids: | ||
2181 | sq_revdeps_new2[point] = tasks | ||
2182 | for dep in rqdata.runtaskentries[point].depends: | ||
2183 | if point in sq_revdeps2[dep]: | ||
2184 | sq_revdeps2[dep].remove(point) | ||
2185 | if tasks: | ||
2186 | sq_revdeps_new2[dep] |= tasks | ||
2187 | if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in rqdata.runq_setscene_tids: | ||
2188 | newendpoints[dep] = tasks | ||
2189 | if len(newendpoints) != 0: | ||
2190 | process_endpoints2(newendpoints) | ||
2191 | for tid in rqdata.runtaskentries: | ||
2192 | sq_revdeps2[tid] = copy.copy(rqdata.runtaskentries[tid].revdeps) | ||
2193 | sq_revdeps_new2[tid] = set() | ||
2194 | if (len(sq_revdeps2[tid]) == 0) and tid not in rqdata.runq_setscene_tids: | ||
2195 | endpoints2[tid] = set() | ||
2196 | process_endpoints2(endpoints2) | ||
2197 | for tid in rqdata.runq_setscene_tids: | ||
2198 | if sq_revdeps_new2[tid]: | ||
2199 | sqdata.unskippable.append(tid) | ||
2200 | |||
2201 | rqdata.init_progress_reporter.next_stage(len(rqdata.runtaskentries)) | ||
2202 | |||
2203 | for taskcounter, tid in enumerate(rqdata.runtaskentries): | ||
2204 | if tid in rqdata.runq_setscene_tids: | ||
2205 | deps = set() | ||
2206 | for dep in sq_revdeps_new[tid]: | ||
2207 | deps.add(dep) | ||
2208 | sq_revdeps_squash[tid] = deps | ||
2209 | elif len(sq_revdeps_new[tid]) != 0: | ||
2210 | bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") | ||
2211 | rqdata.init_progress_reporter.update(taskcounter) | ||
2212 | |||
2213 | rqdata.init_progress_reporter.next_stage() | ||
2214 | |||
2215 | # Resolve setscene inter-task dependencies | ||
2216 | # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" | ||
2217 | # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies | ||
2218 | for tid in rqdata.runq_setscene_tids: | ||
2219 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) | ||
2220 | realtid = tid + "_setscene" | ||
2221 | idepends = rqdata.taskData[mc].taskentries[realtid].idepends | ||
2222 | sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", rqdata.dataCaches[mc], taskfn, noextra=True) | ||
2223 | for (depname, idependtask) in idepends: | ||
2105 | 2224 | ||
2106 | # We need to construct a dependency graph for the setscene functions. Intermediate | 2225 | if depname not in rqdata.taskData[mc].build_targets: |
2107 | # dependencies between the setscene tasks only complicate the code. This code | 2226 | continue |
2108 | # therefore aims to collapse the huge runqueue dependency tree into a smaller one | ||
2109 | # only containing the setscene functions. | ||
2110 | |||
2111 | self.rqdata.init_progress_reporter.next_stage() | ||
2112 | 2227 | ||
2113 | # First process the chains up to the first setscene task. | 2228 | depfn = rqdata.taskData[mc].build_targets[depname][0] |
2114 | endpoints = {} | 2229 | if depfn is None: |
2115 | for tid in self.rqdata.runtaskentries: | 2230 | continue |
2116 | sq_revdeps[tid] = copy.copy(self.rqdata.runtaskentries[tid].revdeps) | 2231 | deptid = depfn + ":" + idependtask.replace("_setscene", "") |
2117 | sq_revdeps_new[tid] = set() | 2232 | if deptid not in rqdata.runtaskentries: |
2118 | if (len(sq_revdeps[tid]) == 0) and tid not in self.rqdata.runq_setscene_tids: | 2233 | bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask)) |
2119 | #bb.warn("Added endpoint %s" % (tid)) | ||
2120 | endpoints[tid] = set() | ||
2121 | 2234 | ||
2122 | self.rqdata.init_progress_reporter.next_stage() | 2235 | if not deptid in sqdata.sq_harddeps: |
2236 | sqdata.sq_harddeps[deptid] = set() | ||
2237 | sqdata.sq_harddeps[deptid].add(tid) | ||
2123 | 2238 | ||
2124 | # Secondly process the chains between setscene tasks. | 2239 | sq_revdeps_squash[tid].add(deptid) |
2125 | for tid in self.rqdata.runq_setscene_tids: | 2240 | # Have to zero this to avoid circular dependencies |
2126 | #bb.warn("Added endpoint 2 %s" % (tid)) | 2241 | sq_revdeps_squash[deptid] = set() |
2127 | for dep in self.rqdata.runtaskentries[tid].depends: | ||
2128 | if tid in sq_revdeps[dep]: | ||
2129 | sq_revdeps[dep].remove(tid) | ||
2130 | if dep not in endpoints: | ||
2131 | endpoints[dep] = set() | ||
2132 | #bb.warn(" Added endpoint 3 %s" % (dep)) | ||
2133 | endpoints[dep].add(tid) | ||
2134 | |||
2135 | self.rqdata.init_progress_reporter.next_stage() | ||
2136 | |||
2137 | def process_endpoints(endpoints): | ||
2138 | newendpoints = {} | ||
2139 | for point, task in endpoints.items(): | ||
2140 | tasks = set() | ||
2141 | if task: | ||
2142 | tasks |= task | ||
2143 | if sq_revdeps_new[point]: | ||
2144 | tasks |= sq_revdeps_new[point] | ||
2145 | sq_revdeps_new[point] = set() | ||
2146 | if point in self.rqdata.runq_setscene_tids: | ||
2147 | sq_revdeps_new[point] = tasks | ||
2148 | tasks = set() | ||
2149 | continue | ||
2150 | for dep in self.rqdata.runtaskentries[point].depends: | ||
2151 | if point in sq_revdeps[dep]: | ||
2152 | sq_revdeps[dep].remove(point) | ||
2153 | if tasks: | ||
2154 | sq_revdeps_new[dep] |= tasks | ||
2155 | if len(sq_revdeps[dep]) == 0 and dep not in self.rqdata.runq_setscene_tids: | ||
2156 | newendpoints[dep] = task | ||
2157 | if len(newendpoints) != 0: | ||
2158 | process_endpoints(newendpoints) | ||
2159 | |||
2160 | process_endpoints(endpoints) | ||
2161 | |||
2162 | self.rqdata.init_progress_reporter.next_stage() | ||
2163 | |||
2164 | # Build a list of setscene tasks which are "unskippable" | ||
2165 | # These are direct endpoints referenced by the build | ||
2166 | endpoints2 = {} | ||
2167 | sq_revdeps2 = {} | ||
2168 | sq_revdeps_new2 = {} | ||
2169 | def process_endpoints2(endpoints): | ||
2170 | newendpoints = {} | ||
2171 | for point, task in endpoints.items(): | ||
2172 | tasks = set([point]) | ||
2173 | if task: | ||
2174 | tasks |= task | ||
2175 | if sq_revdeps_new2[point]: | ||
2176 | tasks |= sq_revdeps_new2[point] | ||
2177 | sq_revdeps_new2[point] = set() | ||
2178 | if point in self.rqdata.runq_setscene_tids: | ||
2179 | sq_revdeps_new2[point] = tasks | ||
2180 | for dep in self.rqdata.runtaskentries[point].depends: | ||
2181 | if point in sq_revdeps2[dep]: | ||
2182 | sq_revdeps2[dep].remove(point) | ||
2183 | if tasks: | ||
2184 | sq_revdeps_new2[dep] |= tasks | ||
2185 | if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in self.rqdata.runq_setscene_tids: | ||
2186 | newendpoints[dep] = tasks | ||
2187 | if len(newendpoints) != 0: | ||
2188 | process_endpoints2(newendpoints) | ||
2189 | for tid in self.rqdata.runtaskentries: | ||
2190 | sq_revdeps2[tid] = copy.copy(self.rqdata.runtaskentries[tid].revdeps) | ||
2191 | sq_revdeps_new2[tid] = set() | ||
2192 | if (len(sq_revdeps2[tid]) == 0) and tid not in self.rqdata.runq_setscene_tids: | ||
2193 | endpoints2[tid] = set() | ||
2194 | process_endpoints2(endpoints2) | ||
2195 | self.unskippable = [] | ||
2196 | for tid in self.rqdata.runq_setscene_tids: | ||
2197 | if sq_revdeps_new2[tid]: | ||
2198 | self.unskippable.append(tid) | ||
2199 | 2242 | ||
2200 | self.rqdata.init_progress_reporter.next_stage(len(self.rqdata.runtaskentries)) | 2243 | rqdata.init_progress_reporter.next_stage() |
2201 | 2244 | ||
2202 | for taskcounter, tid in enumerate(self.rqdata.runtaskentries): | 2245 | for task in sqdata.sq_harddeps: |
2203 | if tid in self.rqdata.runq_setscene_tids: | 2246 | for dep in sqdata.sq_harddeps[task]: |
2204 | deps = set() | 2247 | sq_revdeps_squash[dep].add(task) |
2205 | for dep in sq_revdeps_new[tid]: | ||
2206 | deps.add(dep) | ||
2207 | sq_revdeps_squash[tid] = deps | ||
2208 | elif len(sq_revdeps_new[tid]) != 0: | ||
2209 | bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") | ||
2210 | self.rqdata.init_progress_reporter.update(taskcounter) | ||
2211 | |||
2212 | self.rqdata.init_progress_reporter.next_stage() | ||
2213 | |||
2214 | # Resolve setscene inter-task dependencies | ||
2215 | # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" | ||
2216 | # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies | ||
2217 | for tid in self.rqdata.runq_setscene_tids: | ||
2218 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) | ||
2219 | realtid = tid + "_setscene" | ||
2220 | idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends | ||
2221 | self.stamps[tid] = bb.build.stampfile(taskname + "_setscene", self.rqdata.dataCaches[mc], taskfn, noextra=True) | ||
2222 | for (depname, idependtask) in idepends: | ||
2223 | 2248 | ||
2224 | if depname not in self.rqdata.taskData[mc].build_targets: | 2249 | rqdata.init_progress_reporter.next_stage() |
2225 | continue | ||
2226 | 2250 | ||
2227 | depfn = self.rqdata.taskData[mc].build_targets[depname][0] | 2251 | #for tid in sq_revdeps_squash: |
2228 | if depfn is None: | 2252 | # for dep in sq_revdeps_squash[tid]: |
2229 | continue | 2253 | # data = data + "\n %s" % dep |
2230 | deptid = depfn + ":" + idependtask.replace("_setscene", "") | 2254 | # bb.warn("Task %s_setscene: is %s " % (tid, data |
2231 | if deptid not in self.rqdata.runtaskentries: | ||
2232 | bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask)) | ||
2233 | 2255 | ||
2234 | if not deptid in self.sq_harddeps: | 2256 | sqdata.sq_revdeps = sq_revdeps_squash |
2235 | self.sq_harddeps[deptid] = set() | 2257 | sqdata.sq_revdeps2 = copy.deepcopy(sqdata.sq_revdeps) |
2236 | self.sq_harddeps[deptid].add(tid) | ||
2237 | 2258 | ||
2238 | sq_revdeps_squash[tid].add(deptid) | 2259 | for tid in sqdata.sq_revdeps: |
2239 | # Have to zero this to avoid circular dependencies | 2260 | sqdata.sq_deps[tid] = set() |
2240 | sq_revdeps_squash[deptid] = set() | 2261 | for tid in sqdata.sq_revdeps: |
2262 | for dep in sqdata.sq_revdeps[tid]: | ||
2263 | sqdata.sq_deps[dep].add(tid) | ||
2241 | 2264 | ||
2242 | self.rqdata.init_progress_reporter.next_stage() | 2265 | rqdata.init_progress_reporter.next_stage() |
2243 | 2266 | ||
2244 | for task in self.sq_harddeps: | 2267 | for tid in sqdata.sq_revdeps: |
2245 | for dep in self.sq_harddeps[task]: | 2268 | if len(sqdata.sq_revdeps[tid]) == 0: |
2246 | sq_revdeps_squash[dep].add(task) | 2269 | sqrq.runq_buildable.add(tid) |
2247 | 2270 | ||
2248 | self.rqdata.init_progress_reporter.next_stage() | 2271 | rqdata.init_progress_reporter.finish() |
2249 | 2272 | ||
2250 | #for tid in sq_revdeps_squash: | 2273 | if rq.hashvalidate: |
2251 | # for dep in sq_revdeps_squash[tid]: | 2274 | sq_hash = [] |
2252 | # data = data + "\n %s" % dep | 2275 | sq_hashfn = [] |
2253 | # bb.warn("Task %s_setscene: is %s " % (tid, data | 2276 | sq_unihash = [] |
2277 | sq_fn = [] | ||
2278 | sq_taskname = [] | ||
2279 | sq_task = [] | ||
2280 | noexec = [] | ||
2281 | stamppresent = [] | ||
2282 | for tid in sqdata.sq_revdeps: | ||
2283 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) | ||
2254 | 2284 | ||
2255 | self.sq_deps = {} | 2285 | taskdep = rqdata.dataCaches[mc].task_deps[taskfn] |
2256 | self.sq_revdeps = sq_revdeps_squash | ||
2257 | self.sq_revdeps2 = copy.deepcopy(self.sq_revdeps) | ||
2258 | 2286 | ||
2259 | for tid in self.sq_revdeps: | 2287 | if 'noexec' in taskdep and taskname in taskdep['noexec']: |
2260 | self.sq_deps[tid] = set() | 2288 | noexec.append(tid) |
2261 | for tid in self.sq_revdeps: | 2289 | sqrq.task_skip(tid) |
2262 | for dep in self.sq_revdeps[tid]: | 2290 | bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn) |
2263 | self.sq_deps[dep].add(tid) | 2291 | continue |
2264 | 2292 | ||
2265 | self.rqdata.init_progress_reporter.next_stage() | 2293 | if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache): |
2294 | logger.debug(2, 'Setscene stamp current for task %s', tid) | ||
2295 | stamppresent.append(tid) | ||
2296 | sqrq.task_skip(tid) | ||
2297 | continue | ||
2266 | 2298 | ||
2267 | for tid in self.sq_revdeps: | 2299 | if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache): |
2268 | if len(self.sq_revdeps[tid]) == 0: | 2300 | logger.debug(2, 'Normal stamp current for task %s', tid) |
2269 | self.runq_buildable.add(tid) | 2301 | stamppresent.append(tid) |
2302 | sqrq.task_skip(tid) | ||
2303 | continue | ||
2270 | 2304 | ||
2271 | self.rqdata.init_progress_reporter.finish() | 2305 | sq_fn.append(fn) |
2306 | sq_hashfn.append(rqdata.dataCaches[mc].hashfn[taskfn]) | ||
2307 | sq_hash.append(rqdata.runtaskentries[tid].hash) | ||
2308 | sq_unihash.append(rqdata.runtaskentries[tid].unihash) | ||
2309 | sq_taskname.append(taskname) | ||
2310 | sq_task.append(tid) | ||
2272 | 2311 | ||
2273 | self.outrightfail = [] | 2312 | cooker.data.setVar("BB_SETSCENE_STAMPCURRENT_COUNT", len(stamppresent)) |
2274 | if self.rq.hashvalidate: | ||
2275 | sq_hash = [] | ||
2276 | sq_hashfn = [] | ||
2277 | sq_unihash = [] | ||
2278 | sq_fn = [] | ||
2279 | sq_taskname = [] | ||
2280 | sq_task = [] | ||
2281 | noexec = [] | ||
2282 | stamppresent = [] | ||
2283 | for tid in self.sq_revdeps: | ||
2284 | (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) | ||
2285 | 2313 | ||
2286 | taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] | 2314 | valid = rq.validate_hash(sq_fn=sq_fn, sq_task=sq_taskname, sq_hash=sq_hash, sq_hashfn=sq_hashfn, |
2315 | siginfo=False, sq_unihash=sq_unihash, d=cooker.data) | ||
2287 | 2316 | ||
2288 | if 'noexec' in taskdep and taskname in taskdep['noexec']: | 2317 | cooker.data.delVar("BB_SETSCENE_STAMPCURRENT_COUNT") |
2289 | noexec.append(tid) | ||
2290 | self.task_skip(tid) | ||
2291 | bb.build.make_stamp(taskname + "_setscene", self.rqdata.dataCaches[mc], taskfn) | ||
2292 | continue | ||
2293 | 2318 | ||
2294 | if self.rq.check_stamp_task(tid, taskname + "_setscene", cache=self.stampcache): | 2319 | valid_new = stamppresent |
2295 | logger.debug(2, 'Setscene stamp current for task %s', tid) | 2320 | for v in valid: |
2296 | stamppresent.append(tid) | 2321 | valid_new.append(sq_task[v]) |
2297 | self.task_skip(tid) | ||
2298 | continue | ||
2299 | 2322 | ||
2300 | if self.rq.check_stamp_task(tid, taskname, recurse = True, cache=self.stampcache): | 2323 | for tid in sqdata.sq_revdeps: |
2301 | logger.debug(2, 'Normal stamp current for task %s', tid) | 2324 | if tid not in valid_new and tid not in noexec: |
2302 | stamppresent.append(tid) | 2325 | logger.debug(2, 'No package found, so skipping setscene task %s', tid) |
2303 | self.task_skip(tid) | 2326 | sqdata.outrightfail.append(tid) |
2304 | continue | ||
2305 | 2327 | ||
2306 | sq_fn.append(fn) | ||
2307 | sq_hashfn.append(self.rqdata.dataCaches[mc].hashfn[taskfn]) | ||
2308 | sq_hash.append(self.rqdata.runtaskentries[tid].hash) | ||
2309 | sq_unihash.append(self.rqdata.runtaskentries[tid].unihash) | ||
2310 | sq_taskname.append(taskname) | ||
2311 | sq_task.append(tid) | ||
2312 | 2328 | ||
2313 | self.cooker.data.setVar("BB_SETSCENE_STAMPCURRENT_COUNT", len(stamppresent)) | 2329 | class RunQueueExecuteScenequeue(RunQueueExecute): |
2330 | def __init__(self, rq): | ||
2331 | RunQueueExecute.__init__(self, rq) | ||
2314 | 2332 | ||
2315 | valid = self.rq.validate_hash(sq_fn=sq_fn, sq_task=sq_taskname, sq_hash=sq_hash, sq_hashfn=sq_hashfn, | 2333 | self.scenequeue_covered = set() |
2316 | siginfo=False, sq_unihash=sq_unihash, d=self.cooker.data) | 2334 | self.scenequeue_notcovered = set() |
2335 | self.scenequeue_notneeded = set() | ||
2317 | 2336 | ||
2318 | self.cooker.data.delVar("BB_SETSCENE_STAMPCURRENT_COUNT") | 2337 | # If we don't have any setscene functions, skip this step |
2338 | if len(self.rqdata.runq_setscene_tids) == 0: | ||
2339 | rq.scenequeue_covered = set() | ||
2340 | rq.scenequeue_notcovered = set() | ||
2341 | rq.state = runQueueRunInit | ||
2342 | return | ||
2319 | 2343 | ||
2320 | valid_new = stamppresent | 2344 | self.stats = RunQueueStats(len(self.rqdata.runq_setscene_tids)) |
2321 | for v in valid: | ||
2322 | valid_new.append(sq_task[v]) | ||
2323 | 2345 | ||
2324 | for tid in self.sq_revdeps: | 2346 | self.sqdata = SQData() |
2325 | if tid not in valid_new and tid not in noexec: | 2347 | build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self) |
2326 | logger.debug(2, 'No package found, so skipping setscene task %s', tid) | ||
2327 | self.outrightfail.append(tid) | ||
2328 | 2348 | ||
2329 | logger.info('Executing SetScene Tasks') | 2349 | logger.info('Executing SetScene Tasks') |
2330 | 2350 | ||
2331 | self.rq.state = runQueueSceneRun | 2351 | self.rq.state = runQueueSceneRun |
2332 | 2352 | ||
2333 | def scenequeue_updatecounters(self, task, fail = False): | 2353 | def scenequeue_updatecounters(self, task, fail = False): |
2334 | for dep in self.sq_deps[task]: | 2354 | for dep in self.sqdata.sq_deps[task]: |
2335 | if fail and task in self.sq_harddeps and dep in self.sq_harddeps[task]: | 2355 | if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]: |
2336 | logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep)) | 2356 | logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task, dep)) |
2337 | self.scenequeue_updatecounters(dep, fail) | 2357 | self.scenequeue_updatecounters(dep, fail) |
2338 | continue | 2358 | continue |
2339 | if task not in self.sq_revdeps2[dep]: | 2359 | if task not in self.sqdata.sq_revdeps2[dep]: |
2340 | # May already have been removed by the fail case above | 2360 | # May already have been removed by the fail case above |
2341 | continue | 2361 | continue |
2342 | self.sq_revdeps2[dep].remove(task) | 2362 | self.sqdata.sq_revdeps2[dep].remove(task) |
2343 | if len(self.sq_revdeps2[dep]) == 0: | 2363 | if len(self.sqdata.sq_revdeps2[dep]) == 0: |
2344 | self.runq_buildable.add(dep) | 2364 | self.runq_buildable.add(dep) |
2345 | 2365 | ||
2346 | def task_completeoutright(self, task): | 2366 | def task_completeoutright(self, task): |
@@ -2401,10 +2421,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2401 | if self.can_start_task(): | 2421 | if self.can_start_task(): |
2402 | # Find the next setscene to run | 2422 | # Find the next setscene to run |
2403 | for nexttask in self.rqdata.runq_setscene_tids: | 2423 | for nexttask in self.rqdata.runq_setscene_tids: |
2404 | if nexttask in self.runq_buildable and nexttask not in self.runq_running and self.stamps[nexttask] not in self.build_stamps.values(): | 2424 | if nexttask in self.runq_buildable and nexttask not in self.runq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values(): |
2405 | if nexttask in self.unskippable: | 2425 | if nexttask in self.sqdata.unskippable: |
2406 | logger.debug(2, "Setscene task %s is unskippable" % nexttask) | 2426 | logger.debug(2, "Setscene task %s is unskippable" % nexttask) |
2407 | if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True): | 2427 | if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask], True): |
2408 | fn = fn_from_tid(nexttask) | 2428 | fn = fn_from_tid(nexttask) |
2409 | foundtarget = False | 2429 | foundtarget = False |
2410 | 2430 | ||
@@ -2415,7 +2435,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2415 | self.task_skip(nexttask) | 2435 | self.task_skip(nexttask) |
2416 | self.scenequeue_notneeded.add(nexttask) | 2436 | self.scenequeue_notneeded.add(nexttask) |
2417 | return True | 2437 | return True |
2418 | if nexttask in self.outrightfail: | 2438 | if nexttask in self.sqdata.outrightfail: |
2419 | self.task_failoutright(nexttask) | 2439 | self.task_failoutright(nexttask) |
2420 | return True | 2440 | return True |
2421 | task = nexttask | 2441 | task = nexttask |
@@ -2471,10 +2491,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute): | |||
2471 | self.rq.read_workers() | 2491 | self.rq.read_workers() |
2472 | return self.rq.active_fds() | 2492 | return self.rq.active_fds() |
2473 | 2493 | ||
2474 | #for tid in self.sq_revdeps: | 2494 | #for tid in self.sqdata.sq_revdeps: |
2475 | # if tid not in self.runq_running: | 2495 | # if tid not in self.runq_running: |
2476 | # buildable = tid in self.runq_buildable | 2496 | # buildable = tid in self.runq_buildable |
2477 | # revdeps = self.sq_revdeps[tid] | 2497 | # revdeps = self.sqdata.sq_revdeps[tid] |
2478 | # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps))) | 2498 | # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps))) |
2479 | 2499 | ||
2480 | self.rq.scenequeue_covered = self.scenequeue_covered | 2500 | self.rq.scenequeue_covered = self.scenequeue_covered |