diff options
Diffstat (limited to 'bitbake/lib/bb/cooker.py')
-rw-r--r-- | bitbake/lib/bb/cooker.py | 758 |
1 files changed, 422 insertions, 336 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index f4ab797edf..c5bfef55d6 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -13,7 +13,6 @@ import sys, os, glob, os.path, re, time | |||
13 | import itertools | 13 | import itertools |
14 | import logging | 14 | import logging |
15 | import multiprocessing | 15 | import multiprocessing |
16 | import sre_constants | ||
17 | import threading | 16 | import threading |
18 | from io import StringIO, UnsupportedOperation | 17 | from io import StringIO, UnsupportedOperation |
19 | from contextlib import closing | 18 | from contextlib import closing |
@@ -23,7 +22,6 @@ from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, | |||
23 | import queue | 22 | import queue |
24 | import signal | 23 | import signal |
25 | import prserv.serv | 24 | import prserv.serv |
26 | import pyinotify | ||
27 | import json | 25 | import json |
28 | import pickle | 26 | import pickle |
29 | import codecs | 27 | import codecs |
@@ -81,7 +79,7 @@ class SkippedPackage: | |||
81 | 79 | ||
82 | 80 | ||
83 | class CookerFeatures(object): | 81 | class CookerFeatures(object): |
84 | _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS] = list(range(3)) | 82 | _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS, RECIPE_SIGGEN_INFO] = list(range(4)) |
85 | 83 | ||
86 | def __init__(self): | 84 | def __init__(self): |
87 | self._features=set() | 85 | self._features=set() |
@@ -104,12 +102,15 @@ class CookerFeatures(object): | |||
104 | 102 | ||
105 | class EventWriter: | 103 | class EventWriter: |
106 | def __init__(self, cooker, eventfile): | 104 | def __init__(self, cooker, eventfile): |
107 | self.file_inited = None | ||
108 | self.cooker = cooker | 105 | self.cooker = cooker |
109 | self.eventfile = eventfile | 106 | self.eventfile = eventfile |
110 | self.event_queue = [] | 107 | self.event_queue = [] |
111 | 108 | ||
112 | def write_event(self, event): | 109 | def write_variables(self): |
110 | with open(self.eventfile, "a") as f: | ||
111 | f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) | ||
112 | |||
113 | def send(self, event): | ||
113 | with open(self.eventfile, "a") as f: | 114 | with open(self.eventfile, "a") as f: |
114 | try: | 115 | try: |
115 | str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8') | 116 | str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8') |
@@ -119,28 +120,6 @@ class EventWriter: | |||
119 | import traceback | 120 | import traceback |
120 | print(err, traceback.format_exc()) | 121 | print(err, traceback.format_exc()) |
121 | 122 | ||
122 | def send(self, event): | ||
123 | if self.file_inited: | ||
124 | # we have the file, just write the event | ||
125 | self.write_event(event) | ||
126 | else: | ||
127 | # init on bb.event.BuildStarted | ||
128 | name = "%s.%s" % (event.__module__, event.__class__.__name__) | ||
129 | if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"): | ||
130 | with open(self.eventfile, "w") as f: | ||
131 | f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) | ||
132 | |||
133 | self.file_inited = True | ||
134 | |||
135 | # write pending events | ||
136 | for evt in self.event_queue: | ||
137 | self.write_event(evt) | ||
138 | |||
139 | # also write the current event | ||
140 | self.write_event(event) | ||
141 | else: | ||
142 | # queue all events until the file is inited | ||
143 | self.event_queue.append(event) | ||
144 | 123 | ||
145 | #============================================================================# | 124 | #============================================================================# |
146 | # BBCooker | 125 | # BBCooker |
@@ -150,8 +129,10 @@ class BBCooker: | |||
150 | Manages one bitbake build run | 129 | Manages one bitbake build run |
151 | """ | 130 | """ |
152 | 131 | ||
153 | def __init__(self, featureSet=None, idleCallBackRegister=None): | 132 | def __init__(self, featureSet=None, server=None): |
154 | self.recipecaches = None | 133 | self.recipecaches = None |
134 | self.baseconfig_valid = False | ||
135 | self.parsecache_valid = False | ||
155 | self.eventlog = None | 136 | self.eventlog = None |
156 | self.skiplist = {} | 137 | self.skiplist = {} |
157 | self.featureset = CookerFeatures() | 138 | self.featureset = CookerFeatures() |
@@ -159,34 +140,22 @@ class BBCooker: | |||
159 | for f in featureSet: | 140 | for f in featureSet: |
160 | self.featureset.setFeature(f) | 141 | self.featureset.setFeature(f) |
161 | 142 | ||
143 | self.orig_syspath = sys.path.copy() | ||
144 | self.orig_sysmodules = [*sys.modules] | ||
145 | |||
162 | self.configuration = bb.cookerdata.CookerConfiguration() | 146 | self.configuration = bb.cookerdata.CookerConfiguration() |
163 | 147 | ||
164 | self.idleCallBackRegister = idleCallBackRegister | 148 | self.process_server = server |
149 | self.idleCallBackRegister = None | ||
150 | self.waitIdle = None | ||
151 | if server: | ||
152 | self.idleCallBackRegister = server.register_idle_function | ||
153 | self.waitIdle = server.wait_for_idle | ||
165 | 154 | ||
166 | bb.debug(1, "BBCooker starting %s" % time.time()) | 155 | bb.debug(1, "BBCooker starting %s" % time.time()) |
167 | sys.stdout.flush() | 156 | |
168 | 157 | self.configwatched = {} | |
169 | self.configwatcher = pyinotify.WatchManager() | 158 | self.parsewatched = {} |
170 | bb.debug(1, "BBCooker pyinotify1 %s" % time.time()) | ||
171 | sys.stdout.flush() | ||
172 | |||
173 | self.configwatcher.bbseen = set() | ||
174 | self.configwatcher.bbwatchedfiles = set() | ||
175 | self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications) | ||
176 | bb.debug(1, "BBCooker pyinotify2 %s" % time.time()) | ||
177 | sys.stdout.flush() | ||
178 | self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \ | ||
179 | pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \ | ||
180 | pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO | ||
181 | self.watcher = pyinotify.WatchManager() | ||
182 | bb.debug(1, "BBCooker pyinotify3 %s" % time.time()) | ||
183 | sys.stdout.flush() | ||
184 | self.watcher.bbseen = set() | ||
185 | self.watcher.bbwatchedfiles = set() | ||
186 | self.notifier = pyinotify.Notifier(self.watcher, self.notifications) | ||
187 | |||
188 | bb.debug(1, "BBCooker pyinotify complete %s" % time.time()) | ||
189 | sys.stdout.flush() | ||
190 | 159 | ||
191 | # If being called by something like tinfoil, we need to clean cached data | 160 | # If being called by something like tinfoil, we need to clean cached data |
192 | # which may now be invalid | 161 | # which may now be invalid |
@@ -197,14 +166,6 @@ class BBCooker: | |||
197 | self.hashserv = None | 166 | self.hashserv = None |
198 | self.hashservaddr = None | 167 | self.hashservaddr = None |
199 | 168 | ||
200 | self.inotify_modified_files = [] | ||
201 | |||
202 | def _process_inotify_updates(server, cooker, abort): | ||
203 | cooker.process_inotify_updates() | ||
204 | return 1.0 | ||
205 | |||
206 | self.idleCallBackRegister(_process_inotify_updates, self) | ||
207 | |||
208 | # TOSTOP must not be set or our children will hang when they output | 169 | # TOSTOP must not be set or our children will hang when they output |
209 | try: | 170 | try: |
210 | fd = sys.stdout.fileno() | 171 | fd = sys.stdout.fileno() |
@@ -218,7 +179,7 @@ class BBCooker: | |||
218 | except UnsupportedOperation: | 179 | except UnsupportedOperation: |
219 | pass | 180 | pass |
220 | 181 | ||
221 | self.command = bb.command.Command(self) | 182 | self.command = bb.command.Command(self, self.process_server) |
222 | self.state = state.initial | 183 | self.state = state.initial |
223 | 184 | ||
224 | self.parser = None | 185 | self.parser = None |
@@ -228,84 +189,37 @@ class BBCooker: | |||
228 | signal.signal(signal.SIGHUP, self.sigterm_exception) | 189 | signal.signal(signal.SIGHUP, self.sigterm_exception) |
229 | 190 | ||
230 | bb.debug(1, "BBCooker startup complete %s" % time.time()) | 191 | bb.debug(1, "BBCooker startup complete %s" % time.time()) |
231 | sys.stdout.flush() | ||
232 | 192 | ||
233 | def init_configdata(self): | 193 | def init_configdata(self): |
234 | if not hasattr(self, "data"): | 194 | if not hasattr(self, "data"): |
235 | self.initConfigurationData() | 195 | self.initConfigurationData() |
236 | bb.debug(1, "BBCooker parsed base configuration %s" % time.time()) | 196 | bb.debug(1, "BBCooker parsed base configuration %s" % time.time()) |
237 | sys.stdout.flush() | ||
238 | self.handlePRServ() | 197 | self.handlePRServ() |
239 | 198 | ||
240 | def process_inotify_updates(self): | 199 | def _baseconfig_set(self, value): |
241 | for n in [self.confignotifier, self.notifier]: | 200 | if value and not self.baseconfig_valid: |
242 | if n.check_events(timeout=0): | 201 | bb.server.process.serverlog("Base config valid") |
243 | # read notified events and enqeue them | 202 | elif not value and self.baseconfig_valid: |
244 | n.read_events() | 203 | bb.server.process.serverlog("Base config invalidated") |
245 | n.process_events() | 204 | self.baseconfig_valid = value |
246 | 205 | ||
247 | def config_notifications(self, event): | 206 | def _parsecache_set(self, value): |
248 | if event.maskname == "IN_Q_OVERFLOW": | 207 | if value and not self.parsecache_valid: |
249 | bb.warn("inotify event queue overflowed, invalidating caches.") | 208 | bb.server.process.serverlog("Parse cache valid") |
250 | self.parsecache_valid = False | 209 | elif not value and self.parsecache_valid: |
251 | self.baseconfig_valid = False | 210 | bb.server.process.serverlog("Parse cache invalidated") |
252 | bb.parse.clear_cache() | 211 | self.parsecache_valid = value |
253 | return | 212 | |
254 | if not event.pathname in self.configwatcher.bbwatchedfiles: | 213 | def add_filewatch(self, deps, configwatcher=False): |
255 | return | 214 | if configwatcher: |
256 | if not event.pathname in self.inotify_modified_files: | 215 | watcher = self.configwatched |
257 | self.inotify_modified_files.append(event.pathname) | 216 | else: |
258 | self.baseconfig_valid = False | 217 | watcher = self.parsewatched |
259 | |||
260 | def notifications(self, event): | ||
261 | if event.maskname == "IN_Q_OVERFLOW": | ||
262 | bb.warn("inotify event queue overflowed, invalidating caches.") | ||
263 | self.parsecache_valid = False | ||
264 | bb.parse.clear_cache() | ||
265 | return | ||
266 | if event.pathname.endswith("bitbake-cookerdaemon.log") \ | ||
267 | or event.pathname.endswith("bitbake.lock"): | ||
268 | return | ||
269 | if not event.pathname in self.inotify_modified_files: | ||
270 | self.inotify_modified_files.append(event.pathname) | ||
271 | self.parsecache_valid = False | ||
272 | 218 | ||
273 | def add_filewatch(self, deps, watcher=None, dirs=False): | ||
274 | if not watcher: | ||
275 | watcher = self.watcher | ||
276 | for i in deps: | 219 | for i in deps: |
277 | watcher.bbwatchedfiles.add(i[0]) | 220 | f = i[0] |
278 | if dirs: | 221 | mtime = i[1] |
279 | f = i[0] | 222 | watcher[f] = mtime |
280 | else: | ||
281 | f = os.path.dirname(i[0]) | ||
282 | if f in watcher.bbseen: | ||
283 | continue | ||
284 | watcher.bbseen.add(f) | ||
285 | watchtarget = None | ||
286 | while True: | ||
287 | # We try and add watches for files that don't exist but if they did, would influence | ||
288 | # the parser. The parent directory of these files may not exist, in which case we need | ||
289 | # to watch any parent that does exist for changes. | ||
290 | try: | ||
291 | watcher.add_watch(f, self.watchmask, quiet=False) | ||
292 | if watchtarget: | ||
293 | watcher.bbwatchedfiles.add(watchtarget) | ||
294 | break | ||
295 | except pyinotify.WatchManagerError as e: | ||
296 | if 'ENOENT' in str(e): | ||
297 | watchtarget = f | ||
298 | f = os.path.dirname(f) | ||
299 | if f in watcher.bbseen: | ||
300 | break | ||
301 | watcher.bbseen.add(f) | ||
302 | continue | ||
303 | if 'ENOSPC' in str(e): | ||
304 | providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?") | ||
305 | providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.") | ||
306 | providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=<value>.") | ||
307 | providerlog.error("Root privilege is required to modify max_user_watches.") | ||
308 | raise | ||
309 | 223 | ||
310 | def sigterm_exception(self, signum, stackframe): | 224 | def sigterm_exception(self, signum, stackframe): |
311 | if signum == signal.SIGTERM: | 225 | if signum == signal.SIGTERM: |
@@ -313,6 +227,7 @@ class BBCooker: | |||
313 | elif signum == signal.SIGHUP: | 227 | elif signum == signal.SIGHUP: |
314 | bb.warn("Cooker received SIGHUP, shutting down...") | 228 | bb.warn("Cooker received SIGHUP, shutting down...") |
315 | self.state = state.forceshutdown | 229 | self.state = state.forceshutdown |
230 | bb.event._should_exit.set() | ||
316 | 231 | ||
317 | def setFeatures(self, features): | 232 | def setFeatures(self, features): |
318 | # we only accept a new feature set if we're in state initial, so we can reset without problems | 233 | # we only accept a new feature set if we're in state initial, so we can reset without problems |
@@ -330,6 +245,13 @@ class BBCooker: | |||
330 | self.state = state.initial | 245 | self.state = state.initial |
331 | self.caches_array = [] | 246 | self.caches_array = [] |
332 | 247 | ||
248 | sys.path = self.orig_syspath.copy() | ||
249 | for mod in [*sys.modules]: | ||
250 | if mod not in self.orig_sysmodules: | ||
251 | del sys.modules[mod] | ||
252 | |||
253 | self.configwatched = {} | ||
254 | |||
333 | # Need to preserve BB_CONSOLELOG over resets | 255 | # Need to preserve BB_CONSOLELOG over resets |
334 | consolelog = None | 256 | consolelog = None |
335 | if hasattr(self, "data"): | 257 | if hasattr(self, "data"): |
@@ -338,12 +260,12 @@ class BBCooker: | |||
338 | if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: | 260 | if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset: |
339 | self.enableDataTracking() | 261 | self.enableDataTracking() |
340 | 262 | ||
341 | all_extra_cache_names = [] | 263 | caches_name_array = ['bb.cache:CoreRecipeInfo'] |
342 | # We hardcode all known cache types in a single place, here. | 264 | # We hardcode all known cache types in a single place, here. |
343 | if CookerFeatures.HOB_EXTRA_CACHES in self.featureset: | 265 | if CookerFeatures.HOB_EXTRA_CACHES in self.featureset: |
344 | all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo") | 266 | caches_name_array.append("bb.cache_extra:HobRecipeInfo") |
345 | 267 | if CookerFeatures.RECIPE_SIGGEN_INFO in self.featureset: | |
346 | caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names | 268 | caches_name_array.append("bb.cache:SiggenRecipeInfo") |
347 | 269 | ||
348 | # At least CoreRecipeInfo will be loaded, so caches_array will never be empty! | 270 | # At least CoreRecipeInfo will be loaded, so caches_array will never be empty! |
349 | # This is the entry point, no further check needed! | 271 | # This is the entry point, no further check needed! |
@@ -362,6 +284,10 @@ class BBCooker: | |||
362 | self.data_hash = self.databuilder.data_hash | 284 | self.data_hash = self.databuilder.data_hash |
363 | self.extraconfigdata = {} | 285 | self.extraconfigdata = {} |
364 | 286 | ||
287 | eventlog = self.data.getVar("BB_DEFAULT_EVENTLOG") | ||
288 | if not self.configuration.writeeventlog and eventlog: | ||
289 | self.setupEventLog(eventlog) | ||
290 | |||
365 | if consolelog: | 291 | if consolelog: |
366 | self.data.setVar("BB_CONSOLELOG", consolelog) | 292 | self.data.setVar("BB_CONSOLELOG", consolelog) |
367 | 293 | ||
@@ -371,31 +297,42 @@ class BBCooker: | |||
371 | self.disableDataTracking() | 297 | self.disableDataTracking() |
372 | 298 | ||
373 | for mc in self.databuilder.mcdata.values(): | 299 | for mc in self.databuilder.mcdata.values(): |
374 | mc.renameVar("__depends", "__base_depends") | 300 | self.add_filewatch(mc.getVar("__base_depends", False), configwatcher=True) |
375 | self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher) | ||
376 | 301 | ||
377 | self.baseconfig_valid = True | 302 | self._baseconfig_set(True) |
378 | self.parsecache_valid = False | 303 | self._parsecache_set(False) |
379 | 304 | ||
380 | def handlePRServ(self): | 305 | def handlePRServ(self): |
381 | # Setup a PR Server based on the new configuration | 306 | # Setup a PR Server based on the new configuration |
382 | try: | 307 | try: |
383 | self.prhost = prserv.serv.auto_start(self.data) | 308 | self.prhost = prserv.serv.auto_start(self.data) |
384 | except prserv.serv.PRServiceConfigError as e: | 309 | except prserv.serv.PRServiceConfigError as e: |
385 | bb.fatal("Unable to start PR Server, exitting") | 310 | bb.fatal("Unable to start PR Server, exiting, check the bitbake-cookerdaemon.log") |
386 | 311 | ||
387 | if self.data.getVar("BB_HASHSERVE") == "auto": | 312 | if self.data.getVar("BB_HASHSERVE") == "auto": |
388 | # Create a new hash server bound to a unix domain socket | 313 | # Create a new hash server bound to a unix domain socket |
389 | if not self.hashserv: | 314 | if not self.hashserv: |
390 | dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db" | 315 | dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db" |
316 | upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None | ||
317 | if upstream: | ||
318 | import socket | ||
319 | try: | ||
320 | sock = socket.create_connection(upstream.split(":"), 5) | ||
321 | sock.close() | ||
322 | except socket.error as e: | ||
323 | bb.warn("BB_HASHSERVE_UPSTREAM is not valid, unable to connect hash equivalence server at '%s': %s" | ||
324 | % (upstream, repr(e))) | ||
325 | |||
391 | self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR") | 326 | self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR") |
392 | self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False) | 327 | self.hashserv = hashserv.create_server( |
393 | self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever) | 328 | self.hashservaddr, |
394 | self.hashserv.process.start() | 329 | dbfile, |
395 | self.data.setVar("BB_HASHSERVE", self.hashservaddr) | 330 | sync=False, |
396 | self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr) | 331 | upstream=upstream, |
397 | self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr) | 332 | ) |
333 | self.hashserv.serve_as_process(log_level=logging.WARNING) | ||
398 | for mc in self.databuilder.mcdata: | 334 | for mc in self.databuilder.mcdata: |
335 | self.databuilder.mcorigdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) | ||
399 | self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) | 336 | self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr) |
400 | 337 | ||
401 | bb.parse.init_parser(self.data) | 338 | bb.parse.init_parser(self.data) |
@@ -410,6 +347,29 @@ class BBCooker: | |||
410 | if hasattr(self, "data"): | 347 | if hasattr(self, "data"): |
411 | self.data.disableTracking() | 348 | self.data.disableTracking() |
412 | 349 | ||
350 | def revalidateCaches(self): | ||
351 | bb.parse.clear_cache() | ||
352 | |||
353 | clean = True | ||
354 | for f in self.configwatched: | ||
355 | if not bb.parse.check_mtime(f, self.configwatched[f]): | ||
356 | bb.server.process.serverlog("Found %s changed, invalid cache" % f) | ||
357 | self._baseconfig_set(False) | ||
358 | self._parsecache_set(False) | ||
359 | clean = False | ||
360 | break | ||
361 | |||
362 | if clean: | ||
363 | for f in self.parsewatched: | ||
364 | if not bb.parse.check_mtime(f, self.parsewatched[f]): | ||
365 | bb.server.process.serverlog("Found %s changed, invalid cache" % f) | ||
366 | self._parsecache_set(False) | ||
367 | clean = False | ||
368 | break | ||
369 | |||
370 | if not clean: | ||
371 | bb.parse.BBHandler.cached_statements = {} | ||
372 | |||
413 | def parseConfiguration(self): | 373 | def parseConfiguration(self): |
414 | self.updateCacheSync() | 374 | self.updateCacheSync() |
415 | 375 | ||
@@ -428,8 +388,24 @@ class BBCooker: | |||
428 | self.recipecaches[mc] = bb.cache.CacheData(self.caches_array) | 388 | self.recipecaches[mc] = bb.cache.CacheData(self.caches_array) |
429 | 389 | ||
430 | self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS")) | 390 | self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS")) |
431 | 391 | self.collections = {} | |
432 | self.parsecache_valid = False | 392 | for mc in self.multiconfigs: |
393 | self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) | ||
394 | |||
395 | self._parsecache_set(False) | ||
396 | |||
397 | def setupEventLog(self, eventlog): | ||
398 | if self.eventlog and self.eventlog[0] != eventlog: | ||
399 | bb.event.unregister_UIHhandler(self.eventlog[1]) | ||
400 | self.eventlog = None | ||
401 | if not self.eventlog or self.eventlog[0] != eventlog: | ||
402 | # we log all events to a file if so directed | ||
403 | # register the log file writer as UI Handler | ||
404 | if not os.path.exists(os.path.dirname(eventlog)): | ||
405 | bb.utils.mkdirhier(os.path.dirname(eventlog)) | ||
406 | writer = EventWriter(self, eventlog) | ||
407 | EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event']) | ||
408 | self.eventlog = (eventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)), writer) | ||
433 | 409 | ||
434 | def updateConfigOpts(self, options, environment, cmdline): | 410 | def updateConfigOpts(self, options, environment, cmdline): |
435 | self.ui_cmdline = cmdline | 411 | self.ui_cmdline = cmdline |
@@ -450,14 +426,7 @@ class BBCooker: | |||
450 | setattr(self.configuration, o, options[o]) | 426 | setattr(self.configuration, o, options[o]) |
451 | 427 | ||
452 | if self.configuration.writeeventlog: | 428 | if self.configuration.writeeventlog: |
453 | if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog: | 429 | self.setupEventLog(self.configuration.writeeventlog) |
454 | bb.event.unregister_UIHhandler(self.eventlog[1]) | ||
455 | if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog: | ||
456 | # we log all events to a file if so directed | ||
457 | # register the log file writer as UI Handler | ||
458 | writer = EventWriter(self, self.configuration.writeeventlog) | ||
459 | EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event']) | ||
460 | self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer))) | ||
461 | 430 | ||
462 | bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel | 431 | bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel |
463 | bb.msg.loggerDefaultDomains = self.configuration.debug_domains | 432 | bb.msg.loggerDefaultDomains = self.configuration.debug_domains |
@@ -487,37 +456,37 @@ class BBCooker: | |||
487 | # Now update all the variables not in the datastore to match | 456 | # Now update all the variables not in the datastore to match |
488 | self.configuration.env = environment | 457 | self.configuration.env = environment |
489 | 458 | ||
459 | self.revalidateCaches() | ||
490 | if not clean: | 460 | if not clean: |
491 | logger.debug("Base environment change, triggering reparse") | 461 | logger.debug("Base environment change, triggering reparse") |
492 | self.reset() | 462 | self.reset() |
493 | 463 | ||
494 | def runCommands(self, server, data, abort): | ||
495 | """ | ||
496 | Run any queued asynchronous command | ||
497 | This is done by the idle handler so it runs in true context rather than | ||
498 | tied to any UI. | ||
499 | """ | ||
500 | |||
501 | return self.command.runAsyncCommand() | ||
502 | |||
503 | def showVersions(self): | 464 | def showVersions(self): |
504 | 465 | ||
505 | (latest_versions, preferred_versions) = self.findProviders() | 466 | (latest_versions, preferred_versions, required) = self.findProviders() |
506 | 467 | ||
507 | logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version") | 468 | logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version") |
508 | logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================") | 469 | logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================") |
509 | 470 | ||
510 | for p in sorted(self.recipecaches[''].pkg_pn): | 471 | for p in sorted(self.recipecaches[''].pkg_pn): |
511 | pref = preferred_versions[p] | 472 | preferred = preferred_versions[p] |
512 | latest = latest_versions[p] | 473 | latest = latest_versions[p] |
474 | requiredstr = "" | ||
475 | preferredstr = "" | ||
476 | if required[p]: | ||
477 | if preferred[0] is not None: | ||
478 | requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2] | ||
479 | else: | ||
480 | bb.fatal("REQUIRED_VERSION of package %s not available" % p) | ||
481 | else: | ||
482 | preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2] | ||
513 | 483 | ||
514 | prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2] | ||
515 | lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2] | 484 | lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2] |
516 | 485 | ||
517 | if pref == latest: | 486 | if preferred == latest: |
518 | prefstr = "" | 487 | preferredstr = "" |
519 | 488 | ||
520 | logger.plain("%-35s %25s %25s", p, lateststr, prefstr) | 489 | logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr) |
521 | 490 | ||
522 | def showEnvironment(self, buildfile=None, pkgs_to_build=None): | 491 | def showEnvironment(self, buildfile=None, pkgs_to_build=None): |
523 | """ | 492 | """ |
@@ -533,6 +502,8 @@ class BBCooker: | |||
533 | if not orig_tracking: | 502 | if not orig_tracking: |
534 | self.enableDataTracking() | 503 | self.enableDataTracking() |
535 | self.reset() | 504 | self.reset() |
505 | # reset() resets to the UI requested value so we have to redo this | ||
506 | self.enableDataTracking() | ||
536 | 507 | ||
537 | def mc_base(p): | 508 | def mc_base(p): |
538 | if p.startswith('mc:'): | 509 | if p.startswith('mc:'): |
@@ -556,21 +527,21 @@ class BBCooker: | |||
556 | if pkgs_to_build[0] in set(ignore.split()): | 527 | if pkgs_to_build[0] in set(ignore.split()): |
557 | bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) | 528 | bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0]) |
558 | 529 | ||
559 | taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True) | 530 | taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.halt, allowincomplete=True) |
560 | 531 | ||
561 | mc = runlist[0][0] | 532 | mc = runlist[0][0] |
562 | fn = runlist[0][3] | 533 | fn = runlist[0][3] |
563 | 534 | ||
564 | if fn: | 535 | if fn: |
565 | try: | 536 | try: |
566 | bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) | 537 | layername = self.collections[mc].calc_bbfile_priority(fn)[2] |
567 | envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn)) | 538 | envdata = self.databuilder.parseRecipe(fn, self.collections[mc].get_file_appends(fn), layername) |
568 | except Exception as e: | 539 | except Exception as e: |
569 | parselog.exception("Unable to read %s", fn) | 540 | parselog.exception("Unable to read %s", fn) |
570 | raise | 541 | raise |
571 | else: | 542 | else: |
572 | if not mc in self.databuilder.mcdata: | 543 | if not mc in self.databuilder.mcdata: |
573 | bb.fatal('Not multiconfig named "%s" found' % mc) | 544 | bb.fatal('No multiconfig named "%s" found' % mc) |
574 | envdata = self.databuilder.mcdata[mc] | 545 | envdata = self.databuilder.mcdata[mc] |
575 | data.expandKeys(envdata) | 546 | data.expandKeys(envdata) |
576 | parse.ast.runAnonFuncs(envdata) | 547 | parse.ast.runAnonFuncs(envdata) |
@@ -585,7 +556,7 @@ class BBCooker: | |||
585 | data.emit_env(env, envdata, True) | 556 | data.emit_env(env, envdata, True) |
586 | logger.plain(env.getvalue()) | 557 | logger.plain(env.getvalue()) |
587 | 558 | ||
588 | # emit the metadata which isnt valid shell | 559 | # emit the metadata which isn't valid shell |
589 | for e in sorted(envdata.keys()): | 560 | for e in sorted(envdata.keys()): |
590 | if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False): | 561 | if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False): |
591 | logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False)) | 562 | logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False)) |
@@ -594,7 +565,7 @@ class BBCooker: | |||
594 | self.disableDataTracking() | 565 | self.disableDataTracking() |
595 | self.reset() | 566 | self.reset() |
596 | 567 | ||
597 | def buildTaskData(self, pkgs_to_build, task, abort, allowincomplete=False): | 568 | def buildTaskData(self, pkgs_to_build, task, halt, allowincomplete=False): |
598 | """ | 569 | """ |
599 | Prepare a runqueue and taskdata object for iteration over pkgs_to_build | 570 | Prepare a runqueue and taskdata object for iteration over pkgs_to_build |
600 | """ | 571 | """ |
@@ -641,7 +612,7 @@ class BBCooker: | |||
641 | localdata = {} | 612 | localdata = {} |
642 | 613 | ||
643 | for mc in self.multiconfigs: | 614 | for mc in self.multiconfigs: |
644 | taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete) | 615 | taskdata[mc] = bb.taskdata.TaskData(halt, skiplist=self.skiplist, allowincomplete=allowincomplete) |
645 | localdata[mc] = data.createCopy(self.databuilder.mcdata[mc]) | 616 | localdata[mc] = data.createCopy(self.databuilder.mcdata[mc]) |
646 | bb.data.expandKeys(localdata[mc]) | 617 | bb.data.expandKeys(localdata[mc]) |
647 | 618 | ||
@@ -690,19 +661,18 @@ class BBCooker: | |||
690 | taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) | 661 | taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) |
691 | mcdeps |= set(taskdata[mc].get_mcdepends()) | 662 | mcdeps |= set(taskdata[mc].get_mcdepends()) |
692 | new = False | 663 | new = False |
693 | for mc in self.multiconfigs: | 664 | for k in mcdeps: |
694 | for k in mcdeps: | 665 | if k in seen: |
695 | if k in seen: | 666 | continue |
696 | continue | 667 | l = k.split(':') |
697 | l = k.split(':') | 668 | depmc = l[2] |
698 | depmc = l[2] | 669 | if depmc not in self.multiconfigs: |
699 | if depmc not in self.multiconfigs: | 670 | bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc)) |
700 | bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc)) | 671 | else: |
701 | else: | 672 | logger.debug("Adding providers for multiconfig dependency %s" % l[3]) |
702 | logger.debug("Adding providers for multiconfig dependency %s" % l[3]) | 673 | taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3]) |
703 | taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3]) | 674 | seen.add(k) |
704 | seen.add(k) | 675 | new = True |
705 | new = True | ||
706 | 676 | ||
707 | for mc in self.multiconfigs: | 677 | for mc in self.multiconfigs: |
708 | taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) | 678 | taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc]) |
@@ -715,7 +685,7 @@ class BBCooker: | |||
715 | Prepare a runqueue and taskdata object for iteration over pkgs_to_build | 685 | Prepare a runqueue and taskdata object for iteration over pkgs_to_build |
716 | """ | 686 | """ |
717 | 687 | ||
718 | # We set abort to False here to prevent unbuildable targets raising | 688 | # We set halt to False here to prevent unbuildable targets raising |
719 | # an exception when we're just generating data | 689 | # an exception when we're just generating data |
720 | taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True) | 690 | taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True) |
721 | 691 | ||
@@ -792,7 +762,9 @@ class BBCooker: | |||
792 | for dep in rq.rqdata.runtaskentries[tid].depends: | 762 | for dep in rq.rqdata.runtaskentries[tid].depends: |
793 | (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep) | 763 | (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep) |
794 | deppn = self.recipecaches[depmc].pkg_fn[deptaskfn] | 764 | deppn = self.recipecaches[depmc].pkg_fn[deptaskfn] |
795 | depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep))) | 765 | if depmc: |
766 | depmc = "mc:" + depmc + ":" | ||
767 | depend_tree["tdepends"][dotname].append("%s%s.%s" % (depmc, deppn, bb.runqueue.taskname_from_tid(dep))) | ||
796 | if taskfn not in seen_fns: | 768 | if taskfn not in seen_fns: |
797 | seen_fns.append(taskfn) | 769 | seen_fns.append(taskfn) |
798 | packages = [] | 770 | packages = [] |
@@ -1056,6 +1028,11 @@ class BBCooker: | |||
1056 | if matches: | 1028 | if matches: |
1057 | bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) | 1029 | bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) |
1058 | 1030 | ||
1031 | def testCookerCommandEvent(self, filepattern): | ||
1032 | # Dummy command used by OEQA selftest to test tinfoil without IO | ||
1033 | matches = ["A", "B"] | ||
1034 | bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) | ||
1035 | |||
1059 | def findProviders(self, mc=''): | 1036 | def findProviders(self, mc=''): |
1060 | return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn) | 1037 | return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn) |
1061 | 1038 | ||
@@ -1063,10 +1040,16 @@ class BBCooker: | |||
1063 | if pn in self.recipecaches[mc].providers: | 1040 | if pn in self.recipecaches[mc].providers: |
1064 | filenames = self.recipecaches[mc].providers[pn] | 1041 | filenames = self.recipecaches[mc].providers[pn] |
1065 | eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc]) | 1042 | eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc]) |
1066 | filename = eligible[0] | 1043 | if eligible is not None: |
1044 | filename = eligible[0] | ||
1045 | else: | ||
1046 | filename = None | ||
1067 | return None, None, None, filename | 1047 | return None, None, None, filename |
1068 | elif pn in self.recipecaches[mc].pkg_pn: | 1048 | elif pn in self.recipecaches[mc].pkg_pn: |
1069 | return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn) | 1049 | (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn) |
1050 | if required and preferred_file is None: | ||
1051 | return None, None, None, None | ||
1052 | return (latest, latest_f, preferred_ver, preferred_file) | ||
1070 | else: | 1053 | else: |
1071 | return None, None, None, None | 1054 | return None, None, None, None |
1072 | 1055 | ||
@@ -1211,15 +1194,15 @@ class BBCooker: | |||
1211 | except bb.utils.VersionStringException as vse: | 1194 | except bb.utils.VersionStringException as vse: |
1212 | bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse))) | 1195 | bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse))) |
1213 | if not res: | 1196 | if not res: |
1214 | parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver) | 1197 | parselog.debug3("Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver) |
1215 | continue | 1198 | continue |
1216 | else: | 1199 | else: |
1217 | parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec) | 1200 | parselog.debug3("Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec) |
1218 | continue | 1201 | continue |
1219 | parselog.debug(3,"Layer '%s' recommends layer '%s', so we are adding it", c, rec) | 1202 | parselog.debug3("Layer '%s' recommends layer '%s', so we are adding it", c, rec) |
1220 | collection_depends[c].append(rec) | 1203 | collection_depends[c].append(rec) |
1221 | else: | 1204 | else: |
1222 | parselog.debug(3,"Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec) | 1205 | parselog.debug3("Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec) |
1223 | 1206 | ||
1224 | # Recursively work out collection priorities based on dependencies | 1207 | # Recursively work out collection priorities based on dependencies |
1225 | def calc_layer_priority(collection): | 1208 | def calc_layer_priority(collection): |
@@ -1231,7 +1214,7 @@ class BBCooker: | |||
1231 | if depprio > max_depprio: | 1214 | if depprio > max_depprio: |
1232 | max_depprio = depprio | 1215 | max_depprio = depprio |
1233 | max_depprio += 1 | 1216 | max_depprio += 1 |
1234 | parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio) | 1217 | parselog.debug("Calculated priority of layer %s as %d", collection, max_depprio) |
1235 | collection_priorities[collection] = max_depprio | 1218 | collection_priorities[collection] = max_depprio |
1236 | 1219 | ||
1237 | # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities | 1220 | # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities |
@@ -1243,7 +1226,7 @@ class BBCooker: | |||
1243 | errors = True | 1226 | errors = True |
1244 | continue | 1227 | continue |
1245 | elif regex == "": | 1228 | elif regex == "": |
1246 | parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c) | 1229 | parselog.debug("BBFILE_PATTERN_%s is empty" % c) |
1247 | cre = re.compile('^NULL$') | 1230 | cre = re.compile('^NULL$') |
1248 | errors = False | 1231 | errors = False |
1249 | else: | 1232 | else: |
@@ -1290,8 +1273,8 @@ class BBCooker: | |||
1290 | if bf.startswith("/") or bf.startswith("../"): | 1273 | if bf.startswith("/") or bf.startswith("../"): |
1291 | bf = os.path.abspath(bf) | 1274 | bf = os.path.abspath(bf) |
1292 | 1275 | ||
1293 | self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} | 1276 | collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)} |
1294 | filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) | 1277 | filelist, masked, searchdirs = collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) |
1295 | try: | 1278 | try: |
1296 | os.stat(bf) | 1279 | os.stat(bf) |
1297 | bf = os.path.abspath(bf) | 1280 | bf = os.path.abspath(bf) |
@@ -1357,7 +1340,8 @@ class BBCooker: | |||
1357 | 1340 | ||
1358 | bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) | 1341 | bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array) |
1359 | 1342 | ||
1360 | infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn)) | 1343 | layername = self.collections[mc].calc_bbfile_priority(fn)[2] |
1344 | infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn), layername) | ||
1361 | infos = dict(infos) | 1345 | infos = dict(infos) |
1362 | 1346 | ||
1363 | fn = bb.cache.realfn2virtual(fn, cls, mc) | 1347 | fn = bb.cache.realfn2virtual(fn, cls, mc) |
@@ -1383,14 +1367,16 @@ class BBCooker: | |||
1383 | self.recipecaches[mc].rundeps[fn] = defaultdict(list) | 1367 | self.recipecaches[mc].rundeps[fn] = defaultdict(list) |
1384 | self.recipecaches[mc].runrecs[fn] = defaultdict(list) | 1368 | self.recipecaches[mc].runrecs[fn] = defaultdict(list) |
1385 | 1369 | ||
1370 | bb.parse.siggen.setup_datacache(self.recipecaches) | ||
1371 | |||
1386 | # Invalidate task for target if force mode active | 1372 | # Invalidate task for target if force mode active |
1387 | if self.configuration.force: | 1373 | if self.configuration.force: |
1388 | logger.verbose("Invalidate task %s, %s", task, fn) | 1374 | logger.verbose("Invalidate task %s, %s", task, fn) |
1389 | bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn) | 1375 | bb.parse.siggen.invalidate_task(task, fn) |
1390 | 1376 | ||
1391 | # Setup taskdata structure | 1377 | # Setup taskdata structure |
1392 | taskdata = {} | 1378 | taskdata = {} |
1393 | taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort) | 1379 | taskdata[mc] = bb.taskdata.TaskData(self.configuration.halt) |
1394 | taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item) | 1380 | taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item) |
1395 | 1381 | ||
1396 | if quietlog: | 1382 | if quietlog: |
@@ -1400,17 +1386,20 @@ class BBCooker: | |||
1400 | buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME") | 1386 | buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME") |
1401 | if fireevents: | 1387 | if fireevents: |
1402 | bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc]) | 1388 | bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc]) |
1389 | if self.eventlog: | ||
1390 | self.eventlog[2].write_variables() | ||
1391 | bb.event.enable_heartbeat() | ||
1403 | 1392 | ||
1404 | # Execute the runqueue | 1393 | # Execute the runqueue |
1405 | runlist = [[mc, item, task, fn]] | 1394 | runlist = [[mc, item, task, fn]] |
1406 | 1395 | ||
1407 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) | 1396 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) |
1408 | 1397 | ||
1409 | def buildFileIdle(server, rq, abort): | 1398 | def buildFileIdle(server, rq, halt): |
1410 | 1399 | ||
1411 | msg = None | 1400 | msg = None |
1412 | interrupted = 0 | 1401 | interrupted = 0 |
1413 | if abort or self.state == state.forceshutdown: | 1402 | if halt or self.state == state.forceshutdown: |
1414 | rq.finish_runqueue(True) | 1403 | rq.finish_runqueue(True) |
1415 | msg = "Forced shutdown" | 1404 | msg = "Forced shutdown" |
1416 | interrupted = 2 | 1405 | interrupted = 2 |
@@ -1425,37 +1414,68 @@ class BBCooker: | |||
1425 | failures += len(exc.args) | 1414 | failures += len(exc.args) |
1426 | retval = False | 1415 | retval = False |
1427 | except SystemExit as exc: | 1416 | except SystemExit as exc: |
1428 | self.command.finishAsyncCommand(str(exc)) | ||
1429 | if quietlog: | 1417 | if quietlog: |
1430 | bb.runqueue.logger.setLevel(rqloglevel) | 1418 | bb.runqueue.logger.setLevel(rqloglevel) |
1431 | return False | 1419 | return bb.server.process.idleFinish(str(exc)) |
1432 | 1420 | ||
1433 | if not retval: | 1421 | if not retval: |
1434 | if fireevents: | 1422 | if fireevents: |
1435 | bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc]) | 1423 | bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc]) |
1436 | self.command.finishAsyncCommand(msg) | 1424 | bb.event.disable_heartbeat() |
1437 | # We trashed self.recipecaches above | 1425 | # We trashed self.recipecaches above |
1438 | self.parsecache_valid = False | 1426 | self._parsecache_set(False) |
1439 | self.configuration.limited_deps = False | 1427 | self.configuration.limited_deps = False |
1440 | bb.parse.siggen.reset(self.data) | 1428 | bb.parse.siggen.reset(self.data) |
1441 | if quietlog: | 1429 | if quietlog: |
1442 | bb.runqueue.logger.setLevel(rqloglevel) | 1430 | bb.runqueue.logger.setLevel(rqloglevel) |
1443 | return False | 1431 | return bb.server.process.idleFinish(msg) |
1444 | if retval is True: | 1432 | if retval is True: |
1445 | return True | 1433 | return True |
1446 | return retval | 1434 | return retval |
1447 | 1435 | ||
1448 | self.idleCallBackRegister(buildFileIdle, rq) | 1436 | self.idleCallBackRegister(buildFileIdle, rq) |
1449 | 1437 | ||
1438 | def getTaskSignatures(self, target, tasks): | ||
1439 | sig = [] | ||
1440 | getAllTaskSignatures = False | ||
1441 | |||
1442 | if not tasks: | ||
1443 | tasks = ["do_build"] | ||
1444 | getAllTaskSignatures = True | ||
1445 | |||
1446 | for task in tasks: | ||
1447 | taskdata, runlist = self.buildTaskData(target, task, self.configuration.halt) | ||
1448 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) | ||
1449 | rq.rqdata.prepare() | ||
1450 | |||
1451 | for l in runlist: | ||
1452 | mc, pn, taskname, fn = l | ||
1453 | |||
1454 | taskdep = rq.rqdata.dataCaches[mc].task_deps[fn] | ||
1455 | for t in taskdep['tasks']: | ||
1456 | if t in taskdep['nostamp'] or "setscene" in t: | ||
1457 | continue | ||
1458 | tid = bb.runqueue.build_tid(mc, fn, t) | ||
1459 | |||
1460 | if t in task or getAllTaskSignatures: | ||
1461 | try: | ||
1462 | rq.rqdata.prepare_task_hash(tid) | ||
1463 | sig.append([pn, t, rq.rqdata.get_task_unihash(tid)]) | ||
1464 | except KeyError: | ||
1465 | sig.append(self.getTaskSignatures(target, [t])[0]) | ||
1466 | |||
1467 | return sig | ||
1468 | |||
1450 | def buildTargets(self, targets, task): | 1469 | def buildTargets(self, targets, task): |
1451 | """ | 1470 | """ |
1452 | Attempt to build the targets specified | 1471 | Attempt to build the targets specified |
1453 | """ | 1472 | """ |
1454 | 1473 | ||
1455 | def buildTargetsIdle(server, rq, abort): | 1474 | def buildTargetsIdle(server, rq, halt): |
1456 | msg = None | 1475 | msg = None |
1457 | interrupted = 0 | 1476 | interrupted = 0 |
1458 | if abort or self.state == state.forceshutdown: | 1477 | if halt or self.state == state.forceshutdown: |
1478 | bb.event._should_exit.set() | ||
1459 | rq.finish_runqueue(True) | 1479 | rq.finish_runqueue(True) |
1460 | msg = "Forced shutdown" | 1480 | msg = "Forced shutdown" |
1461 | interrupted = 2 | 1481 | interrupted = 2 |
@@ -1470,16 +1490,16 @@ class BBCooker: | |||
1470 | failures += len(exc.args) | 1490 | failures += len(exc.args) |
1471 | retval = False | 1491 | retval = False |
1472 | except SystemExit as exc: | 1492 | except SystemExit as exc: |
1473 | self.command.finishAsyncCommand(str(exc)) | 1493 | return bb.server.process.idleFinish(str(exc)) |
1474 | return False | ||
1475 | 1494 | ||
1476 | if not retval: | 1495 | if not retval: |
1477 | try: | 1496 | try: |
1478 | for mc in self.multiconfigs: | 1497 | for mc in self.multiconfigs: |
1479 | bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc]) | 1498 | bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc]) |
1480 | finally: | 1499 | finally: |
1481 | self.command.finishAsyncCommand(msg) | 1500 | bb.event.disable_heartbeat() |
1482 | return False | 1501 | return bb.server.process.idleFinish(msg) |
1502 | |||
1483 | if retval is True: | 1503 | if retval is True: |
1484 | return True | 1504 | return True |
1485 | return retval | 1505 | return retval |
@@ -1498,7 +1518,7 @@ class BBCooker: | |||
1498 | 1518 | ||
1499 | bb.event.fire(bb.event.BuildInit(packages), self.data) | 1519 | bb.event.fire(bb.event.BuildInit(packages), self.data) |
1500 | 1520 | ||
1501 | taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort) | 1521 | taskdata, runlist = self.buildTaskData(targets, task, self.configuration.halt) |
1502 | 1522 | ||
1503 | buildname = self.data.getVar("BUILDNAME", False) | 1523 | buildname = self.data.getVar("BUILDNAME", False) |
1504 | 1524 | ||
@@ -1511,6 +1531,9 @@ class BBCooker: | |||
1511 | 1531 | ||
1512 | for mc in self.multiconfigs: | 1532 | for mc in self.multiconfigs: |
1513 | bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc]) | 1533 | bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc]) |
1534 | if self.eventlog: | ||
1535 | self.eventlog[2].write_variables() | ||
1536 | bb.event.enable_heartbeat() | ||
1514 | 1537 | ||
1515 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) | 1538 | rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist) |
1516 | if 'universe' in targets: | 1539 | if 'universe' in targets: |
@@ -1520,7 +1543,13 @@ class BBCooker: | |||
1520 | 1543 | ||
1521 | 1544 | ||
1522 | def getAllKeysWithFlags(self, flaglist): | 1545 | def getAllKeysWithFlags(self, flaglist): |
1546 | def dummy_autorev(d): | ||
1547 | return | ||
1548 | |||
1523 | dump = {} | 1549 | dump = {} |
1550 | # Horrible but for now we need to avoid any sideeffects of autorev being called | ||
1551 | saved = bb.fetch2.get_autorev | ||
1552 | bb.fetch2.get_autorev = dummy_autorev | ||
1524 | for k in self.data.keys(): | 1553 | for k in self.data.keys(): |
1525 | try: | 1554 | try: |
1526 | expand = True | 1555 | expand = True |
@@ -1540,6 +1569,7 @@ class BBCooker: | |||
1540 | dump[k][d] = None | 1569 | dump[k][d] = None |
1541 | except Exception as e: | 1570 | except Exception as e: |
1542 | print(e) | 1571 | print(e) |
1572 | bb.fetch2.get_autorev = saved | ||
1543 | return dump | 1573 | return dump |
1544 | 1574 | ||
1545 | 1575 | ||
@@ -1547,13 +1577,6 @@ class BBCooker: | |||
1547 | if self.state == state.running: | 1577 | if self.state == state.running: |
1548 | return | 1578 | return |
1549 | 1579 | ||
1550 | # reload files for which we got notifications | ||
1551 | for p in self.inotify_modified_files: | ||
1552 | bb.parse.update_cache(p) | ||
1553 | if p in bb.parse.BBHandler.cached_statements: | ||
1554 | del bb.parse.BBHandler.cached_statements[p] | ||
1555 | self.inotify_modified_files = [] | ||
1556 | |||
1557 | if not self.baseconfig_valid: | 1580 | if not self.baseconfig_valid: |
1558 | logger.debug("Reloading base configuration data") | 1581 | logger.debug("Reloading base configuration data") |
1559 | self.initConfigurationData() | 1582 | self.initConfigurationData() |
@@ -1566,7 +1589,7 @@ class BBCooker: | |||
1566 | 1589 | ||
1567 | if self.state in (state.shutdown, state.forceshutdown, state.error): | 1590 | if self.state in (state.shutdown, state.forceshutdown, state.error): |
1568 | if hasattr(self.parser, 'shutdown'): | 1591 | if hasattr(self.parser, 'shutdown'): |
1569 | self.parser.shutdown(clean=False, force = True) | 1592 | self.parser.shutdown(clean=False) |
1570 | self.parser.final_cleanup() | 1593 | self.parser.final_cleanup() |
1571 | raise bb.BBHandledException() | 1594 | raise bb.BBHandledException() |
1572 | 1595 | ||
@@ -1574,6 +1597,9 @@ class BBCooker: | |||
1574 | self.updateCacheSync() | 1597 | self.updateCacheSync() |
1575 | 1598 | ||
1576 | if self.state != state.parsing and not self.parsecache_valid: | 1599 | if self.state != state.parsing and not self.parsecache_valid: |
1600 | bb.server.process.serverlog("Parsing started") | ||
1601 | self.parsewatched = {} | ||
1602 | |||
1577 | bb.parse.siggen.reset(self.data) | 1603 | bb.parse.siggen.reset(self.data) |
1578 | self.parseConfiguration () | 1604 | self.parseConfiguration () |
1579 | if CookerFeatures.SEND_SANITYEVENTS in self.featureset: | 1605 | if CookerFeatures.SEND_SANITYEVENTS in self.featureset: |
@@ -1587,30 +1613,27 @@ class BBCooker: | |||
1587 | for dep in self.configuration.extra_assume_provided: | 1613 | for dep in self.configuration.extra_assume_provided: |
1588 | self.recipecaches[mc].ignored_dependencies.add(dep) | 1614 | self.recipecaches[mc].ignored_dependencies.add(dep) |
1589 | 1615 | ||
1590 | self.collections = {} | ||
1591 | |||
1592 | mcfilelist = {} | 1616 | mcfilelist = {} |
1593 | total_masked = 0 | 1617 | total_masked = 0 |
1594 | searchdirs = set() | 1618 | searchdirs = set() |
1595 | for mc in self.multiconfigs: | 1619 | for mc in self.multiconfigs: |
1596 | self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc) | ||
1597 | (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) | 1620 | (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc]) |
1598 | 1621 | ||
1599 | mcfilelist[mc] = filelist | 1622 | mcfilelist[mc] = filelist |
1600 | total_masked += masked | 1623 | total_masked += masked |
1601 | searchdirs |= set(search) | 1624 | searchdirs |= set(search) |
1602 | 1625 | ||
1603 | # Add inotify watches for directories searched for bb/bbappend files | 1626 | # Add mtimes for directories searched for bb/bbappend files |
1604 | for dirent in searchdirs: | 1627 | for dirent in searchdirs: |
1605 | self.add_filewatch([[dirent]], dirs=True) | 1628 | self.add_filewatch([(dirent, bb.parse.cached_mtime_noerror(dirent))]) |
1606 | 1629 | ||
1607 | self.parser = CookerParser(self, mcfilelist, total_masked) | 1630 | self.parser = CookerParser(self, mcfilelist, total_masked) |
1608 | self.parsecache_valid = True | 1631 | self._parsecache_set(True) |
1609 | 1632 | ||
1610 | self.state = state.parsing | 1633 | self.state = state.parsing |
1611 | 1634 | ||
1612 | if not self.parser.parse_next(): | 1635 | if not self.parser.parse_next(): |
1613 | collectlog.debug(1, "parsing complete") | 1636 | collectlog.debug("parsing complete") |
1614 | if self.parser.error: | 1637 | if self.parser.error: |
1615 | raise bb.BBHandledException() | 1638 | raise bb.BBHandledException() |
1616 | self.show_appends_with_no_recipes() | 1639 | self.show_appends_with_no_recipes() |
@@ -1633,7 +1656,7 @@ class BBCooker: | |||
1633 | # Return a copy, don't modify the original | 1656 | # Return a copy, don't modify the original |
1634 | pkgs_to_build = pkgs_to_build[:] | 1657 | pkgs_to_build = pkgs_to_build[:] |
1635 | 1658 | ||
1636 | if len(pkgs_to_build) == 0: | 1659 | if not pkgs_to_build: |
1637 | raise NothingToBuild | 1660 | raise NothingToBuild |
1638 | 1661 | ||
1639 | ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split() | 1662 | ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split() |
@@ -1655,7 +1678,7 @@ class BBCooker: | |||
1655 | 1678 | ||
1656 | if 'universe' in pkgs_to_build: | 1679 | if 'universe' in pkgs_to_build: |
1657 | parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.") | 1680 | parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.") |
1658 | parselog.debug(1, "collating packages for \"universe\"") | 1681 | parselog.debug("collating packages for \"universe\"") |
1659 | pkgs_to_build.remove('universe') | 1682 | pkgs_to_build.remove('universe') |
1660 | for mc in self.multiconfigs: | 1683 | for mc in self.multiconfigs: |
1661 | for t in self.recipecaches[mc].universe_target: | 1684 | for t in self.recipecaches[mc].universe_target: |
@@ -1680,26 +1703,36 @@ class BBCooker: | |||
1680 | def post_serve(self): | 1703 | def post_serve(self): |
1681 | self.shutdown(force=True) | 1704 | self.shutdown(force=True) |
1682 | prserv.serv.auto_shutdown() | 1705 | prserv.serv.auto_shutdown() |
1706 | if hasattr(bb.parse, "siggen"): | ||
1707 | bb.parse.siggen.exit() | ||
1683 | if self.hashserv: | 1708 | if self.hashserv: |
1684 | self.hashserv.process.terminate() | 1709 | self.hashserv.process.terminate() |
1685 | self.hashserv.process.join() | 1710 | self.hashserv.process.join() |
1686 | if hasattr(self, "data"): | 1711 | if hasattr(self, "data"): |
1687 | bb.event.fire(CookerExit(), self.data) | 1712 | bb.event.fire(CookerExit(), self.data) |
1688 | 1713 | ||
1689 | def shutdown(self, force = False): | 1714 | def shutdown(self, force=False): |
1690 | if force: | 1715 | if force: |
1691 | self.state = state.forceshutdown | 1716 | self.state = state.forceshutdown |
1717 | bb.event._should_exit.set() | ||
1692 | else: | 1718 | else: |
1693 | self.state = state.shutdown | 1719 | self.state = state.shutdown |
1694 | 1720 | ||
1695 | if self.parser: | 1721 | if self.parser: |
1696 | self.parser.shutdown(clean=not force, force=force) | 1722 | self.parser.shutdown(clean=False) |
1697 | self.parser.final_cleanup() | 1723 | self.parser.final_cleanup() |
1698 | 1724 | ||
1699 | def finishcommand(self): | 1725 | def finishcommand(self): |
1726 | if hasattr(self.parser, 'shutdown'): | ||
1727 | self.parser.shutdown(clean=False) | ||
1728 | self.parser.final_cleanup() | ||
1700 | self.state = state.initial | 1729 | self.state = state.initial |
1730 | bb.event._should_exit.clear() | ||
1701 | 1731 | ||
1702 | def reset(self): | 1732 | def reset(self): |
1733 | if hasattr(bb.parse, "siggen"): | ||
1734 | bb.parse.siggen.exit() | ||
1735 | self.finishcommand() | ||
1703 | self.initConfigurationData() | 1736 | self.initConfigurationData() |
1704 | self.handlePRServ() | 1737 | self.handlePRServ() |
1705 | 1738 | ||
@@ -1711,9 +1744,9 @@ class BBCooker: | |||
1711 | if hasattr(self, "data"): | 1744 | if hasattr(self, "data"): |
1712 | self.databuilder.reset() | 1745 | self.databuilder.reset() |
1713 | self.data = self.databuilder.data | 1746 | self.data = self.databuilder.data |
1714 | self.parsecache_valid = False | 1747 | # In theory tinfoil could have modified the base data before parsing, |
1715 | self.baseconfig_valid = False | 1748 | # ideally need to track if anything did modify the datastore |
1716 | 1749 | self._parsecache_set(False) | |
1717 | 1750 | ||
1718 | class CookerExit(bb.event.Event): | 1751 | class CookerExit(bb.event.Event): |
1719 | """ | 1752 | """ |
@@ -1728,16 +1761,16 @@ class CookerCollectFiles(object): | |||
1728 | def __init__(self, priorities, mc=''): | 1761 | def __init__(self, priorities, mc=''): |
1729 | self.mc = mc | 1762 | self.mc = mc |
1730 | self.bbappends = [] | 1763 | self.bbappends = [] |
1731 | # Priorities is a list of tupples, with the second element as the pattern. | 1764 | # Priorities is a list of tuples, with the second element as the pattern. |
1732 | # We need to sort the list with the longest pattern first, and so on to | 1765 | # We need to sort the list with the longest pattern first, and so on to |
1733 | # the shortest. This allows nested layers to be properly evaluated. | 1766 | # the shortest. This allows nested layers to be properly evaluated. |
1734 | self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) | 1767 | self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True) |
1735 | 1768 | ||
1736 | def calc_bbfile_priority(self, filename): | 1769 | def calc_bbfile_priority(self, filename): |
1737 | for _, _, regex, pri in self.bbfile_config_priorities: | 1770 | for layername, _, regex, pri in self.bbfile_config_priorities: |
1738 | if regex.match(filename): | 1771 | if regex.match(filename): |
1739 | return pri, regex | 1772 | return pri, regex, layername |
1740 | return 0, None | 1773 | return 0, None, None |
1741 | 1774 | ||
1742 | def get_bbfiles(self): | 1775 | def get_bbfiles(self): |
1743 | """Get list of default .bb files by reading out the current directory""" | 1776 | """Get list of default .bb files by reading out the current directory""" |
@@ -1756,7 +1789,7 @@ class CookerCollectFiles(object): | |||
1756 | for ignored in ('SCCS', 'CVS', '.svn'): | 1789 | for ignored in ('SCCS', 'CVS', '.svn'): |
1757 | if ignored in dirs: | 1790 | if ignored in dirs: |
1758 | dirs.remove(ignored) | 1791 | dirs.remove(ignored) |
1759 | found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))] | 1792 | found += [os.path.join(dir, f) for f in files if (f.endswith(('.bb', '.bbappend')))] |
1760 | 1793 | ||
1761 | return found | 1794 | return found |
1762 | 1795 | ||
@@ -1764,7 +1797,7 @@ class CookerCollectFiles(object): | |||
1764 | """Collect all available .bb build files""" | 1797 | """Collect all available .bb build files""" |
1765 | masked = 0 | 1798 | masked = 0 |
1766 | 1799 | ||
1767 | collectlog.debug(1, "collecting .bb files") | 1800 | collectlog.debug("collecting .bb files") |
1768 | 1801 | ||
1769 | files = (config.getVar( "BBFILES") or "").split() | 1802 | files = (config.getVar( "BBFILES") or "").split() |
1770 | 1803 | ||
@@ -1772,14 +1805,14 @@ class CookerCollectFiles(object): | |||
1772 | files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] ) | 1805 | files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] ) |
1773 | config.setVar("BBFILES_PRIORITIZED", " ".join(files)) | 1806 | config.setVar("BBFILES_PRIORITIZED", " ".join(files)) |
1774 | 1807 | ||
1775 | if not len(files): | 1808 | if not files: |
1776 | files = self.get_bbfiles() | 1809 | files = self.get_bbfiles() |
1777 | 1810 | ||
1778 | if not len(files): | 1811 | if not files: |
1779 | collectlog.error("no recipe files to build, check your BBPATH and BBFILES?") | 1812 | collectlog.error("no recipe files to build, check your BBPATH and BBFILES?") |
1780 | bb.event.fire(CookerExit(), eventdata) | 1813 | bb.event.fire(CookerExit(), eventdata) |
1781 | 1814 | ||
1782 | # We need to track where we look so that we can add inotify watches. There | 1815 | # We need to track where we look so that we can know when the cache is invalid. There |
1783 | # is no nice way to do this, this is horrid. We intercept the os.listdir() | 1816 | # is no nice way to do this, this is horrid. We intercept the os.listdir() |
1784 | # (or os.scandir() for python 3.6+) calls while we run glob(). | 1817 | # (or os.scandir() for python 3.6+) calls while we run glob(). |
1785 | origlistdir = os.listdir | 1818 | origlistdir = os.listdir |
@@ -1835,7 +1868,7 @@ class CookerCollectFiles(object): | |||
1835 | try: | 1868 | try: |
1836 | re.compile(mask) | 1869 | re.compile(mask) |
1837 | bbmasks.append(mask) | 1870 | bbmasks.append(mask) |
1838 | except sre_constants.error: | 1871 | except re.error: |
1839 | collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask) | 1872 | collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask) |
1840 | 1873 | ||
1841 | # Then validate the combined regular expressions. This should never | 1874 | # Then validate the combined regular expressions. This should never |
@@ -1843,7 +1876,7 @@ class CookerCollectFiles(object): | |||
1843 | bbmask = "|".join(bbmasks) | 1876 | bbmask = "|".join(bbmasks) |
1844 | try: | 1877 | try: |
1845 | bbmask_compiled = re.compile(bbmask) | 1878 | bbmask_compiled = re.compile(bbmask) |
1846 | except sre_constants.error: | 1879 | except re.error: |
1847 | collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask) | 1880 | collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask) |
1848 | bbmask = None | 1881 | bbmask = None |
1849 | 1882 | ||
@@ -1851,7 +1884,7 @@ class CookerCollectFiles(object): | |||
1851 | bbappend = [] | 1884 | bbappend = [] |
1852 | for f in newfiles: | 1885 | for f in newfiles: |
1853 | if bbmask and bbmask_compiled.search(f): | 1886 | if bbmask and bbmask_compiled.search(f): |
1854 | collectlog.debug(1, "skipping masked file %s", f) | 1887 | collectlog.debug("skipping masked file %s", f) |
1855 | masked += 1 | 1888 | masked += 1 |
1856 | continue | 1889 | continue |
1857 | if f.endswith('.bb'): | 1890 | if f.endswith('.bb'): |
@@ -1859,7 +1892,7 @@ class CookerCollectFiles(object): | |||
1859 | elif f.endswith('.bbappend'): | 1892 | elif f.endswith('.bbappend'): |
1860 | bbappend.append(f) | 1893 | bbappend.append(f) |
1861 | else: | 1894 | else: |
1862 | collectlog.debug(1, "skipping %s: unknown file extension", f) | 1895 | collectlog.debug("skipping %s: unknown file extension", f) |
1863 | 1896 | ||
1864 | # Build a list of .bbappend files for each .bb file | 1897 | # Build a list of .bbappend files for each .bb file |
1865 | for f in bbappend: | 1898 | for f in bbappend: |
@@ -1910,7 +1943,7 @@ class CookerCollectFiles(object): | |||
1910 | # Calculate priorities for each file | 1943 | # Calculate priorities for each file |
1911 | for p in pkgfns: | 1944 | for p in pkgfns: |
1912 | realfn, cls, mc = bb.cache.virtualfn2realfn(p) | 1945 | realfn, cls, mc = bb.cache.virtualfn2realfn(p) |
1913 | priorities[p], regex = self.calc_bbfile_priority(realfn) | 1946 | priorities[p], regex, _ = self.calc_bbfile_priority(realfn) |
1914 | if regex in unmatched_regex: | 1947 | if regex in unmatched_regex: |
1915 | matched_regex.add(regex) | 1948 | matched_regex.add(regex) |
1916 | unmatched_regex.remove(regex) | 1949 | unmatched_regex.remove(regex) |
@@ -1961,15 +1994,30 @@ class ParsingFailure(Exception): | |||
1961 | Exception.__init__(self, realexception, recipe) | 1994 | Exception.__init__(self, realexception, recipe) |
1962 | 1995 | ||
1963 | class Parser(multiprocessing.Process): | 1996 | class Parser(multiprocessing.Process): |
1964 | def __init__(self, jobs, results, quit, init, profile): | 1997 | def __init__(self, jobs, results, quit, profile): |
1965 | self.jobs = jobs | 1998 | self.jobs = jobs |
1966 | self.results = results | 1999 | self.results = results |
1967 | self.quit = quit | 2000 | self.quit = quit |
1968 | self.init = init | ||
1969 | multiprocessing.Process.__init__(self) | 2001 | multiprocessing.Process.__init__(self) |
1970 | self.context = bb.utils.get_context().copy() | 2002 | self.context = bb.utils.get_context().copy() |
1971 | self.handlers = bb.event.get_class_handlers().copy() | 2003 | self.handlers = bb.event.get_class_handlers().copy() |
1972 | self.profile = profile | 2004 | self.profile = profile |
2005 | self.queue_signals = False | ||
2006 | self.signal_received = [] | ||
2007 | self.signal_threadlock = threading.Lock() | ||
2008 | |||
2009 | def catch_sig(self, signum, frame): | ||
2010 | if self.queue_signals: | ||
2011 | self.signal_received.append(signum) | ||
2012 | else: | ||
2013 | self.handle_sig(signum, frame) | ||
2014 | |||
2015 | def handle_sig(self, signum, frame): | ||
2016 | if signum == signal.SIGTERM: | ||
2017 | signal.signal(signal.SIGTERM, signal.SIG_DFL) | ||
2018 | os.kill(os.getpid(), signal.SIGTERM) | ||
2019 | elif signum == signal.SIGINT: | ||
2020 | signal.default_int_handler(signum, frame) | ||
1973 | 2021 | ||
1974 | def run(self): | 2022 | def run(self): |
1975 | 2023 | ||
@@ -1989,38 +2037,50 @@ class Parser(multiprocessing.Process): | |||
1989 | prof.dump_stats(logfile) | 2037 | prof.dump_stats(logfile) |
1990 | 2038 | ||
1991 | def realrun(self): | 2039 | def realrun(self): |
1992 | if self.init: | 2040 | # Signal handling here is hard. We must not terminate any process or thread holding the write |
1993 | self.init() | 2041 | # lock for the event stream as it will not be released, ever, and things will hang. |
2042 | # Python handles signals in the main thread/process but they can be raised from any thread and | ||
2043 | # we want to defer processing of any SIGTERM/SIGINT signal until we're outside the critical section | ||
2044 | # and don't hold the lock (see server/process.py). We therefore always catch the signals (so any | ||
2045 | # new thread should also do so) and we defer handling but we handle with the local thread lock | ||
2046 | # held (a threading lock, not a multiprocessing one) so that no other thread in the process | ||
2047 | # can be in the critical section. | ||
2048 | signal.signal(signal.SIGTERM, self.catch_sig) | ||
2049 | signal.signal(signal.SIGHUP, signal.SIG_DFL) | ||
2050 | signal.signal(signal.SIGINT, self.catch_sig) | ||
2051 | bb.utils.set_process_name(multiprocessing.current_process().name) | ||
2052 | multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) | ||
2053 | multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) | ||
1994 | 2054 | ||
1995 | pending = [] | 2055 | pending = [] |
1996 | while True: | 2056 | havejobs = True |
1997 | try: | 2057 | try: |
1998 | self.quit.get_nowait() | 2058 | while havejobs or pending: |
1999 | except queue.Empty: | 2059 | if self.quit.is_set(): |
2000 | pass | 2060 | break |
2001 | else: | ||
2002 | self.results.close() | ||
2003 | self.results.join_thread() | ||
2004 | break | ||
2005 | 2061 | ||
2006 | if pending: | 2062 | job = None |
2007 | result = pending.pop() | ||
2008 | else: | ||
2009 | try: | 2063 | try: |
2010 | job = self.jobs.pop() | 2064 | job = self.jobs.pop() |
2011 | except IndexError: | 2065 | except IndexError: |
2012 | self.results.close() | 2066 | havejobs = False |
2013 | self.results.join_thread() | 2067 | if job: |
2014 | break | 2068 | result = self.parse(*job) |
2015 | result = self.parse(*job) | 2069 | # Clear the siggen cache after parsing to control memory usage, its huge |
2016 | # Clear the siggen cache after parsing to control memory usage, its huge | 2070 | bb.parse.siggen.postparsing_clean_cache() |
2017 | bb.parse.siggen.postparsing_clean_cache() | 2071 | pending.append(result) |
2018 | try: | 2072 | |
2019 | self.results.put(result, timeout=0.25) | 2073 | if pending: |
2020 | except queue.Full: | 2074 | try: |
2021 | pending.append(result) | 2075 | result = pending.pop() |
2076 | self.results.put(result, timeout=0.05) | ||
2077 | except queue.Full: | ||
2078 | pending.append(result) | ||
2079 | finally: | ||
2080 | self.results.close() | ||
2081 | self.results.join_thread() | ||
2022 | 2082 | ||
2023 | def parse(self, mc, cache, filename, appends): | 2083 | def parse(self, mc, cache, filename, appends, layername): |
2024 | try: | 2084 | try: |
2025 | origfilter = bb.event.LogHandler.filter | 2085 | origfilter = bb.event.LogHandler.filter |
2026 | # Record the filename we're parsing into any events generated | 2086 | # Record the filename we're parsing into any events generated |
@@ -2034,17 +2094,17 @@ class Parser(multiprocessing.Process): | |||
2034 | bb.event.set_class_handlers(self.handlers.copy()) | 2094 | bb.event.set_class_handlers(self.handlers.copy()) |
2035 | bb.event.LogHandler.filter = parse_filter | 2095 | bb.event.LogHandler.filter = parse_filter |
2036 | 2096 | ||
2037 | return True, mc, cache.parse(filename, appends) | 2097 | return True, mc, cache.parse(filename, appends, layername) |
2038 | except Exception as exc: | 2098 | except Exception as exc: |
2039 | tb = sys.exc_info()[2] | 2099 | tb = sys.exc_info()[2] |
2040 | exc.recipe = filename | 2100 | exc.recipe = filename |
2041 | exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3)) | 2101 | exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3)) |
2042 | return True, exc | 2102 | return True, None, exc |
2043 | # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown | 2103 | # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown |
2044 | # and for example a worker thread doesn't just exit on its own in response to | 2104 | # and for example a worker thread doesn't just exit on its own in response to |
2045 | # a SystemExit event for example. | 2105 | # a SystemExit event for example. |
2046 | except BaseException as exc: | 2106 | except BaseException as exc: |
2047 | return True, ParsingFailure(exc, filename) | 2107 | return True, None, ParsingFailure(exc, filename) |
2048 | finally: | 2108 | finally: |
2049 | bb.event.LogHandler.filter = origfilter | 2109 | bb.event.LogHandler.filter = origfilter |
2050 | 2110 | ||
@@ -2074,10 +2134,11 @@ class CookerParser(object): | |||
2074 | for mc in self.cooker.multiconfigs: | 2134 | for mc in self.cooker.multiconfigs: |
2075 | for filename in self.mcfilelist[mc]: | 2135 | for filename in self.mcfilelist[mc]: |
2076 | appends = self.cooker.collections[mc].get_file_appends(filename) | 2136 | appends = self.cooker.collections[mc].get_file_appends(filename) |
2137 | layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2] | ||
2077 | if not self.bb_caches[mc].cacheValid(filename, appends): | 2138 | if not self.bb_caches[mc].cacheValid(filename, appends): |
2078 | self.willparse.add((mc, self.bb_caches[mc], filename, appends)) | 2139 | self.willparse.add((mc, self.bb_caches[mc], filename, appends, layername)) |
2079 | else: | 2140 | else: |
2080 | self.fromcache.add((mc, self.bb_caches[mc], filename, appends)) | 2141 | self.fromcache.add((mc, self.bb_caches[mc], filename, appends, layername)) |
2081 | 2142 | ||
2082 | self.total = len(self.fromcache) + len(self.willparse) | 2143 | self.total = len(self.fromcache) + len(self.willparse) |
2083 | self.toparse = len(self.willparse) | 2144 | self.toparse = len(self.willparse) |
@@ -2086,6 +2147,7 @@ class CookerParser(object): | |||
2086 | self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or | 2147 | self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or |
2087 | multiprocessing.cpu_count()), self.toparse) | 2148 | multiprocessing.cpu_count()), self.toparse) |
2088 | 2149 | ||
2150 | bb.cache.SiggenRecipeInfo.reset() | ||
2089 | self.start() | 2151 | self.start() |
2090 | self.haveshutdown = False | 2152 | self.haveshutdown = False |
2091 | self.syncthread = None | 2153 | self.syncthread = None |
@@ -2095,15 +2157,8 @@ class CookerParser(object): | |||
2095 | self.processes = [] | 2157 | self.processes = [] |
2096 | if self.toparse: | 2158 | if self.toparse: |
2097 | bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) | 2159 | bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata) |
2098 | def init(): | 2160 | |
2099 | signal.signal(signal.SIGTERM, signal.SIG_DFL) | 2161 | self.parser_quit = multiprocessing.Event() |
2100 | signal.signal(signal.SIGHUP, signal.SIG_DFL) | ||
2101 | signal.signal(signal.SIGINT, signal.SIG_IGN) | ||
2102 | bb.utils.set_process_name(multiprocessing.current_process().name) | ||
2103 | multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1) | ||
2104 | multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1) | ||
2105 | |||
2106 | self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes) | ||
2107 | self.result_queue = multiprocessing.Queue() | 2162 | self.result_queue = multiprocessing.Queue() |
2108 | 2163 | ||
2109 | def chunkify(lst,n): | 2164 | def chunkify(lst,n): |
@@ -2111,14 +2166,14 @@ class CookerParser(object): | |||
2111 | self.jobs = chunkify(list(self.willparse), self.num_processes) | 2166 | self.jobs = chunkify(list(self.willparse), self.num_processes) |
2112 | 2167 | ||
2113 | for i in range(0, self.num_processes): | 2168 | for i in range(0, self.num_processes): |
2114 | parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile) | 2169 | parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, self.cooker.configuration.profile) |
2115 | parser.start() | 2170 | parser.start() |
2116 | self.process_names.append(parser.name) | 2171 | self.process_names.append(parser.name) |
2117 | self.processes.append(parser) | 2172 | self.processes.append(parser) |
2118 | 2173 | ||
2119 | self.results = itertools.chain(self.results, self.parse_generator()) | 2174 | self.results = itertools.chain(self.results, self.parse_generator()) |
2120 | 2175 | ||
2121 | def shutdown(self, clean=True, force=False): | 2176 | def shutdown(self, clean=True, eventmsg="Parsing halted due to errors"): |
2122 | if not self.toparse: | 2177 | if not self.toparse: |
2123 | return | 2178 | return |
2124 | if self.haveshutdown: | 2179 | if self.haveshutdown: |
@@ -2132,9 +2187,9 @@ class CookerParser(object): | |||
2132 | self.total) | 2187 | self.total) |
2133 | 2188 | ||
2134 | bb.event.fire(event, self.cfgdata) | 2189 | bb.event.fire(event, self.cfgdata) |
2135 | 2190 | else: | |
2136 | for process in self.processes: | 2191 | bb.event.fire(bb.event.ParseError(eventmsg), self.cfgdata) |
2137 | self.parser_quit.put(None) | 2192 | bb.error("Parsing halted due to errors, see error messages above") |
2138 | 2193 | ||
2139 | # Cleanup the queue before call process.join(), otherwise there might be | 2194 | # Cleanup the queue before call process.join(), otherwise there might be |
2140 | # deadlocks. | 2195 | # deadlocks. |
@@ -2144,25 +2199,39 @@ class CookerParser(object): | |||
2144 | except queue.Empty: | 2199 | except queue.Empty: |
2145 | break | 2200 | break |
2146 | 2201 | ||
2147 | for process in self.processes: | ||
2148 | if force: | ||
2149 | process.join(.1) | ||
2150 | process.terminate() | ||
2151 | else: | ||
2152 | process.join() | ||
2153 | |||
2154 | self.parser_quit.close() | ||
2155 | # Allow data left in the cancel queue to be discarded | ||
2156 | self.parser_quit.cancel_join_thread() | ||
2157 | |||
2158 | def sync_caches(): | 2202 | def sync_caches(): |
2159 | for c in self.bb_caches.values(): | 2203 | for c in self.bb_caches.values(): |
2204 | bb.cache.SiggenRecipeInfo.reset() | ||
2160 | c.sync() | 2205 | c.sync() |
2161 | 2206 | ||
2162 | sync = threading.Thread(target=sync_caches, name="SyncThread") | 2207 | self.syncthread = threading.Thread(target=sync_caches, name="SyncThread") |
2163 | self.syncthread = sync | 2208 | self.syncthread.start() |
2164 | sync.start() | 2209 | |
2210 | self.parser_quit.set() | ||
2211 | |||
2212 | for process in self.processes: | ||
2213 | process.join(0.5) | ||
2214 | |||
2215 | for process in self.processes: | ||
2216 | if process.exitcode is None: | ||
2217 | os.kill(process.pid, signal.SIGINT) | ||
2218 | |||
2219 | for process in self.processes: | ||
2220 | process.join(0.5) | ||
2221 | |||
2222 | for process in self.processes: | ||
2223 | if process.exitcode is None: | ||
2224 | process.terminate() | ||
2225 | |||
2226 | for process in self.processes: | ||
2227 | process.join() | ||
2228 | # Added in 3.7, cleans up zombies | ||
2229 | if hasattr(process, "close"): | ||
2230 | process.close() | ||
2231 | |||
2232 | bb.codeparser.parser_cache_save() | ||
2165 | bb.codeparser.parser_cache_savemerge() | 2233 | bb.codeparser.parser_cache_savemerge() |
2234 | bb.cache.SiggenRecipeInfo.reset() | ||
2166 | bb.fetch.fetcher_parse_done() | 2235 | bb.fetch.fetcher_parse_done() |
2167 | if self.cooker.configuration.profile: | 2236 | if self.cooker.configuration.profile: |
2168 | profiles = [] | 2237 | profiles = [] |
@@ -2180,49 +2249,64 @@ class CookerParser(object): | |||
2180 | self.syncthread.join() | 2249 | self.syncthread.join() |
2181 | 2250 | ||
2182 | def load_cached(self): | 2251 | def load_cached(self): |
2183 | for mc, cache, filename, appends in self.fromcache: | 2252 | for mc, cache, filename, appends, layername in self.fromcache: |
2184 | cached, infos = cache.load(filename, appends) | 2253 | infos = cache.loadCached(filename, appends) |
2185 | yield not cached, mc, infos | 2254 | yield False, mc, infos |
2186 | 2255 | ||
2187 | def parse_generator(self): | 2256 | def parse_generator(self): |
2188 | while True: | 2257 | empty = False |
2258 | while self.processes or not empty: | ||
2259 | for process in self.processes.copy(): | ||
2260 | if not process.is_alive(): | ||
2261 | process.join() | ||
2262 | self.processes.remove(process) | ||
2263 | |||
2189 | if self.parsed >= self.toparse: | 2264 | if self.parsed >= self.toparse: |
2190 | break | 2265 | break |
2191 | 2266 | ||
2192 | try: | 2267 | try: |
2193 | result = self.result_queue.get(timeout=0.25) | 2268 | result = self.result_queue.get(timeout=0.25) |
2194 | except queue.Empty: | 2269 | except queue.Empty: |
2195 | pass | 2270 | empty = True |
2271 | yield None, None, None | ||
2196 | else: | 2272 | else: |
2197 | value = result[1] | 2273 | empty = False |
2198 | if isinstance(value, BaseException): | 2274 | yield result |
2199 | raise value | 2275 | |
2200 | else: | 2276 | if not (self.parsed >= self.toparse): |
2201 | yield result | 2277 | raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None) |
2278 | |||
2202 | 2279 | ||
2203 | def parse_next(self): | 2280 | def parse_next(self): |
2204 | result = [] | 2281 | result = [] |
2205 | parsed = None | 2282 | parsed = None |
2206 | try: | 2283 | try: |
2207 | parsed, mc, result = next(self.results) | 2284 | parsed, mc, result = next(self.results) |
2285 | if isinstance(result, BaseException): | ||
2286 | # Turn exceptions back into exceptions | ||
2287 | raise result | ||
2288 | if parsed is None: | ||
2289 | # Timeout, loop back through the main loop | ||
2290 | return True | ||
2291 | |||
2208 | except StopIteration: | 2292 | except StopIteration: |
2209 | self.shutdown() | 2293 | self.shutdown() |
2210 | return False | 2294 | return False |
2211 | except bb.BBHandledException as exc: | 2295 | except bb.BBHandledException as exc: |
2212 | self.error += 1 | 2296 | self.error += 1 |
2213 | logger.error('Failed to parse recipe: %s' % exc.recipe) | 2297 | logger.debug('Failed to parse recipe: %s' % exc.recipe) |
2214 | self.shutdown(clean=False, force=True) | 2298 | self.shutdown(clean=False) |
2215 | return False | 2299 | return False |
2216 | except ParsingFailure as exc: | 2300 | except ParsingFailure as exc: |
2217 | self.error += 1 | 2301 | self.error += 1 |
2218 | logger.error('Unable to parse %s: %s' % | 2302 | logger.error('Unable to parse %s: %s' % |
2219 | (exc.recipe, bb.exceptions.to_string(exc.realexception))) | 2303 | (exc.recipe, bb.exceptions.to_string(exc.realexception))) |
2220 | self.shutdown(clean=False, force=True) | 2304 | self.shutdown(clean=False) |
2221 | return False | 2305 | return False |
2222 | except bb.parse.ParseError as exc: | 2306 | except bb.parse.ParseError as exc: |
2223 | self.error += 1 | 2307 | self.error += 1 |
2224 | logger.error(str(exc)) | 2308 | logger.error(str(exc)) |
2225 | self.shutdown(clean=False, force=True) | 2309 | self.shutdown(clean=False, eventmsg=str(exc)) |
2226 | return False | 2310 | return False |
2227 | except bb.data_smart.ExpansionError as exc: | 2311 | except bb.data_smart.ExpansionError as exc: |
2228 | self.error += 1 | 2312 | self.error += 1 |
@@ -2231,7 +2315,7 @@ class CookerParser(object): | |||
2231 | tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback)) | 2315 | tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback)) |
2232 | logger.error('ExpansionError during parsing %s', value.recipe, | 2316 | logger.error('ExpansionError during parsing %s', value.recipe, |
2233 | exc_info=(etype, value, tb)) | 2317 | exc_info=(etype, value, tb)) |
2234 | self.shutdown(clean=False, force=True) | 2318 | self.shutdown(clean=False) |
2235 | return False | 2319 | return False |
2236 | except Exception as exc: | 2320 | except Exception as exc: |
2237 | self.error += 1 | 2321 | self.error += 1 |
@@ -2243,7 +2327,7 @@ class CookerParser(object): | |||
2243 | # Most likely, an exception occurred during raising an exception | 2327 | # Most likely, an exception occurred during raising an exception |
2244 | import traceback | 2328 | import traceback |
2245 | logger.error('Exception during parse: %s' % traceback.format_exc()) | 2329 | logger.error('Exception during parse: %s' % traceback.format_exc()) |
2246 | self.shutdown(clean=False, force=True) | 2330 | self.shutdown(clean=False) |
2247 | return False | 2331 | return False |
2248 | 2332 | ||
2249 | self.current += 1 | 2333 | self.current += 1 |
@@ -2265,11 +2349,13 @@ class CookerParser(object): | |||
2265 | return True | 2349 | return True |
2266 | 2350 | ||
2267 | def reparse(self, filename): | 2351 | def reparse(self, filename): |
2352 | bb.cache.SiggenRecipeInfo.reset() | ||
2268 | to_reparse = set() | 2353 | to_reparse = set() |
2269 | for mc in self.cooker.multiconfigs: | 2354 | for mc in self.cooker.multiconfigs: |
2270 | to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename))) | 2355 | layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2] |
2356 | to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename), layername)) | ||
2271 | 2357 | ||
2272 | for mc, filename, appends in to_reparse: | 2358 | for mc, filename, appends, layername in to_reparse: |
2273 | infos = self.bb_caches[mc].parse(filename, appends) | 2359 | infos = self.bb_caches[mc].parse(filename, appends, layername) |
2274 | for vfn, info_array in infos: | 2360 | for vfn, info_array in infos: |
2275 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) | 2361 | self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array) |