diff options
Diffstat (limited to 'bitbake/lib/bb/cache.py')
-rw-r--r-- | bitbake/lib/bb/cache.py | 340 |
1 files changed, 160 insertions, 180 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py index aea2b8bc11..18d5574a31 100644 --- a/bitbake/lib/bb/cache.py +++ b/bitbake/lib/bb/cache.py | |||
@@ -19,14 +19,16 @@ | |||
19 | import os | 19 | import os |
20 | import logging | 20 | import logging |
21 | import pickle | 21 | import pickle |
22 | from collections import defaultdict, Mapping | 22 | from collections import defaultdict |
23 | from collections.abc import Mapping | ||
23 | import bb.utils | 24 | import bb.utils |
24 | from bb import PrefixLoggerAdapter | 25 | from bb import PrefixLoggerAdapter |
25 | import re | 26 | import re |
27 | import shutil | ||
26 | 28 | ||
27 | logger = logging.getLogger("BitBake.Cache") | 29 | logger = logging.getLogger("BitBake.Cache") |
28 | 30 | ||
29 | __cache_version__ = "154" | 31 | __cache_version__ = "155" |
30 | 32 | ||
31 | def getCacheFile(path, filename, mc, data_hash): | 33 | def getCacheFile(path, filename, mc, data_hash): |
32 | mcspec = '' | 34 | mcspec = '' |
@@ -53,12 +55,12 @@ class RecipeInfoCommon(object): | |||
53 | 55 | ||
54 | @classmethod | 56 | @classmethod |
55 | def pkgvar(cls, var, packages, metadata): | 57 | def pkgvar(cls, var, packages, metadata): |
56 | return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata)) | 58 | return dict((pkg, cls.depvar("%s:%s" % (var, pkg), metadata)) |
57 | for pkg in packages) | 59 | for pkg in packages) |
58 | 60 | ||
59 | @classmethod | 61 | @classmethod |
60 | def taskvar(cls, var, tasks, metadata): | 62 | def taskvar(cls, var, tasks, metadata): |
61 | return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata)) | 63 | return dict((task, cls.getvar("%s:task-%s" % (var, task), metadata)) |
62 | for task in tasks) | 64 | for task in tasks) |
63 | 65 | ||
64 | @classmethod | 66 | @classmethod |
@@ -103,7 +105,7 @@ class CoreRecipeInfo(RecipeInfoCommon): | |||
103 | 105 | ||
104 | self.tasks = metadata.getVar('__BBTASKS', False) | 106 | self.tasks = metadata.getVar('__BBTASKS', False) |
105 | 107 | ||
106 | self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata) | 108 | self.basetaskhashes = metadata.getVar('__siggen_basehashes', False) or {} |
107 | self.hashfilename = self.getvar('BB_HASHFILENAME', metadata) | 109 | self.hashfilename = self.getvar('BB_HASHFILENAME', metadata) |
108 | 110 | ||
109 | self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}} | 111 | self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}} |
@@ -126,6 +128,7 @@ class CoreRecipeInfo(RecipeInfoCommon): | |||
126 | self.inherits = self.getvar('__inherit_cache', metadata, expand=False) | 128 | self.inherits = self.getvar('__inherit_cache', metadata, expand=False) |
127 | self.fakerootenv = self.getvar('FAKEROOTENV', metadata) | 129 | self.fakerootenv = self.getvar('FAKEROOTENV', metadata) |
128 | self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata) | 130 | self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata) |
131 | self.fakerootlogs = self.getvar('FAKEROOTLOGS', metadata) | ||
129 | self.fakerootnoenv = self.getvar('FAKEROOTNOENV', metadata) | 132 | self.fakerootnoenv = self.getvar('FAKEROOTNOENV', metadata) |
130 | self.extradepsfunc = self.getvar('calculate_extra_depends', metadata) | 133 | self.extradepsfunc = self.getvar('calculate_extra_depends', metadata) |
131 | 134 | ||
@@ -163,6 +166,7 @@ class CoreRecipeInfo(RecipeInfoCommon): | |||
163 | cachedata.fakerootenv = {} | 166 | cachedata.fakerootenv = {} |
164 | cachedata.fakerootnoenv = {} | 167 | cachedata.fakerootnoenv = {} |
165 | cachedata.fakerootdirs = {} | 168 | cachedata.fakerootdirs = {} |
169 | cachedata.fakerootlogs = {} | ||
166 | cachedata.extradepsfunc = {} | 170 | cachedata.extradepsfunc = {} |
167 | 171 | ||
168 | def add_cacheData(self, cachedata, fn): | 172 | def add_cacheData(self, cachedata, fn): |
@@ -212,7 +216,7 @@ class CoreRecipeInfo(RecipeInfoCommon): | |||
212 | 216 | ||
213 | # Collect files we may need for possible world-dep | 217 | # Collect files we may need for possible world-dep |
214 | # calculations | 218 | # calculations |
215 | if not self.not_world: | 219 | if not bb.utils.to_boolean(self.not_world): |
216 | cachedata.possible_world.append(fn) | 220 | cachedata.possible_world.append(fn) |
217 | #else: | 221 | #else: |
218 | # logger.debug2("EXCLUDE FROM WORLD: %s", fn) | 222 | # logger.debug2("EXCLUDE FROM WORLD: %s", fn) |
@@ -231,17 +235,116 @@ class CoreRecipeInfo(RecipeInfoCommon): | |||
231 | cachedata.fakerootenv[fn] = self.fakerootenv | 235 | cachedata.fakerootenv[fn] = self.fakerootenv |
232 | cachedata.fakerootnoenv[fn] = self.fakerootnoenv | 236 | cachedata.fakerootnoenv[fn] = self.fakerootnoenv |
233 | cachedata.fakerootdirs[fn] = self.fakerootdirs | 237 | cachedata.fakerootdirs[fn] = self.fakerootdirs |
238 | cachedata.fakerootlogs[fn] = self.fakerootlogs | ||
234 | cachedata.extradepsfunc[fn] = self.extradepsfunc | 239 | cachedata.extradepsfunc[fn] = self.extradepsfunc |
235 | 240 | ||
241 | |||
242 | class SiggenRecipeInfo(RecipeInfoCommon): | ||
243 | __slots__ = () | ||
244 | |||
245 | classname = "SiggenRecipeInfo" | ||
246 | cachefile = "bb_cache_" + classname +".dat" | ||
247 | # we don't want to show this information in graph files so don't set cachefields | ||
248 | #cachefields = [] | ||
249 | |||
250 | def __init__(self, filename, metadata): | ||
251 | self.siggen_gendeps = metadata.getVar("__siggen_gendeps", False) | ||
252 | self.siggen_varvals = metadata.getVar("__siggen_varvals", False) | ||
253 | self.siggen_taskdeps = metadata.getVar("__siggen_taskdeps", False) | ||
254 | |||
255 | @classmethod | ||
256 | def init_cacheData(cls, cachedata): | ||
257 | cachedata.siggen_taskdeps = {} | ||
258 | cachedata.siggen_gendeps = {} | ||
259 | cachedata.siggen_varvals = {} | ||
260 | |||
261 | def add_cacheData(self, cachedata, fn): | ||
262 | cachedata.siggen_gendeps[fn] = self.siggen_gendeps | ||
263 | cachedata.siggen_varvals[fn] = self.siggen_varvals | ||
264 | cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps | ||
265 | |||
266 | # The siggen variable data is large and impacts: | ||
267 | # - bitbake's overall memory usage | ||
268 | # - the amount of data sent over IPC between parsing processes and the server | ||
269 | # - the size of the cache files on disk | ||
270 | # - the size of "sigdata" hash information files on disk | ||
271 | # The data consists of strings (some large) or frozenset lists of variables | ||
272 | # As such, we a) deplicate the data here and b) pass references to the object at second | ||
273 | # access (e.g. over IPC or saving into pickle). | ||
274 | |||
275 | store = {} | ||
276 | save_map = {} | ||
277 | save_count = 1 | ||
278 | restore_map = {} | ||
279 | restore_count = {} | ||
280 | |||
281 | @classmethod | ||
282 | def reset(cls): | ||
283 | # Needs to be called before starting new streamed data in a given process | ||
284 | # (e.g. writing out the cache again) | ||
285 | cls.save_map = {} | ||
286 | cls.save_count = 1 | ||
287 | cls.restore_map = {} | ||
288 | |||
289 | @classmethod | ||
290 | def _save(cls, deps): | ||
291 | ret = [] | ||
292 | if not deps: | ||
293 | return deps | ||
294 | for dep in deps: | ||
295 | fs = deps[dep] | ||
296 | if fs is None: | ||
297 | ret.append((dep, None, None)) | ||
298 | elif fs in cls.save_map: | ||
299 | ret.append((dep, None, cls.save_map[fs])) | ||
300 | else: | ||
301 | cls.save_map[fs] = cls.save_count | ||
302 | ret.append((dep, fs, cls.save_count)) | ||
303 | cls.save_count = cls.save_count + 1 | ||
304 | return ret | ||
305 | |||
306 | @classmethod | ||
307 | def _restore(cls, deps, pid): | ||
308 | ret = {} | ||
309 | if not deps: | ||
310 | return deps | ||
311 | if pid not in cls.restore_map: | ||
312 | cls.restore_map[pid] = {} | ||
313 | map = cls.restore_map[pid] | ||
314 | for dep, fs, mapnum in deps: | ||
315 | if fs is None and mapnum is None: | ||
316 | ret[dep] = None | ||
317 | elif fs is None: | ||
318 | ret[dep] = map[mapnum] | ||
319 | else: | ||
320 | try: | ||
321 | fs = cls.store[fs] | ||
322 | except KeyError: | ||
323 | cls.store[fs] = fs | ||
324 | map[mapnum] = fs | ||
325 | ret[dep] = fs | ||
326 | return ret | ||
327 | |||
328 | def __getstate__(self): | ||
329 | ret = {} | ||
330 | for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]: | ||
331 | ret[key] = self._save(self.__dict__[key]) | ||
332 | ret['pid'] = os.getpid() | ||
333 | return ret | ||
334 | |||
335 | def __setstate__(self, state): | ||
336 | pid = state['pid'] | ||
337 | for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]: | ||
338 | setattr(self, key, self._restore(state[key], pid)) | ||
339 | |||
340 | |||
236 | def virtualfn2realfn(virtualfn): | 341 | def virtualfn2realfn(virtualfn): |
237 | """ | 342 | """ |
238 | Convert a virtual file name to a real one + the associated subclass keyword | 343 | Convert a virtual file name to a real one + the associated subclass keyword |
239 | """ | 344 | """ |
240 | mc = "" | 345 | mc = "" |
241 | if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2: | 346 | if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2: |
242 | elems = virtualfn.split(':') | 347 | (_, mc, virtualfn) = virtualfn.split(':', 2) |
243 | mc = elems[1] | ||
244 | virtualfn = ":".join(elems[2:]) | ||
245 | 348 | ||
246 | fn = virtualfn | 349 | fn = virtualfn |
247 | cls = "" | 350 | cls = "" |
@@ -264,7 +367,7 @@ def realfn2virtual(realfn, cls, mc): | |||
264 | 367 | ||
265 | def variant2virtual(realfn, variant): | 368 | def variant2virtual(realfn, variant): |
266 | """ | 369 | """ |
267 | Convert a real filename + the associated subclass keyword to a virtual filename | 370 | Convert a real filename + a variant to a virtual filename |
268 | """ | 371 | """ |
269 | if variant == "": | 372 | if variant == "": |
270 | return realfn | 373 | return realfn |
@@ -275,96 +378,18 @@ def variant2virtual(realfn, variant): | |||
275 | return "mc:" + elems[1] + ":" + realfn | 378 | return "mc:" + elems[1] + ":" + realfn |
276 | return "virtual:" + variant + ":" + realfn | 379 | return "virtual:" + variant + ":" + realfn |
277 | 380 | ||
278 | def parse_recipe(bb_data, bbfile, appends, mc=''): | 381 | # |
279 | """ | 382 | # Cooker calls cacheValid on its recipe list, then either calls loadCached |
280 | Parse a recipe | 383 | # from it's main thread or parse from separate processes to generate an up to |
281 | """ | 384 | # date cache |
282 | 385 | # | |
283 | chdir_back = False | 386 | class Cache(object): |
284 | |||
285 | bb_data.setVar("__BBMULTICONFIG", mc) | ||
286 | |||
287 | # expand tmpdir to include this topdir | ||
288 | bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR') or "") | ||
289 | bbfile_loc = os.path.abspath(os.path.dirname(bbfile)) | ||
290 | oldpath = os.path.abspath(os.getcwd()) | ||
291 | bb.parse.cached_mtime_noerror(bbfile_loc) | ||
292 | |||
293 | # The ConfHandler first looks if there is a TOPDIR and if not | ||
294 | # then it would call getcwd(). | ||
295 | # Previously, we chdir()ed to bbfile_loc, called the handler | ||
296 | # and finally chdir()ed back, a couple of thousand times. We now | ||
297 | # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet. | ||
298 | if not bb_data.getVar('TOPDIR', False): | ||
299 | chdir_back = True | ||
300 | bb_data.setVar('TOPDIR', bbfile_loc) | ||
301 | try: | ||
302 | if appends: | ||
303 | bb_data.setVar('__BBAPPEND', " ".join(appends)) | ||
304 | bb_data = bb.parse.handle(bbfile, bb_data) | ||
305 | if chdir_back: | ||
306 | os.chdir(oldpath) | ||
307 | return bb_data | ||
308 | except: | ||
309 | if chdir_back: | ||
310 | os.chdir(oldpath) | ||
311 | raise | ||
312 | |||
313 | |||
314 | |||
315 | class NoCache(object): | ||
316 | |||
317 | def __init__(self, databuilder): | ||
318 | self.databuilder = databuilder | ||
319 | self.data = databuilder.data | ||
320 | |||
321 | def loadDataFull(self, virtualfn, appends): | ||
322 | """ | ||
323 | Return a complete set of data for fn. | ||
324 | To do this, we need to parse the file. | ||
325 | """ | ||
326 | logger.debug("Parsing %s (full)" % virtualfn) | ||
327 | (fn, virtual, mc) = virtualfn2realfn(virtualfn) | ||
328 | bb_data = self.load_bbfile(virtualfn, appends, virtonly=True) | ||
329 | return bb_data[virtual] | ||
330 | |||
331 | def load_bbfile(self, bbfile, appends, virtonly = False, mc=None): | ||
332 | """ | ||
333 | Load and parse one .bb build file | ||
334 | Return the data and whether parsing resulted in the file being skipped | ||
335 | """ | ||
336 | |||
337 | if virtonly: | ||
338 | (bbfile, virtual, mc) = virtualfn2realfn(bbfile) | ||
339 | bb_data = self.databuilder.mcdata[mc].createCopy() | ||
340 | bb_data.setVar("__ONLYFINALISE", virtual or "default") | ||
341 | datastores = parse_recipe(bb_data, bbfile, appends, mc) | ||
342 | return datastores | ||
343 | |||
344 | if mc is not None: | ||
345 | bb_data = self.databuilder.mcdata[mc].createCopy() | ||
346 | return parse_recipe(bb_data, bbfile, appends, mc) | ||
347 | |||
348 | bb_data = self.data.createCopy() | ||
349 | datastores = parse_recipe(bb_data, bbfile, appends) | ||
350 | |||
351 | for mc in self.databuilder.mcdata: | ||
352 | if not mc: | ||
353 | continue | ||
354 | bb_data = self.databuilder.mcdata[mc].createCopy() | ||
355 | newstores = parse_recipe(bb_data, bbfile, appends, mc) | ||
356 | for ns in newstores: | ||
357 | datastores["mc:%s:%s" % (mc, ns)] = newstores[ns] | ||
358 | |||
359 | return datastores | ||
360 | |||
361 | class Cache(NoCache): | ||
362 | """ | 387 | """ |
363 | BitBake Cache implementation | 388 | BitBake Cache implementation |
364 | """ | 389 | """ |
365 | def __init__(self, databuilder, mc, data_hash, caches_array): | 390 | def __init__(self, databuilder, mc, data_hash, caches_array): |
366 | super().__init__(databuilder) | 391 | self.databuilder = databuilder |
367 | data = databuilder.data | 392 | self.data = databuilder.data |
368 | 393 | ||
369 | # Pass caches_array information into Cache Constructor | 394 | # Pass caches_array information into Cache Constructor |
370 | # It will be used later for deciding whether we | 395 | # It will be used later for deciding whether we |
@@ -372,7 +397,7 @@ class Cache(NoCache): | |||
372 | self.mc = mc | 397 | self.mc = mc |
373 | self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) | 398 | self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger) |
374 | self.caches_array = caches_array | 399 | self.caches_array = caches_array |
375 | self.cachedir = data.getVar("CACHE") | 400 | self.cachedir = self.data.getVar("CACHE") |
376 | self.clean = set() | 401 | self.clean = set() |
377 | self.checked = set() | 402 | self.checked = set() |
378 | self.depends_cache = {} | 403 | self.depends_cache = {} |
@@ -382,20 +407,12 @@ class Cache(NoCache): | |||
382 | self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+') | 407 | self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+') |
383 | 408 | ||
384 | if self.cachedir in [None, '']: | 409 | if self.cachedir in [None, '']: |
385 | self.has_cache = False | 410 | bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use") |
386 | self.logger.info("Not using a cache. " | ||
387 | "Set CACHE = <directory> to enable.") | ||
388 | return | ||
389 | |||
390 | self.has_cache = True | ||
391 | 411 | ||
392 | def getCacheFile(self, cachefile): | 412 | def getCacheFile(self, cachefile): |
393 | return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) | 413 | return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash) |
394 | 414 | ||
395 | def prepare_cache(self, progress): | 415 | def prepare_cache(self, progress): |
396 | if not self.has_cache: | ||
397 | return 0 | ||
398 | |||
399 | loaded = 0 | 416 | loaded = 0 |
400 | 417 | ||
401 | self.cachefile = self.getCacheFile("bb_cache.dat") | 418 | self.cachefile = self.getCacheFile("bb_cache.dat") |
@@ -434,9 +451,6 @@ class Cache(NoCache): | |||
434 | return loaded | 451 | return loaded |
435 | 452 | ||
436 | def cachesize(self): | 453 | def cachesize(self): |
437 | if not self.has_cache: | ||
438 | return 0 | ||
439 | |||
440 | cachesize = 0 | 454 | cachesize = 0 |
441 | for cache_class in self.caches_array: | 455 | for cache_class in self.caches_array: |
442 | cachefile = self.getCacheFile(cache_class.cachefile) | 456 | cachefile = self.getCacheFile(cache_class.cachefile) |
@@ -498,11 +512,11 @@ class Cache(NoCache): | |||
498 | 512 | ||
499 | return len(self.depends_cache) | 513 | return len(self.depends_cache) |
500 | 514 | ||
501 | def parse(self, filename, appends): | 515 | def parse(self, filename, appends, layername): |
502 | """Parse the specified filename, returning the recipe information""" | 516 | """Parse the specified filename, returning the recipe information""" |
503 | self.logger.debug("Parsing %s", filename) | 517 | self.logger.debug("Parsing %s", filename) |
504 | infos = [] | 518 | infos = [] |
505 | datastores = self.load_bbfile(filename, appends, mc=self.mc) | 519 | datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc, layername=layername) |
506 | depends = [] | 520 | depends = [] |
507 | variants = [] | 521 | variants = [] |
508 | # Process the "real" fn last so we can store variants list | 522 | # Process the "real" fn last so we can store variants list |
@@ -524,43 +538,19 @@ class Cache(NoCache): | |||
524 | 538 | ||
525 | return infos | 539 | return infos |
526 | 540 | ||
527 | def load(self, filename, appends): | 541 | def loadCached(self, filename, appends): |
528 | """Obtain the recipe information for the specified filename, | 542 | """Obtain the recipe information for the specified filename, |
529 | using cached values if available, otherwise parsing. | 543 | using cached values. |
530 | 544 | """ | |
531 | Note that if it does parse to obtain the info, it will not | ||
532 | automatically add the information to the cache or to your | ||
533 | CacheData. Use the add or add_info method to do so after | ||
534 | running this, or use loadData instead.""" | ||
535 | cached = self.cacheValid(filename, appends) | ||
536 | if cached: | ||
537 | infos = [] | ||
538 | # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo] | ||
539 | info_array = self.depends_cache[filename] | ||
540 | for variant in info_array[0].variants: | ||
541 | virtualfn = variant2virtual(filename, variant) | ||
542 | infos.append((virtualfn, self.depends_cache[virtualfn])) | ||
543 | else: | ||
544 | return self.parse(filename, appends, configdata, self.caches_array) | ||
545 | |||
546 | return cached, infos | ||
547 | |||
548 | def loadData(self, fn, appends, cacheData): | ||
549 | """Load the recipe info for the specified filename, | ||
550 | parsing and adding to the cache if necessary, and adding | ||
551 | the recipe information to the supplied CacheData instance.""" | ||
552 | skipped, virtuals = 0, 0 | ||
553 | 545 | ||
554 | cached, infos = self.load(fn, appends) | 546 | infos = [] |
555 | for virtualfn, info_array in infos: | 547 | # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo] |
556 | if info_array[0].skipped: | 548 | info_array = self.depends_cache[filename] |
557 | self.logger.debug("Skipping %s: %s", virtualfn, info_array[0].skipreason) | 549 | for variant in info_array[0].variants: |
558 | skipped += 1 | 550 | virtualfn = variant2virtual(filename, variant) |
559 | else: | 551 | infos.append((virtualfn, self.depends_cache[virtualfn])) |
560 | self.add_info(virtualfn, info_array, cacheData, not cached) | ||
561 | virtuals += 1 | ||
562 | 552 | ||
563 | return cached, skipped, virtuals | 553 | return infos |
564 | 554 | ||
565 | def cacheValid(self, fn, appends): | 555 | def cacheValid(self, fn, appends): |
566 | """ | 556 | """ |
@@ -569,10 +559,6 @@ class Cache(NoCache): | |||
569 | """ | 559 | """ |
570 | if fn not in self.checked: | 560 | if fn not in self.checked: |
571 | self.cacheValidUpdate(fn, appends) | 561 | self.cacheValidUpdate(fn, appends) |
572 | |||
573 | # Is cache enabled? | ||
574 | if not self.has_cache: | ||
575 | return False | ||
576 | if fn in self.clean: | 562 | if fn in self.clean: |
577 | return True | 563 | return True |
578 | return False | 564 | return False |
@@ -582,10 +568,6 @@ class Cache(NoCache): | |||
582 | Is the cache valid for fn? | 568 | Is the cache valid for fn? |
583 | Make thorough (slower) checks including timestamps. | 569 | Make thorough (slower) checks including timestamps. |
584 | """ | 570 | """ |
585 | # Is cache enabled? | ||
586 | if not self.has_cache: | ||
587 | return False | ||
588 | |||
589 | self.checked.add(fn) | 571 | self.checked.add(fn) |
590 | 572 | ||
591 | # File isn't in depends_cache | 573 | # File isn't in depends_cache |
@@ -636,7 +618,7 @@ class Cache(NoCache): | |||
636 | for f in flist: | 618 | for f in flist: |
637 | if not f: | 619 | if not f: |
638 | continue | 620 | continue |
639 | f, exist = f.split(":") | 621 | f, exist = f.rsplit(":", 1) |
640 | if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): | 622 | if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)): |
641 | self.logger.debug2("%s's file checksum list file %s changed", | 623 | self.logger.debug2("%s's file checksum list file %s changed", |
642 | fn, f) | 624 | fn, f) |
@@ -692,10 +674,6 @@ class Cache(NoCache): | |||
692 | Save the cache | 674 | Save the cache |
693 | Called from the parser when complete (or exiting) | 675 | Called from the parser when complete (or exiting) |
694 | """ | 676 | """ |
695 | |||
696 | if not self.has_cache: | ||
697 | return | ||
698 | |||
699 | if self.cacheclean: | 677 | if self.cacheclean: |
700 | self.logger.debug2("Cache is clean, not saving.") | 678 | self.logger.debug2("Cache is clean, not saving.") |
701 | return | 679 | return |
@@ -716,6 +694,7 @@ class Cache(NoCache): | |||
716 | p.dump(info) | 694 | p.dump(info) |
717 | 695 | ||
718 | del self.depends_cache | 696 | del self.depends_cache |
697 | SiggenRecipeInfo.reset() | ||
719 | 698 | ||
720 | @staticmethod | 699 | @staticmethod |
721 | def mtime(cachefile): | 700 | def mtime(cachefile): |
@@ -738,26 +717,11 @@ class Cache(NoCache): | |||
738 | if watcher: | 717 | if watcher: |
739 | watcher(info_array[0].file_depends) | 718 | watcher(info_array[0].file_depends) |
740 | 719 | ||
741 | if not self.has_cache: | ||
742 | return | ||
743 | |||
744 | if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache: | 720 | if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache: |
745 | if parsed: | 721 | if parsed: |
746 | self.cacheclean = False | 722 | self.cacheclean = False |
747 | self.depends_cache[filename] = info_array | 723 | self.depends_cache[filename] = info_array |
748 | 724 | ||
749 | def add(self, file_name, data, cacheData, parsed=None): | ||
750 | """ | ||
751 | Save data we need into the cache | ||
752 | """ | ||
753 | |||
754 | realfn = virtualfn2realfn(file_name)[0] | ||
755 | |||
756 | info_array = [] | ||
757 | for cache_class in self.caches_array: | ||
758 | info_array.append(cache_class(realfn, data)) | ||
759 | self.add_info(file_name, info_array, cacheData, parsed) | ||
760 | |||
761 | class MulticonfigCache(Mapping): | 725 | class MulticonfigCache(Mapping): |
762 | def __init__(self, databuilder, data_hash, caches_array): | 726 | def __init__(self, databuilder, data_hash, caches_array): |
763 | def progress(p): | 727 | def progress(p): |
@@ -794,6 +758,7 @@ class MulticonfigCache(Mapping): | |||
794 | loaded = 0 | 758 | loaded = 0 |
795 | 759 | ||
796 | for c in self.__caches.values(): | 760 | for c in self.__caches.values(): |
761 | SiggenRecipeInfo.reset() | ||
797 | loaded += c.prepare_cache(progress) | 762 | loaded += c.prepare_cache(progress) |
798 | previous_progress = current_progress | 763 | previous_progress = current_progress |
799 | 764 | ||
@@ -871,11 +836,10 @@ class MultiProcessCache(object): | |||
871 | self.cachedata = self.create_cachedata() | 836 | self.cachedata = self.create_cachedata() |
872 | self.cachedata_extras = self.create_cachedata() | 837 | self.cachedata_extras = self.create_cachedata() |
873 | 838 | ||
874 | def init_cache(self, d, cache_file_name=None): | 839 | def init_cache(self, cachedir, cache_file_name=None): |
875 | cachedir = (d.getVar("PERSISTENT_DIR") or | 840 | if not cachedir: |
876 | d.getVar("CACHE")) | ||
877 | if cachedir in [None, '']: | ||
878 | return | 841 | return |
842 | |||
879 | bb.utils.mkdirhier(cachedir) | 843 | bb.utils.mkdirhier(cachedir) |
880 | self.cachefile = os.path.join(cachedir, | 844 | self.cachefile = os.path.join(cachedir, |
881 | cache_file_name or self.__class__.cache_file_name) | 845 | cache_file_name or self.__class__.cache_file_name) |
@@ -906,6 +870,10 @@ class MultiProcessCache(object): | |||
906 | if not self.cachefile: | 870 | if not self.cachefile: |
907 | return | 871 | return |
908 | 872 | ||
873 | have_data = any(self.cachedata_extras) | ||
874 | if not have_data: | ||
875 | return | ||
876 | |||
909 | glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True) | 877 | glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True) |
910 | 878 | ||
911 | i = os.getpid() | 879 | i = os.getpid() |
@@ -940,6 +908,8 @@ class MultiProcessCache(object): | |||
940 | 908 | ||
941 | data = self.cachedata | 909 | data = self.cachedata |
942 | 910 | ||
911 | have_data = False | ||
912 | |||
943 | for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]: | 913 | for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]: |
944 | f = os.path.join(os.path.dirname(self.cachefile), f) | 914 | f = os.path.join(os.path.dirname(self.cachefile), f) |
945 | try: | 915 | try: |
@@ -954,12 +924,14 @@ class MultiProcessCache(object): | |||
954 | os.unlink(f) | 924 | os.unlink(f) |
955 | continue | 925 | continue |
956 | 926 | ||
927 | have_data = True | ||
957 | self.merge_data(extradata, data) | 928 | self.merge_data(extradata, data) |
958 | os.unlink(f) | 929 | os.unlink(f) |
959 | 930 | ||
960 | with open(self.cachefile, "wb") as f: | 931 | if have_data: |
961 | p = pickle.Pickler(f, -1) | 932 | with open(self.cachefile, "wb") as f: |
962 | p.dump([data, self.__class__.CACHE_VERSION]) | 933 | p = pickle.Pickler(f, -1) |
934 | p.dump([data, self.__class__.CACHE_VERSION]) | ||
963 | 935 | ||
964 | bb.utils.unlockfile(glf) | 936 | bb.utils.unlockfile(glf) |
965 | 937 | ||
@@ -1015,3 +987,11 @@ class SimpleCache(object): | |||
1015 | p.dump([data, self.cacheversion]) | 987 | p.dump([data, self.cacheversion]) |
1016 | 988 | ||
1017 | bb.utils.unlockfile(glf) | 989 | bb.utils.unlockfile(glf) |
990 | |||
991 | def copyfile(self, target): | ||
992 | if not self.cachefile: | ||
993 | return | ||
994 | |||
995 | glf = bb.utils.lockfile(self.cachefile + ".lock") | ||
996 | shutil.copy(self.cachefile, target) | ||
997 | bb.utils.unlockfile(glf) | ||