summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/cache.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/cache.py')
-rw-r--r--bitbake/lib/bb/cache.py988
1 files changed, 0 insertions, 988 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
deleted file mode 100644
index 2361c5684d..0000000000
--- a/bitbake/lib/bb/cache.py
+++ /dev/null
@@ -1,988 +0,0 @@
1#
2# BitBake Cache implementation
3#
4# Caching of bitbake variables before task execution
5
6# Copyright (C) 2006 Richard Purdie
7# Copyright (C) 2012 Intel Corporation
8
9# but small sections based on code from bin/bitbake:
10# Copyright (C) 2003, 2004 Chris Larson
11# Copyright (C) 2003, 2004 Phil Blundell
12# Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer
13# Copyright (C) 2005 Holger Hans Peter Freyther
14# Copyright (C) 2005 ROAD GmbH
15#
16# SPDX-License-Identifier: GPL-2.0-only
17#
18
19import os
20import logging
21import pickle
22from collections import defaultdict
23from collections.abc import Mapping
24import bb.utils
25from bb import PrefixLoggerAdapter
26import re
27import shutil
28
29logger = logging.getLogger("BitBake.Cache")
30
31__cache_version__ = "156"
32
33def getCacheFile(path, filename, mc, data_hash):
34 mcspec = ''
35 if mc:
36 mcspec = ".%s" % mc
37 return os.path.join(path, filename + mcspec + "." + data_hash)
38
39# RecipeInfoCommon defines common data retrieving methods
40# from meta data for caches. CoreRecipeInfo as well as other
41# Extra RecipeInfo needs to inherit this class
42class RecipeInfoCommon(object):
43
44 @classmethod
45 def listvar(cls, var, metadata):
46 return cls.getvar(var, metadata).split()
47
48 @classmethod
49 def intvar(cls, var, metadata):
50 return int(cls.getvar(var, metadata) or 0)
51
52 @classmethod
53 def depvar(cls, var, metadata):
54 return bb.utils.explode_deps(cls.getvar(var, metadata))
55
56 @classmethod
57 def pkgvar(cls, var, packages, metadata):
58 return dict((pkg, cls.depvar("%s:%s" % (var, pkg), metadata))
59 for pkg in packages)
60
61 @classmethod
62 def taskvar(cls, var, tasks, metadata):
63 return dict((task, cls.getvar("%s:task-%s" % (var, task), metadata))
64 for task in tasks)
65
66 @classmethod
67 def flaglist(cls, flag, varlist, metadata, squash=False):
68 out_dict = dict((var, metadata.getVarFlag(var, flag))
69 for var in varlist)
70 if squash:
71 return dict((k,v) for (k,v) in out_dict.items() if v)
72 else:
73 return out_dict
74
75 @classmethod
76 def getvar(cls, var, metadata, expand = True):
77 return metadata.getVar(var, expand) or ''
78
79
80class CoreRecipeInfo(RecipeInfoCommon):
81 __slots__ = ()
82
83 cachefile = "bb_cache.dat"
84
85 def __init__(self, filename, metadata):
86 self.file_depends = metadata.getVar('__depends', False)
87 self.timestamp = bb.parse.cached_mtime(filename)
88 self.variants = self.listvar('__VARIANTS', metadata) + ['']
89 self.appends = self.listvar('__BBAPPEND', metadata)
90 self.nocache = self.getvar('BB_DONT_CACHE', metadata)
91
92 self.provides = self.depvar('PROVIDES', metadata)
93 self.rprovides = self.depvar('RPROVIDES', metadata)
94 self.pn = self.getvar('PN', metadata) or bb.parse.vars_from_file(filename,metadata)[0]
95 self.packages = self.listvar('PACKAGES', metadata)
96 if not self.packages:
97 self.packages.append(self.pn)
98 self.packages_dynamic = self.listvar('PACKAGES_DYNAMIC', metadata)
99 self.rprovides_pkg = self.pkgvar('RPROVIDES', self.packages, metadata)
100
101 self.skipreason = self.getvar('__SKIPPED', metadata)
102 if self.skipreason:
103 self.skipped = True
104 return
105
106 self.tasks = metadata.getVar('__BBTASKS', False)
107
108 self.basetaskhashes = metadata.getVar('__siggen_basehashes', False) or {}
109 self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
110
111 self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}
112
113 self.skipped = False
114 self.pe = self.getvar('PE', metadata)
115 self.pv = self.getvar('PV', metadata)
116 self.pr = self.getvar('PR', metadata)
117 self.defaultpref = self.intvar('DEFAULT_PREFERENCE', metadata)
118 self.not_world = self.getvar('EXCLUDE_FROM_WORLD', metadata)
119 self.stamp = self.getvar('STAMP', metadata)
120 self.stampclean = self.getvar('STAMPCLEAN', metadata)
121 self.stamp_extrainfo = self.flaglist('stamp-extra-info', self.tasks, metadata)
122 self.file_checksums = self.flaglist('file-checksums', self.tasks, metadata, True)
123 self.depends = self.depvar('DEPENDS', metadata)
124 self.rdepends = self.depvar('RDEPENDS', metadata)
125 self.rrecommends = self.depvar('RRECOMMENDS', metadata)
126 self.rdepends_pkg = self.pkgvar('RDEPENDS', self.packages, metadata)
127 self.rrecommends_pkg = self.pkgvar('RRECOMMENDS', self.packages, metadata)
128 self.inherits = self.getvar('__inherit_cache', metadata, expand=False)
129 self.fakerootenv = self.getvar('FAKEROOTENV', metadata)
130 self.fakerootdirs = self.getvar('FAKEROOTDIRS', metadata)
131 self.fakerootlogs = self.getvar('FAKEROOTLOGS', metadata)
132 self.fakerootnoenv = self.getvar('FAKEROOTNOENV', metadata)
133 self.extradepsfunc = self.getvar('calculate_extra_depends', metadata)
134
135 @classmethod
136 def init_cacheData(cls, cachedata):
137 # CacheData in Core RecipeInfo Class
138 cachedata.task_deps = {}
139 cachedata.pkg_fn = {}
140 cachedata.pkg_pn = defaultdict(list)
141 cachedata.pkg_pepvpr = {}
142 cachedata.pkg_dp = {}
143
144 cachedata.stamp = {}
145 cachedata.stampclean = {}
146 cachedata.stamp_extrainfo = {}
147 cachedata.file_checksums = {}
148 cachedata.fn_provides = {}
149 cachedata.pn_provides = defaultdict(list)
150 cachedata.all_depends = []
151
152 cachedata.deps = defaultdict(list)
153 cachedata.packages = defaultdict(list)
154 cachedata.providers = defaultdict(list)
155 cachedata.rproviders = defaultdict(list)
156 cachedata.packages_dynamic = defaultdict(list)
157
158 cachedata.rundeps = defaultdict(lambda: defaultdict(list))
159 cachedata.runrecs = defaultdict(lambda: defaultdict(list))
160 cachedata.possible_world = []
161 cachedata.universe_target = []
162 cachedata.hashfn = {}
163
164 cachedata.basetaskhash = {}
165 cachedata.inherits = {}
166 cachedata.fakerootenv = {}
167 cachedata.fakerootnoenv = {}
168 cachedata.fakerootdirs = {}
169 cachedata.fakerootlogs = {}
170 cachedata.extradepsfunc = {}
171
172 def add_cacheData(self, cachedata, fn):
173 cachedata.task_deps[fn] = self.task_deps
174 cachedata.pkg_fn[fn] = self.pn
175 cachedata.pkg_pn[self.pn].append(fn)
176 cachedata.pkg_pepvpr[fn] = (self.pe, self.pv, self.pr)
177 cachedata.pkg_dp[fn] = self.defaultpref
178 cachedata.stamp[fn] = self.stamp
179 cachedata.stampclean[fn] = self.stampclean
180 cachedata.stamp_extrainfo[fn] = self.stamp_extrainfo
181 cachedata.file_checksums[fn] = self.file_checksums
182
183 provides = [self.pn]
184 for provide in self.provides:
185 if provide not in provides:
186 provides.append(provide)
187 cachedata.fn_provides[fn] = provides
188
189 for provide in provides:
190 cachedata.providers[provide].append(fn)
191 if provide not in cachedata.pn_provides[self.pn]:
192 cachedata.pn_provides[self.pn].append(provide)
193
194 for dep in self.depends:
195 if dep not in cachedata.deps[fn]:
196 cachedata.deps[fn].append(dep)
197 if dep not in cachedata.all_depends:
198 cachedata.all_depends.append(dep)
199
200 rprovides = self.rprovides
201 for package in self.packages:
202 cachedata.packages[package].append(fn)
203 rprovides += self.rprovides_pkg[package]
204
205 for rprovide in rprovides:
206 if fn not in cachedata.rproviders[rprovide]:
207 cachedata.rproviders[rprovide].append(fn)
208
209 for package in self.packages_dynamic:
210 cachedata.packages_dynamic[package].append(fn)
211
212 # Build hash of runtime depends and recommends
213 for package in self.packages:
214 cachedata.rundeps[fn][package] = list(self.rdepends) + self.rdepends_pkg[package]
215 cachedata.runrecs[fn][package] = list(self.rrecommends) + self.rrecommends_pkg[package]
216
217 # Collect files we may need for possible world-dep
218 # calculations
219 if not bb.utils.to_boolean(self.not_world):
220 cachedata.possible_world.append(fn)
221 #else:
222 # logger.debug2("EXCLUDE FROM WORLD: %s", fn)
223
224 # create a collection of all targets for sanity checking
225 # tasks, such as upstream versions, license, and tools for
226 # task and image creation.
227 cachedata.universe_target.append(self.pn)
228
229 cachedata.hashfn[fn] = self.hashfilename
230 for task, taskhash in self.basetaskhashes.items():
231 identifier = '%s:%s' % (fn, task)
232 cachedata.basetaskhash[identifier] = taskhash
233
234 cachedata.inherits[fn] = self.inherits
235 cachedata.fakerootenv[fn] = self.fakerootenv
236 cachedata.fakerootnoenv[fn] = self.fakerootnoenv
237 cachedata.fakerootdirs[fn] = self.fakerootdirs
238 cachedata.fakerootlogs[fn] = self.fakerootlogs
239 cachedata.extradepsfunc[fn] = self.extradepsfunc
240
241
242class SiggenRecipeInfo(RecipeInfoCommon):
243 __slots__ = ()
244
245 classname = "SiggenRecipeInfo"
246 cachefile = "bb_cache_" + classname +".dat"
247 # we don't want to show this information in graph files so don't set cachefields
248 #cachefields = []
249
250 def __init__(self, filename, metadata):
251 self.siggen_gendeps = metadata.getVar("__siggen_gendeps", False)
252 self.siggen_varvals = metadata.getVar("__siggen_varvals", False)
253 self.siggen_taskdeps = metadata.getVar("__siggen_taskdeps", False)
254
255 @classmethod
256 def init_cacheData(cls, cachedata):
257 cachedata.siggen_taskdeps = {}
258 cachedata.siggen_gendeps = {}
259 cachedata.siggen_varvals = {}
260
261 def add_cacheData(self, cachedata, fn):
262 cachedata.siggen_gendeps[fn] = self.siggen_gendeps
263 cachedata.siggen_varvals[fn] = self.siggen_varvals
264 cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps
265
266 # The siggen variable data is large and impacts:
267 # - bitbake's overall memory usage
268 # - the amount of data sent over IPC between parsing processes and the server
269 # - the size of the cache files on disk
270 # - the size of "sigdata" hash information files on disk
271 # The data consists of strings (some large) or frozenset lists of variables
272 # As such, we a) deplicate the data here and b) pass references to the object at second
273 # access (e.g. over IPC or saving into pickle).
274
275 store = {}
276 save_map = {}
277 save_count = 1
278 restore_map = {}
279 restore_count = {}
280
281 @classmethod
282 def reset(cls):
283 # Needs to be called before starting new streamed data in a given process
284 # (e.g. writing out the cache again)
285 cls.save_map = {}
286 cls.save_count = 1
287 cls.restore_map = {}
288
289 @classmethod
290 def _save(cls, deps):
291 ret = []
292 if not deps:
293 return deps
294 for dep in deps:
295 fs = deps[dep]
296 if fs is None:
297 ret.append((dep, None, None))
298 elif fs in cls.save_map:
299 ret.append((dep, None, cls.save_map[fs]))
300 else:
301 cls.save_map[fs] = cls.save_count
302 ret.append((dep, fs, cls.save_count))
303 cls.save_count = cls.save_count + 1
304 return ret
305
306 @classmethod
307 def _restore(cls, deps, pid):
308 ret = {}
309 if not deps:
310 return deps
311 if pid not in cls.restore_map:
312 cls.restore_map[pid] = {}
313 map = cls.restore_map[pid]
314 for dep, fs, mapnum in deps:
315 if fs is None and mapnum is None:
316 ret[dep] = None
317 elif fs is None:
318 ret[dep] = map[mapnum]
319 else:
320 try:
321 fs = cls.store[fs]
322 except KeyError:
323 cls.store[fs] = fs
324 map[mapnum] = fs
325 ret[dep] = fs
326 return ret
327
328 def __getstate__(self):
329 ret = {}
330 for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
331 ret[key] = self._save(self.__dict__[key])
332 ret['pid'] = os.getpid()
333 return ret
334
335 def __setstate__(self, state):
336 pid = state['pid']
337 for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
338 setattr(self, key, self._restore(state[key], pid))
339
340
341def virtualfn2realfn(virtualfn):
342 """
343 Convert a virtual file name to a real one + the associated subclass keyword
344 """
345 mc = ""
346 if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2:
347 (_, mc, virtualfn) = virtualfn.split(':', 2)
348
349 fn = virtualfn
350 cls = ""
351 if virtualfn.startswith('virtual:'):
352 elems = virtualfn.split(':')
353 cls = ":".join(elems[1:-1])
354 fn = elems[-1]
355
356 return (fn, cls, mc)
357
358def realfn2virtual(realfn, cls, mc):
359 """
360 Convert a real filename + the associated subclass keyword to a virtual filename
361 """
362 if cls:
363 realfn = "virtual:" + cls + ":" + realfn
364 if mc:
365 realfn = "mc:" + mc + ":" + realfn
366 return realfn
367
368def variant2virtual(realfn, variant):
369 """
370 Convert a real filename + a variant to a virtual filename
371 """
372 if variant == "":
373 return realfn
374 if variant.startswith("mc:") and variant.count(':') >= 2:
375 elems = variant.split(":")
376 if elems[2]:
377 return "mc:" + elems[1] + ":virtual:" + ":".join(elems[2:]) + ":" + realfn
378 return "mc:" + elems[1] + ":" + realfn
379 return "virtual:" + variant + ":" + realfn
380
381#
382# Cooker calls cacheValid on its recipe list, then either calls loadCached
383# from it's main thread or parse from separate processes to generate an up to
384# date cache
385#
386class Cache(object):
387 """
388 BitBake Cache implementation
389 """
390 def __init__(self, databuilder, mc, data_hash, caches_array):
391 self.databuilder = databuilder
392 self.data = databuilder.data
393
394 # Pass caches_array information into Cache Constructor
395 # It will be used later for deciding whether we
396 # need extra cache file dump/load support
397 self.mc = mc
398 self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else ''), logger)
399 self.caches_array = caches_array
400 self.cachedir = self.data.getVar("CACHE")
401 self.clean = set()
402 self.checked = set()
403 self.depends_cache = {}
404 self.data_fn = None
405 self.cacheclean = True
406 self.data_hash = data_hash
407 self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
408
409 if self.cachedir in [None, '']:
410 bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use")
411
412 def getCacheFile(self, cachefile):
413 return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash)
414
415 def prepare_cache(self, progress):
416 loaded = 0
417
418 self.cachefile = self.getCacheFile("bb_cache.dat")
419
420 self.logger.debug("Cache dir: %s", self.cachedir)
421 bb.utils.mkdirhier(self.cachedir)
422
423 cache_ok = True
424 if self.caches_array:
425 for cache_class in self.caches_array:
426 cachefile = self.getCacheFile(cache_class.cachefile)
427 cache_exists = os.path.exists(cachefile)
428 self.logger.debug2("Checking if %s exists: %r", cachefile, cache_exists)
429 cache_ok = cache_ok and cache_exists
430 cache_class.init_cacheData(self)
431 if cache_ok:
432 loaded = self.load_cachefile(progress)
433 elif os.path.isfile(self.cachefile):
434 self.logger.info("Out of date cache found, rebuilding...")
435 else:
436 self.logger.debug("Cache file %s not found, building..." % self.cachefile)
437
438 # We don't use the symlink, its just for debugging convinience
439 if self.mc:
440 symlink = os.path.join(self.cachedir, "bb_cache.dat.%s" % self.mc)
441 else:
442 symlink = os.path.join(self.cachedir, "bb_cache.dat")
443
444 if os.path.exists(symlink) or os.path.islink(symlink):
445 bb.utils.remove(symlink)
446 try:
447 os.symlink(os.path.basename(self.cachefile), symlink)
448 except OSError:
449 pass
450
451 return loaded
452
453 def cachesize(self):
454 cachesize = 0
455 for cache_class in self.caches_array:
456 cachefile = self.getCacheFile(cache_class.cachefile)
457 try:
458 with open(cachefile, "rb") as cachefile:
459 cachesize += os.fstat(cachefile.fileno()).st_size
460 except FileNotFoundError:
461 pass
462
463 return cachesize
464
465 def load_cachefile(self, progress):
466 previous_progress = 0
467
468 for cache_class in self.caches_array:
469 cachefile = self.getCacheFile(cache_class.cachefile)
470 self.logger.debug('Loading cache file: %s' % cachefile)
471 with open(cachefile, "rb") as cachefile:
472 pickled = pickle.Unpickler(cachefile)
473 # Check cache version information
474 try:
475 cache_ver = pickled.load()
476 bitbake_ver = pickled.load()
477 except Exception:
478 self.logger.info('Invalid cache, rebuilding...')
479 return 0
480
481 if cache_ver != __cache_version__:
482 self.logger.info('Cache version mismatch, rebuilding...')
483 return 0
484 elif bitbake_ver != bb.__version__:
485 self.logger.info('Bitbake version mismatch, rebuilding...')
486 return 0
487
488 # Load the rest of the cache file
489 current_progress = 0
490 while cachefile:
491 try:
492 key = pickled.load()
493 value = pickled.load()
494 except Exception:
495 break
496 if not isinstance(key, str):
497 bb.warn("%s from extras cache is not a string?" % key)
498 break
499 if not isinstance(value, RecipeInfoCommon):
500 bb.warn("%s from extras cache is not a RecipeInfoCommon class?" % value)
501 break
502
503 if key in self.depends_cache:
504 self.depends_cache[key].append(value)
505 else:
506 self.depends_cache[key] = [value]
507 # only fire events on even percentage boundaries
508 current_progress = cachefile.tell() + previous_progress
509 progress(cachefile.tell() + previous_progress)
510
511 previous_progress += current_progress
512
513 return len(self.depends_cache)
514
515 def parse(self, filename, appends, layername):
516 """Parse the specified filename, returning the recipe information"""
517 self.logger.debug("Parsing %s", filename)
518 infos = []
519 datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc, layername=layername)
520 depends = []
521 variants = []
522 # Process the "real" fn last so we can store variants list
523 for variant, data in sorted(datastores.items(),
524 key=lambda i: i[0],
525 reverse=True):
526 virtualfn = variant2virtual(filename, variant)
527 variants.append(variant)
528 depends = depends + (data.getVar("__depends", False) or [])
529 if depends and not variant:
530 data.setVar("__depends", depends)
531 if virtualfn == filename:
532 data.setVar("__VARIANTS", " ".join(variants))
533 info_array = []
534 for cache_class in self.caches_array:
535 info = cache_class(filename, data)
536 info_array.append(info)
537 infos.append((virtualfn, info_array))
538
539 return infos
540
541 def loadCached(self, filename, appends):
542 """Obtain the recipe information for the specified filename,
543 using cached values.
544 """
545
546 infos = []
547 # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
548 info_array = self.depends_cache[filename]
549 for variant in info_array[0].variants:
550 virtualfn = variant2virtual(filename, variant)
551 infos.append((virtualfn, self.depends_cache[virtualfn]))
552
553 return infos
554
555 def cacheValid(self, fn, appends):
556 """
557 Is the cache valid for fn?
558 Fast version, no timestamps checked.
559 """
560 if fn not in self.checked:
561 self.cacheValidUpdate(fn, appends)
562 if fn in self.clean:
563 return True
564 return False
565
566 def cacheValidUpdate(self, fn, appends):
567 """
568 Is the cache valid for fn?
569 Make thorough (slower) checks including timestamps.
570 """
571 self.checked.add(fn)
572
573 # File isn't in depends_cache
574 if not fn in self.depends_cache:
575 self.logger.debug2("%s is not cached", fn)
576 return False
577
578 mtime = bb.parse.cached_mtime_noerror(fn)
579
580 # Check file still exists
581 if mtime == 0:
582 self.logger.debug2("%s no longer exists", fn)
583 self.remove(fn)
584 return False
585
586 info_array = self.depends_cache[fn]
587 # Check the file's timestamp
588 if mtime != info_array[0].timestamp:
589 self.logger.debug2("%s changed", fn)
590 self.remove(fn)
591 return False
592
593 # Check dependencies are still valid
594 depends = info_array[0].file_depends
595 if depends:
596 for f, old_mtime in depends:
597 fmtime = bb.parse.cached_mtime_noerror(f)
598 # Check if file still exists
599 if old_mtime != 0 and fmtime == 0:
600 self.logger.debug2("%s's dependency %s was removed",
601 fn, f)
602 self.remove(fn)
603 return False
604
605 if (fmtime != old_mtime):
606 self.logger.debug2("%s's dependency %s changed",
607 fn, f)
608 self.remove(fn)
609 return False
610
611 if hasattr(info_array[0], 'file_checksums'):
612 for _, fl in info_array[0].file_checksums.items():
613 fl = fl.strip()
614 if not fl:
615 continue
616 # Have to be careful about spaces and colons in filenames
617 flist = self.filelist_regex.split(fl)
618 for f in flist:
619 if not f:
620 continue
621 f, exist = f.rsplit(":", 1)
622 if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
623 self.logger.debug2("%s's file checksum list file %s changed",
624 fn, f)
625 self.remove(fn)
626 return False
627
628 if tuple(appends) != tuple(info_array[0].appends):
629 self.logger.debug2("appends for %s changed", fn)
630 self.logger.debug2("%s to %s" % (str(appends), str(info_array[0].appends)))
631 self.remove(fn)
632 return False
633
634 invalid = False
635 for cls in info_array[0].variants:
636 virtualfn = variant2virtual(fn, cls)
637 self.clean.add(virtualfn)
638 if virtualfn not in self.depends_cache:
639 self.logger.debug2("%s is not cached", virtualfn)
640 invalid = True
641 elif len(self.depends_cache[virtualfn]) != len(self.caches_array):
642 self.logger.debug2("Extra caches missing for %s?" % virtualfn)
643 invalid = True
644
645 # If any one of the variants is not present, mark as invalid for all
646 if invalid:
647 for cls in info_array[0].variants:
648 virtualfn = variant2virtual(fn, cls)
649 if virtualfn in self.clean:
650 self.logger.debug2("Removing %s from cache", virtualfn)
651 self.clean.remove(virtualfn)
652 if fn in self.clean:
653 self.logger.debug2("Marking %s as not clean", fn)
654 self.clean.remove(fn)
655 return False
656
657 self.clean.add(fn)
658 return True
659
660 def remove(self, fn):
661 """
662 Remove a fn from the cache
663 Called from the parser in error cases
664 """
665 if fn in self.depends_cache:
666 self.logger.debug("Removing %s from cache", fn)
667 del self.depends_cache[fn]
668 if fn in self.clean:
669 self.logger.debug("Marking %s as unclean", fn)
670 self.clean.remove(fn)
671
672 def sync(self):
673 """
674 Save the cache
675 Called from the parser when complete (or exiting)
676 """
677 if self.cacheclean:
678 self.logger.debug2("Cache is clean, not saving.")
679 return
680
681 for cache_class in self.caches_array:
682 cache_class_name = cache_class.__name__
683 cachefile = self.getCacheFile(cache_class.cachefile)
684 self.logger.debug2("Writing %s", cachefile)
685 with open(cachefile, "wb") as f:
686 p = pickle.Pickler(f, pickle.HIGHEST_PROTOCOL)
687 p.dump(__cache_version__)
688 p.dump(bb.__version__)
689
690 for key, info_array in self.depends_cache.items():
691 for info in info_array:
692 if isinstance(info, RecipeInfoCommon) and info.__class__.__name__ == cache_class_name:
693 p.dump(key)
694 p.dump(info)
695
696 del self.depends_cache
697 SiggenRecipeInfo.reset()
698
699 @staticmethod
700 def mtime(cachefile):
701 return bb.parse.cached_mtime_noerror(cachefile)
702
703 def add_info(self, filename, info_array, cacheData, parsed=None, watcher=None):
704 if self.mc is not None:
705 (fn, cls, mc) = virtualfn2realfn(filename)
706 if mc:
707 self.logger.error("Unexpected multiconfig %s", filename)
708 return
709
710 vfn = realfn2virtual(fn, cls, self.mc)
711 else:
712 vfn = filename
713
714 if isinstance(info_array[0], CoreRecipeInfo) and (not info_array[0].skipped):
715 cacheData.add_from_recipeinfo(vfn, info_array)
716
717 if watcher:
718 watcher(info_array[0].file_depends)
719
720 if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache:
721 if parsed:
722 self.cacheclean = False
723 self.depends_cache[filename] = info_array
724
725class MulticonfigCache(Mapping):
726 def __init__(self, databuilder, data_hash, caches_array):
727 def progress(p):
728 nonlocal current_progress
729 nonlocal previous_progress
730 nonlocal previous_percent
731 nonlocal cachesize
732
733 current_progress = previous_progress + p
734
735 if current_progress > cachesize:
736 # we might have calculated incorrect total size because a file
737 # might've been written out just after we checked its size
738 cachesize = current_progress
739 current_percent = 100 * current_progress / cachesize
740 if current_percent > previous_percent:
741 previous_percent = current_percent
742 bb.event.fire(bb.event.CacheLoadProgress(current_progress, cachesize),
743 databuilder.data)
744
745
746 cachesize = 0
747 current_progress = 0
748 previous_progress = 0
749 previous_percent = 0
750 self.__caches = {}
751
752 for mc, mcdata in databuilder.mcdata.items():
753 self.__caches[mc] = Cache(databuilder, mc, data_hash, caches_array)
754
755 cachesize += self.__caches[mc].cachesize()
756
757 bb.event.fire(bb.event.CacheLoadStarted(cachesize), databuilder.data)
758 loaded = 0
759
760 for c in self.__caches.values():
761 SiggenRecipeInfo.reset()
762 loaded += c.prepare_cache(progress)
763 previous_progress = current_progress
764
765 # Note: depends cache number is corresponding to the parsing file numbers.
766 # The same file has several caches, still regarded as one item in the cache
767 bb.event.fire(bb.event.CacheLoadCompleted(cachesize, loaded), databuilder.data)
768
769 def __len__(self):
770 return len(self.__caches)
771
772 def __getitem__(self, key):
773 return self.__caches[key]
774
775 def __contains__(self, key):
776 return key in self.__caches
777
778 def __iter__(self):
779 for k in self.__caches:
780 yield k
781
782
783class CacheData(object):
784 """
785 The data structures we compile from the cached data
786 """
787
788 def __init__(self, caches_array):
789 self.caches_array = caches_array
790 for cache_class in self.caches_array:
791 if not issubclass(cache_class, RecipeInfoCommon):
792 bb.error("Extra cache data class %s should subclass RecipeInfoCommon class" % cache_class)
793 cache_class.init_cacheData(self)
794
795 # Direct cache variables
796 self.task_queues = {}
797 self.preferred = {}
798 self.tasks = {}
799 # Indirect Cache variables (set elsewhere)
800 self.ignored_dependencies = []
801 self.world_target = set()
802 self.bbfile_priority = {}
803
804 def add_from_recipeinfo(self, fn, info_array):
805 for info in info_array:
806 info.add_cacheData(self, fn)
807
808class MultiProcessCache(object):
809 """
810 BitBake multi-process cache implementation
811
812 Used by the codeparser & file checksum caches
813 """
814
815 def __init__(self):
816 self.cachefile = None
817 self.cachedata = self.create_cachedata()
818 self.cachedata_extras = self.create_cachedata()
819
820 def init_cache(self, cachedir, cache_file_name=None):
821 if not cachedir:
822 return
823
824 bb.utils.mkdirhier(cachedir)
825 self.cachefile = os.path.join(cachedir,
826 cache_file_name or self.__class__.cache_file_name)
827 logger.debug("Using cache in '%s'", self.cachefile)
828
829 glf = bb.utils.lockfile(self.cachefile + ".lock")
830
831 try:
832 with open(self.cachefile, "rb") as f:
833 p = pickle.Unpickler(f)
834 data, version = p.load()
835 except:
836 bb.utils.unlockfile(glf)
837 return
838
839 bb.utils.unlockfile(glf)
840
841 if version != self.__class__.CACHE_VERSION:
842 return
843
844 self.cachedata = data
845
846 def create_cachedata(self):
847 data = [{}]
848 return data
849
850 def clear_cache(self):
851 if not self.cachefile:
852 bb.fatal("Can't clear invalid cachefile")
853
854 self.cachedata = self.create_cachedata()
855 self.cachedata_extras = self.create_cachedata()
856 with bb.utils.fileslocked([self.cachefile + ".lock"]):
857 bb.utils.remove(self.cachefile)
858 bb.utils.remove(self.cachefile + "-*")
859
860 def save_extras(self):
861 if not self.cachefile:
862 return
863
864 have_data = any(self.cachedata_extras)
865 if not have_data:
866 return
867
868 glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True)
869
870 i = os.getpid()
871 lf = None
872 while not lf:
873 lf = bb.utils.lockfile(self.cachefile + ".lock." + str(i), retry=False)
874 if not lf or os.path.exists(self.cachefile + "-" + str(i)):
875 if lf:
876 bb.utils.unlockfile(lf)
877 lf = None
878 i = i + 1
879 continue
880
881 with open(self.cachefile + "-" + str(i), "wb") as f:
882 p = pickle.Pickler(f, -1)
883 p.dump([self.cachedata_extras, self.__class__.CACHE_VERSION])
884
885 bb.utils.unlockfile(lf)
886 bb.utils.unlockfile(glf)
887
888 def merge_data(self, source, dest):
889 for j in range(0,len(dest)):
890 for h in source[j]:
891 if h not in dest[j]:
892 dest[j][h] = source[j][h]
893
894 def save_merge(self):
895 if not self.cachefile:
896 return
897
898 glf = bb.utils.lockfile(self.cachefile + ".lock")
899
900 data = self.cachedata
901
902 have_data = False
903
904 for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]:
905 f = os.path.join(os.path.dirname(self.cachefile), f)
906 try:
907 with open(f, "rb") as fd:
908 p = pickle.Unpickler(fd)
909 extradata, version = p.load()
910 except (IOError, EOFError):
911 os.unlink(f)
912 continue
913
914 if version != self.__class__.CACHE_VERSION:
915 os.unlink(f)
916 continue
917
918 have_data = True
919 self.merge_data(extradata, data)
920 os.unlink(f)
921
922 if have_data:
923 with open(self.cachefile, "wb") as f:
924 p = pickle.Pickler(f, -1)
925 p.dump([data, self.__class__.CACHE_VERSION])
926
927 bb.utils.unlockfile(glf)
928
929
930class SimpleCache(object):
931 """
932 BitBake multi-process cache implementation
933
934 Used by the codeparser & file checksum caches
935 """
936
937 def __init__(self, version):
938 self.cachefile = None
939 self.cachedata = None
940 self.cacheversion = version
941
942 def init_cache(self, d, cache_file_name=None, defaultdata=None):
943 cachedir = (d.getVar("PERSISTENT_DIR") or
944 d.getVar("CACHE"))
945 if not cachedir:
946 return defaultdata
947
948 bb.utils.mkdirhier(cachedir)
949 self.cachefile = os.path.join(cachedir,
950 cache_file_name or self.__class__.cache_file_name)
951 logger.debug("Using cache in '%s'", self.cachefile)
952
953 glf = bb.utils.lockfile(self.cachefile + ".lock")
954
955 try:
956 with open(self.cachefile, "rb") as f:
957 p = pickle.Unpickler(f)
958 data, version = p.load()
959 except:
960 bb.utils.unlockfile(glf)
961 return defaultdata
962
963 bb.utils.unlockfile(glf)
964
965 if version != self.cacheversion:
966 return defaultdata
967
968 return data
969
970 def save(self, data):
971 if not self.cachefile:
972 return
973
974 glf = bb.utils.lockfile(self.cachefile + ".lock")
975
976 with open(self.cachefile, "wb") as f:
977 p = pickle.Pickler(f, -1)
978 p.dump([data, self.cacheversion])
979
980 bb.utils.unlockfile(glf)
981
982 def copyfile(self, target):
983 if not self.cachefile:
984 return
985
986 glf = bb.utils.lockfile(self.cachefile + ".lock")
987 shutil.copy(self.cachefile, target)
988 bb.utils.unlockfile(glf)
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2026-01-22 00:27:41 +0000