summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/cache.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2022-11-17 11:09:53 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2022-12-21 14:15:26 +0000
commit7d010055e2af3294e17db862f42664ca689a9356 (patch)
tree73e522b98f5c6aac0e1782e474af96e5959ea924 /bitbake/lib/bb/cache.py
parent26f255da09e0f704af3d1535d3f8c5cd8b702a73 (diff)
downloadpoky-7d010055e2af3294e17db862f42664ca689a9356.tar.gz
bitbake: cache: Allow compression of the data in SiggenRecipeInfo
The data in SiggenRecipeInfo is large and has a lot of duplication. The size causes a few problems, impacting: - bitbake's overall memory usage - the amount of data sent over IPC between parsing processes and the server - the size of the cache files on disk - the size of "sigdata" hash information files on disk The data consists of strings (some large) or frozenset lists of variables. To reduce the impact we can: a) deplicate the data b) pass references to the object on the second usage (e.g. over IPC or saving into pickle). This patch does this for SiggenRecipeInfo mostly behind the scenes but we do need a couple of reset points so that streamed data is written correctly on the second usage. (Bitbake rev: 9a2b13af483c20763d6559a823310954884f6ab1) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/cache.py')
-rw-r--r--bitbake/lib/bb/cache.py75
1 files changed, 75 insertions, 0 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index 96ab069180..f5b527ba6a 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -263,6 +263,80 @@ class SiggenRecipeInfo(RecipeInfoCommon):
263 cachedata.siggen_varvals[fn] = self.siggen_varvals 263 cachedata.siggen_varvals[fn] = self.siggen_varvals
264 cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps 264 cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps
265 265
266 # The siggen variable data is large and impacts:
267 # - bitbake's overall memory usage
268 # - the amount of data sent over IPC between parsing processes and the server
269 # - the size of the cache files on disk
270 # - the size of "sigdata" hash information files on disk
271 # The data consists of strings (some large) or frozenset lists of variables
272 # As such, we a) deplicate the data here and b) pass references to the object at second
273 # access (e.g. over IPC or saving into pickle).
274
275 store = {}
276 save_map = {}
277 save_count = 1
278 restore_map = {}
279 restore_count = {}
280
281 @classmethod
282 def reset(cls):
283 # Needs to be called before starting new streamed data in a given process
284 # (e.g. writing out the cache again)
285 cls.save_map = {}
286 cls.save_count = 1
287 cls.restore_map = {}
288 cls.restore_count = {}
289
290 @classmethod
291 def _save(cls, deps):
292 ret = []
293 if not deps:
294 return deps
295 for dep in deps:
296 fs = deps[dep]
297 if fs in cls.save_map:
298 ret.append((dep, None, cls.save_map[fs]))
299 else:
300 cls.save_map[fs] = cls.save_count
301 ret.append((dep, fs, None))
302 cls.save_count = cls.save_count + 1
303 return ret
304
305 @classmethod
306 def _restore(cls, deps, pid):
307 ret = {}
308 if not deps:
309 return deps
310 if pid not in cls.restore_map:
311 cls.restore_map[pid] = {}
312 cls.restore_count[pid] = 1
313 map = cls.restore_map[pid]
314 for fs, dep, mapnum in deps:
315 if mapnum:
316 ret[dep] = map[mapnum]
317 else:
318 try:
319 fs = cls.store[fs]
320 except KeyError:
321 cls.store[fs] = fs
322 map[cls.restore_count[pid]] = fs
323 cls.restore_count[pid] = cls.restore_count[pid] + 1
324 ret[dep] = fs
325 return ret
326
327 def __getstate__(self):
328 ret = {}
329 for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
330 ret[key] = self._save(self.__dict__[key])
331 ret['pid'] = os.getpid()
332 return ret
333
334 def __setstate__(self, state):
335 pid = state['pid']
336 for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
337 setattr(self, key, self._restore(state[key], pid))
338
339
266def virtualfn2realfn(virtualfn): 340def virtualfn2realfn(virtualfn):
267 """ 341 """
268 Convert a virtual file name to a real one + the associated subclass keyword 342 Convert a virtual file name to a real one + the associated subclass keyword
@@ -621,6 +695,7 @@ class Cache(object):
621 p.dump(info) 695 p.dump(info)
622 696
623 del self.depends_cache 697 del self.depends_cache
698 SiggenRecipeInfo.reset()
624 699
625 @staticmethod 700 @staticmethod
626 def mtime(cachefile): 701 def mtime(cachefile):