summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2020-02-17 16:18:25 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-02-17 23:11:38 +0000
commit925800570f9093cc8de90091008759998e5cd4f2 (patch)
treedbbb7e3f3ed7db8c346f7377ebf71ab0710c98af
parentd276df78f24a00a919472b6fadb7b78e3b3b36f4 (diff)
downloadpoky-925800570f9093cc8de90091008759998e5cd4f2.tar.gz
bitbake: cooker/siggen: Empty siggen cache during parsing
When parsing recipes its apparent the memory usage of bitbake rises linearly with number of recipes parsed. It shouldn't. Using tracemalloc (thanks for the tip Joshua Lock) it was clear that the dependency information left behind in siggen was the culprit. Add a new method to allow us to drop this information. We don't need it after the recipe has been parsed and hashes calculated (at runtime its different but only the currently executing task would be in memory). This should give signficant memory usage improvements for bitbake and that in turn should help speed on more constrained systems, as well as when used in multiconfig environments. (Bitbake rev: 5d98d8e39bba42f458532b1eef3619f2321d8a2b) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cooker.py3
-rw-r--r--bitbake/lib/bb/siggen.py11
2 files changed, 13 insertions, 1 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 1124332186..a05630d52d 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1940,7 +1940,8 @@ class Parser(multiprocessing.Process):
1940 except IndexError: 1940 except IndexError:
1941 break 1941 break
1942 result = self.parse(*job) 1942 result = self.parse(*job)
1943 1943 # Clear the siggen cache after parsing to control memory usage, its huge
1944 bb.parse.siggen.postparsing_clean_cache()
1944 try: 1945 try:
1945 self.results.put(result, timeout=0.25) 1946 self.results.put(result, timeout=0.25)
1946 except queue.Full: 1947 except queue.Full:
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index ffd8fcaf36..c2d0c736cf 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -51,6 +51,9 @@ class SignatureGenerator(object):
51 def finalise(self, fn, d, varient): 51 def finalise(self, fn, d, varient):
52 return 52 return
53 53
54 def postparsing_clean_cache(self):
55 return
56
54 def get_unihash(self, tid): 57 def get_unihash(self, tid):
55 return self.taskhash[tid] 58 return self.taskhash[tid]
56 59
@@ -188,6 +191,14 @@ class SignatureGeneratorBasic(SignatureGenerator):
188 for task in taskdeps: 191 for task in taskdeps:
189 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task]) 192 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task])
190 193
194 def postparsing_clean_cache(self):
195 #
196 # After parsing we can remove some things from memory to reduce our memory footprint
197 #
198 self.gendeps = {}
199 self.lookupcache = {}
200 self.taskdeps = {}
201
191 def rundep_check(self, fn, recipename, task, dep, depname, dataCache): 202 def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
192 # Return True if we should keep the dependency, False to drop it 203 # Return True if we should keep the dependency, False to drop it
193 # We only manipulate the dependencies for packages not in the whitelist 204 # We only manipulate the dependencies for packages not in the whitelist