summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2020-02-17 16:18:25 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-02-18 13:59:11 (GMT)
commit2ce6c7b190fc418a5653085f83016b29d52a2ae5 (patch)
tree78ce3450bcf6e68e2deb829f32acf342d3ede82d
parent4edf4eb9cc86fbd4b51240fae527c1b97a7ace74 (diff)
downloadpoky-2ce6c7b190fc418a5653085f83016b29d52a2ae5.tar.gz
bitbake: cooker/siggen: Empty siggen cache during parsing
When parsing recipes its apparent the memory usage of bitbake rises linearly with number of recipes parsed. It shouldn't. Using tracemalloc (thanks for the tip Joshua Lock) it was clear that the dependency information left behind in siggen was the culprit. Add a new method to allow us to drop this information. We don't need it after the recipe has been parsed and hashes calculated (at runtime its different but only the currently executing task would be in memory). This should give signficant memory usage improvements for bitbake and that in turn should help speed on more constrained systems, as well as when used in multiconfig environments. (Bitbake rev: d411b097d810c386d35dc561f8812bb3f35c9a36) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/cooker.py3
-rw-r--r--bitbake/lib/bb/siggen.py11
2 files changed, 13 insertions, 1 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 77d18a5..cfac91b 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1934,7 +1934,8 @@ class Parser(multiprocessing.Process):
1934 except IndexError: 1934 except IndexError:
1935 break 1935 break
1936 result = self.parse(*job) 1936 result = self.parse(*job)
1937 1937 # Clear the siggen cache after parsing to control memory usage, its huge
1938 bb.parse.siggen.postparsing_clean_cache()
1938 try: 1939 try:
1939 self.results.put(result, timeout=0.25) 1940 self.results.put(result, timeout=0.25)
1940 except queue.Full: 1941 except queue.Full:
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 3e2a85f..9c299d4 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -48,6 +48,9 @@ class SignatureGenerator(object):
48 def get_unihash(self, task): 48 def get_unihash(self, task):
49 return self.taskhash[task] 49 return self.taskhash[task]
50 50
51 def postparsing_clean_cache(self):
52 return
53
51 def get_taskhash(self, fn, task, deps, dataCache): 54 def get_taskhash(self, fn, task, deps, dataCache):
52 return "0" 55 return "0"
53 56
@@ -152,6 +155,14 @@ class SignatureGeneratorBasic(SignatureGenerator):
152 for task in taskdeps: 155 for task in taskdeps:
153 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) 156 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
154 157
158 def postparsing_clean_cache(self):
159 #
160 # After parsing we can remove some things from memory to reduce our memory footprint
161 #
162 self.gendeps = {}
163 self.lookupcache = {}
164 self.taskdeps = {}
165
155 def rundep_check(self, fn, recipename, task, dep, depname, dataCache): 166 def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
156 # Return True if we should keep the dependency, False to drop it 167 # Return True if we should keep the dependency, False to drop it
157 # We only manipulate the dependencies for packages not in the whitelist 168 # We only manipulate the dependencies for packages not in the whitelist