diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-02-17 16:18:25 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-02-18 13:59:11 +0000 |
commit | 2ce6c7b190fc418a5653085f83016b29d52a2ae5 (patch) | |
tree | 78ce3450bcf6e68e2deb829f32acf342d3ede82d /bitbake/lib | |
parent | 4edf4eb9cc86fbd4b51240fae527c1b97a7ace74 (diff) | |
download | poky-2ce6c7b190fc418a5653085f83016b29d52a2ae5.tar.gz |
bitbake: cooker/siggen: Empty siggen cache during parsing
When parsing recipes its apparent the memory usage of bitbake rises linearly
with number of recipes parsed. It shouldn't.
Using tracemalloc (thanks for the tip Joshua Lock) it was clear that the
dependency information left behind in siggen was the culprit. Add a new
method to allow us to drop this information. We don't need it after the recipe
has been parsed and hashes calculated (at runtime its different but only the
currently executing task would be in memory).
This should give signficant memory usage improvements for bitbake and that
in turn should help speed on more constrained systems, as well as when used in
multiconfig environments.
(Bitbake rev: d411b097d810c386d35dc561f8812bb3f35c9a36)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r-- | bitbake/lib/bb/cooker.py | 3 | ||||
-rw-r--r-- | bitbake/lib/bb/siggen.py | 11 |
2 files changed, 13 insertions, 1 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index 77d18a5d0c..cfac91bae5 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -1934,7 +1934,8 @@ class Parser(multiprocessing.Process): | |||
1934 | except IndexError: | 1934 | except IndexError: |
1935 | break | 1935 | break |
1936 | result = self.parse(*job) | 1936 | result = self.parse(*job) |
1937 | 1937 | # Clear the siggen cache after parsing to control memory usage, its huge | |
1938 | bb.parse.siggen.postparsing_clean_cache() | ||
1938 | try: | 1939 | try: |
1939 | self.results.put(result, timeout=0.25) | 1940 | self.results.put(result, timeout=0.25) |
1940 | except queue.Full: | 1941 | except queue.Full: |
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 3e2a85fc8c..9c299d45e3 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py | |||
@@ -48,6 +48,9 @@ class SignatureGenerator(object): | |||
48 | def get_unihash(self, task): | 48 | def get_unihash(self, task): |
49 | return self.taskhash[task] | 49 | return self.taskhash[task] |
50 | 50 | ||
51 | def postparsing_clean_cache(self): | ||
52 | return | ||
53 | |||
51 | def get_taskhash(self, fn, task, deps, dataCache): | 54 | def get_taskhash(self, fn, task, deps, dataCache): |
52 | return "0" | 55 | return "0" |
53 | 56 | ||
@@ -152,6 +155,14 @@ class SignatureGeneratorBasic(SignatureGenerator): | |||
152 | for task in taskdeps: | 155 | for task in taskdeps: |
153 | d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) | 156 | d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) |
154 | 157 | ||
158 | def postparsing_clean_cache(self): | ||
159 | # | ||
160 | # After parsing we can remove some things from memory to reduce our memory footprint | ||
161 | # | ||
162 | self.gendeps = {} | ||
163 | self.lookupcache = {} | ||
164 | self.taskdeps = {} | ||
165 | |||
155 | def rundep_check(self, fn, recipename, task, dep, depname, dataCache): | 166 | def rundep_check(self, fn, recipename, task, dep, depname, dataCache): |
156 | # Return True if we should keep the dependency, False to drop it | 167 | # Return True if we should keep the dependency, False to drop it |
157 | # We only manipulate the dependencies for packages not in the whitelist | 168 | # We only manipulate the dependencies for packages not in the whitelist |