summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <rpurdie@linux.intel.com>2010-08-31 14:19:22 +0100
committerRichard Purdie <rpurdie@linux.intel.com>2010-08-31 14:49:14 +0100
commit9897b81e56eae97dea11b9489a3b691b8fc86441 (patch)
tree78b9e73fd6c8d13b797b54cb3c3d885d9b4ca7eb
parent332c33af188c14dd0051d8a45fe0ad680611db69 (diff)
downloadpoky-9897b81e56eae97dea11b9489a3b691b8fc86441.tar.gz
bitbake/codeparser: Implement persistent cache
For a given input to this code, the output doesn't change to implement a persistent cache of the data to speed up parsing. Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
-rw-r--r--bitbake/lib/bb/codeparser.py66
-rw-r--r--bitbake/lib/bb/cooker.py2
2 files changed, 58 insertions, 10 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 88a26c82a7..faffe390d9 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -4,6 +4,14 @@ from bb import msg, utils
4import ast 4import ast
5import codegen 5import codegen
6 6
7PARSERCACHE_VERSION = 1
8
9try:
10 import cPickle as pickle
11except ImportError:
12 import pickle
13 bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.")
14
7def check_indent(codestr): 15def check_indent(codestr):
8 """If the code is indented, add a top level piece of code to 'remove' the indentation""" 16 """If the code is indented, add a top level piece of code to 'remove' the indentation"""
9 17
@@ -13,6 +21,43 @@ def check_indent(codestr):
13 return codestr 21 return codestr
14 22
15pythonparsecache = {} 23pythonparsecache = {}
24shellparsecache = {}
25
26def parser_cachefile(d):
27 cachedir = bb.data.getVar("PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True)
28 if cachedir in [None, '']:
29 return None
30 bb.utils.mkdirhier(cachedir)
31 cachefile = os.path.join(cachedir, "bb_codeparser.dat")
32 bb.msg.debug(1, bb.msg.domain.Cache, "Using cache in '%s' for codeparser cache" % cachefile)
33 return cachefile
34
35def parser_cache_init(d):
36
37 cachefile = parser_cachefile(d)
38 if not cachefile:
39 return
40
41 try:
42 p = pickle.Unpickler(file(cachefile, "rb"))
43 data, version = p.load()
44 except:
45 return
46
47 if version != PARSERCACHE_VERSION:
48 return
49
50 bb.codeparser.pythonparsecache = data[0]
51 bb.codeparser.shellparsecache = data[1]
52
53def parser_cache_save(d):
54
55 cachefile = parser_cachefile(d)
56 if not cachefile:
57 return
58
59 p = pickle.Pickler(file(cachefile, "wb"), -1)
60 p.dump([[bb.codeparser.pythonparsecache, bb.codeparser.shellparsecache], PARSERCACHE_VERSION])
16 61
17class PythonParser(): 62class PythonParser():
18 class ValueVisitor(): 63 class ValueVisitor():
@@ -125,9 +170,11 @@ class PythonParser():
125 170
126 def parse_python(self, node): 171 def parse_python(self, node):
127 172
128 if node in pythonparsecache: 173 h = hash(node)
129 self.references = pythonparsecache[node].references 174
130 self.execs = pythonparsecache[node].execs 175 if h in pythonparsecache:
176 self.references = pythonparsecache[h].references
177 self.execs = pythonparsecache[h].execs
131 return 178 return
132 179
133 code = compile(check_indent(str(node)), "<string>", "exec", 180 code = compile(check_indent(str(node)), "<string>", "exec",
@@ -142,10 +189,7 @@ class PythonParser():
142 self.references.update(visitor.var_execs) 189 self.references.update(visitor.var_execs)
143 self.execs = visitor.direct_func_calls 190 self.execs = visitor.direct_func_calls
144 191
145 pythonparsecache[node] = self 192 pythonparsecache[h] = self
146
147
148shellparsecache = {}
149 193
150class ShellParser(): 194class ShellParser():
151 def __init__(self): 195 def __init__(self):
@@ -158,8 +202,10 @@ class ShellParser():
158 commands it executes. 202 commands it executes.
159 """ 203 """
160 204
161 if value in pythonparsecache: 205 h = hash(value)
162 self.execs = shellparsecache[value].execs 206
207 if h in shellparsecache:
208 self.execs = shellparsecache[h].execs
163 return 209 return
164 210
165 try: 211 try:
@@ -171,7 +217,7 @@ class ShellParser():
171 self.process_tokens(token) 217 self.process_tokens(token)
172 self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs) 218 self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
173 219
174 shellparsecache[value] = self 220 shellparsecache[h] = self
175 221
176 return self.execs 222 return self.execs
177 223
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index dcdcb7d443..ea33693ddb 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -546,6 +546,7 @@ class BBCooker:
546 546
547 if bb.data.getVar("BB_WORKERCONTEXT", self.configuration.data) is None: 547 if bb.data.getVar("BB_WORKERCONTEXT", self.configuration.data) is None:
548 bb.fetch.fetcher_init(self.configuration.data) 548 bb.fetch.fetcher_init(self.configuration.data)
549 bb.codeparser.parser_cache_init(self.configuration.data)
549 550
550 bb.event.fire(bb.event.ConfigParsed(), self.configuration.data) 551 bb.event.fire(bb.event.ConfigParsed(), self.configuration.data)
551 552
@@ -1011,6 +1012,7 @@ class CookerParser:
1011 1012
1012 if self.pointer >= self.total: 1013 if self.pointer >= self.total:
1013 cooker.bb_cache.sync() 1014 cooker.bb_cache.sync()
1015 bb.codeparser.parser_cache_save(cooker.configuration.data)
1014 if self.error > 0: 1016 if self.error > 0:
1015 raise ParsingErrorsFound 1017 raise ParsingErrorsFound
1016 return False 1018 return False