summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2012-03-11 14:18:02 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-03-12 15:52:33 +0000
commit7bf0a790b23833a1b426d2349885459112fb5d7c (patch)
tree46be04031f415f0ed982e266f368cfd0c566f97d
parentc003a308607d8d96b22cff72f77812b6c324302d (diff)
downloadpoky-7bf0a790b23833a1b426d2349885459112fb5d7c.tar.gz
codeparser: Compute extra cache components on the fly rather than later
In the initial implementation there was a relcutance on my part to generate incremental cache components on the fly since it would lead to some duplicate code. We are now seeing problems where each thread reading in the saved cache file causes significant overhead and can make the process appear to hang on a many core build, particularly when the cache file is large. This patch changes the code to maintain the delta in a separate dict right from the start. The code duplication isn't too bad and could be mitigated in other ways if it becomes an issue. [YOCTO #2039 partial] (Bitbake rev: cdd5d0dee6ab12326b252b6b505a316a52638cac) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/codeparser.py41
1 files changed, 20 insertions, 21 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 2590e5cae7..04a34f944a 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -34,6 +34,9 @@ def check_indent(codestr):
34 34
35pythonparsecache = {} 35pythonparsecache = {}
36shellparsecache = {} 36shellparsecache = {}
37pythonparsecacheextras = {}
38shellparsecacheextras = {}
39
37 40
38def parser_cachefile(d): 41def parser_cachefile(d):
39 cachedir = (d.getVar("PERSISTENT_DIR", True) or 42 cachedir = (d.getVar("PERSISTENT_DIR", True) or
@@ -86,22 +89,8 @@ def parser_cache_save(d):
86 i = i + 1 89 i = i + 1
87 continue 90 continue
88 91
89 try: 92 shellcache = shellparsecacheextras
90 p = pickle.Unpickler(file(cachefile, "rb")) 93 pythoncache = pythonparsecacheextras
91 data, version = p.load()
92 except (IOError, EOFError, ValueError):
93 data, version = None, None
94
95 if version != PARSERCACHE_VERSION:
96 shellcache = shellparsecache
97 pythoncache = pythonparsecache
98 else:
99 for h in pythonparsecache:
100 if h not in data[0]:
101 pythoncache[h] = pythonparsecache[h]
102 for h in shellparsecache:
103 if h not in data[1]:
104 shellcache[h] = shellparsecache[h]
105 94
106 p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1) 95 p = pickle.Pickler(file(cachefile + "-" + str(i), "wb"), -1)
107 p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION]) 96 p.dump([[pythoncache, shellcache], PARSERCACHE_VERSION])
@@ -230,6 +219,12 @@ class PythonParser():
230 self.execs = pythonparsecache[h]["execs"] 219 self.execs = pythonparsecache[h]["execs"]
231 return 220 return
232 221
222 if h in pythonparsecacheextras:
223 self.references = pythonparsecacheextras[h]["refs"]
224 self.execs = pythonparsecacheextras[h]["execs"]
225 return
226
227
233 code = compile(check_indent(str(node)), "<string>", "exec", 228 code = compile(check_indent(str(node)), "<string>", "exec",
234 ast.PyCF_ONLY_AST) 229 ast.PyCF_ONLY_AST)
235 230
@@ -240,9 +235,9 @@ class PythonParser():
240 self.references.update(self.var_references) 235 self.references.update(self.var_references)
241 self.references.update(self.var_execs) 236 self.references.update(self.var_execs)
242 237
243 pythonparsecache[h] = {} 238 pythonparsecacheextras[h] = {}
244 pythonparsecache[h]["refs"] = self.references 239 pythonparsecacheextras[h]["refs"] = self.references
245 pythonparsecache[h]["execs"] = self.execs 240 pythonparsecacheextras[h]["execs"] = self.execs
246 241
247class ShellParser(): 242class ShellParser():
248 def __init__(self, name, log): 243 def __init__(self, name, log):
@@ -264,6 +259,10 @@ class ShellParser():
264 self.execs = shellparsecache[h]["execs"] 259 self.execs = shellparsecache[h]["execs"]
265 return self.execs 260 return self.execs
266 261
262 if h in shellparsecacheextras:
263 self.execs = shellparsecacheextras[h]["execs"]
264 return self.execs
265
267 try: 266 try:
268 tokens, _ = pyshyacc.parse(value, eof=True, debug=False) 267 tokens, _ = pyshyacc.parse(value, eof=True, debug=False)
269 except pyshlex.NeedMore: 268 except pyshlex.NeedMore:
@@ -273,8 +272,8 @@ class ShellParser():
273 self.process_tokens(token) 272 self.process_tokens(token)
274 self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs) 273 self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
275 274
276 shellparsecache[h] = {} 275 shellparsecacheextras[h] = {}
277 shellparsecache[h]["execs"] = self.execs 276 shellparsecacheextras[h]["execs"] = self.execs
278 277
279 return self.execs 278 return self.execs
280 279