diff options
Diffstat (limited to 'bitbake/lib/bb/codeparser.py')
-rw-r--r-- | bitbake/lib/bb/codeparser.py | 110 |
1 files changed, 84 insertions, 26 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py index 25a7ac69d3..2e8b7ced3c 100644 --- a/bitbake/lib/bb/codeparser.py +++ b/bitbake/lib/bb/codeparser.py | |||
@@ -1,4 +1,6 @@ | |||
1 | # | 1 | # |
2 | # Copyright BitBake Contributors | ||
3 | # | ||
2 | # SPDX-License-Identifier: GPL-2.0-only | 4 | # SPDX-License-Identifier: GPL-2.0-only |
3 | # | 5 | # |
4 | 6 | ||
@@ -25,6 +27,7 @@ import ast | |||
25 | import sys | 27 | import sys |
26 | import codegen | 28 | import codegen |
27 | import logging | 29 | import logging |
30 | import inspect | ||
28 | import bb.pysh as pysh | 31 | import bb.pysh as pysh |
29 | import bb.utils, bb.data | 32 | import bb.utils, bb.data |
30 | import hashlib | 33 | import hashlib |
@@ -56,10 +59,40 @@ def check_indent(codestr): | |||
56 | 59 | ||
57 | return codestr | 60 | return codestr |
58 | 61 | ||
59 | # A custom getstate/setstate using tuples is actually worth 15% cachesize by | 62 | modulecode_deps = {} |
60 | # avoiding duplication of the attribute names! | ||
61 | 63 | ||
64 | def add_module_functions(fn, functions, namespace): | ||
65 | import os | ||
66 | fstat = os.stat(fn) | ||
67 | fixedhash = fn + ":" + str(fstat.st_size) + ":" + str(fstat.st_mtime) | ||
68 | for f in functions: | ||
69 | name = "%s.%s" % (namespace, f) | ||
70 | parser = PythonParser(name, logger) | ||
71 | try: | ||
72 | parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f) | ||
73 | #bb.warn("Cached %s" % f) | ||
74 | except KeyError: | ||
75 | lines, lineno = inspect.getsourcelines(functions[f]) | ||
76 | src = "".join(lines) | ||
77 | parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f) | ||
78 | #bb.warn("Not cached %s" % f) | ||
79 | execs = parser.execs.copy() | ||
80 | # Expand internal module exec references | ||
81 | for e in parser.execs: | ||
82 | if e in functions: | ||
83 | execs.remove(e) | ||
84 | execs.add(namespace + "." + e) | ||
85 | modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy()] | ||
86 | #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, fn, parser.references, parser.execs, parser.var_execs, parser.contains)) | ||
87 | |||
88 | def update_module_dependencies(d): | ||
89 | for mod in modulecode_deps: | ||
90 | excludes = set((d.getVarFlag(mod, "vardepsexclude") or "").split()) | ||
91 | if excludes: | ||
92 | modulecode_deps[mod] = [modulecode_deps[mod][0] - excludes, modulecode_deps[mod][1] - excludes, modulecode_deps[mod][2] - excludes, modulecode_deps[mod][3]] | ||
62 | 93 | ||
94 | # A custom getstate/setstate using tuples is actually worth 15% cachesize by | ||
95 | # avoiding duplication of the attribute names! | ||
63 | class SetCache(object): | 96 | class SetCache(object): |
64 | def __init__(self): | 97 | def __init__(self): |
65 | self.setcache = {} | 98 | self.setcache = {} |
@@ -152,12 +185,12 @@ class CodeParserCache(MultiProcessCache): | |||
152 | self.shellcachelines[h] = cacheline | 185 | self.shellcachelines[h] = cacheline |
153 | return cacheline | 186 | return cacheline |
154 | 187 | ||
155 | def init_cache(self, d): | 188 | def init_cache(self, cachedir): |
156 | # Check if we already have the caches | 189 | # Check if we already have the caches |
157 | if self.pythoncache: | 190 | if self.pythoncache: |
158 | return | 191 | return |
159 | 192 | ||
160 | MultiProcessCache.init_cache(self, d) | 193 | MultiProcessCache.init_cache(self, cachedir) |
161 | 194 | ||
162 | # cachedata gets re-assigned in the parent | 195 | # cachedata gets re-assigned in the parent |
163 | self.pythoncache = self.cachedata[0] | 196 | self.pythoncache = self.cachedata[0] |
@@ -169,8 +202,8 @@ class CodeParserCache(MultiProcessCache): | |||
169 | 202 | ||
170 | codeparsercache = CodeParserCache() | 203 | codeparsercache = CodeParserCache() |
171 | 204 | ||
172 | def parser_cache_init(d): | 205 | def parser_cache_init(cachedir): |
173 | codeparsercache.init_cache(d) | 206 | codeparsercache.init_cache(cachedir) |
174 | 207 | ||
175 | def parser_cache_save(): | 208 | def parser_cache_save(): |
176 | codeparsercache.save_extras() | 209 | codeparsercache.save_extras() |
@@ -195,6 +228,10 @@ class BufferedLogger(Logger): | |||
195 | self.target.handle(record) | 228 | self.target.handle(record) |
196 | self.buffer = [] | 229 | self.buffer = [] |
197 | 230 | ||
231 | class DummyLogger(): | ||
232 | def flush(self): | ||
233 | return | ||
234 | |||
198 | class PythonParser(): | 235 | class PythonParser(): |
199 | getvars = (".getVar", ".appendVar", ".prependVar", "oe.utils.conditional") | 236 | getvars = (".getVar", ".appendVar", ".prependVar", "oe.utils.conditional") |
200 | getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag") | 237 | getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag") |
@@ -212,26 +249,26 @@ class PythonParser(): | |||
212 | funcstr = codegen.to_source(func) | 249 | funcstr = codegen.to_source(func) |
213 | argstr = codegen.to_source(arg) | 250 | argstr = codegen.to_source(arg) |
214 | except TypeError: | 251 | except TypeError: |
215 | self.log.debug(2, 'Failed to convert function and argument to source form') | 252 | self.log.debug2('Failed to convert function and argument to source form') |
216 | else: | 253 | else: |
217 | self.log.debug(1, self.unhandled_message % (funcstr, argstr)) | 254 | self.log.debug(self.unhandled_message % (funcstr, argstr)) |
218 | 255 | ||
219 | def visit_Call(self, node): | 256 | def visit_Call(self, node): |
220 | name = self.called_node_name(node.func) | 257 | name = self.called_node_name(node.func) |
221 | if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs): | 258 | if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs): |
222 | if isinstance(node.args[0], ast.Str): | 259 | if isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str): |
223 | varname = node.args[0].s | 260 | varname = node.args[0].value |
224 | if name in self.containsfuncs and isinstance(node.args[1], ast.Str): | 261 | if name in self.containsfuncs and isinstance(node.args[1], ast.Constant): |
225 | if varname not in self.contains: | 262 | if varname not in self.contains: |
226 | self.contains[varname] = set() | 263 | self.contains[varname] = set() |
227 | self.contains[varname].add(node.args[1].s) | 264 | self.contains[varname].add(node.args[1].value) |
228 | elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Str): | 265 | elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Constant): |
229 | if varname not in self.contains: | 266 | if varname not in self.contains: |
230 | self.contains[varname] = set() | 267 | self.contains[varname] = set() |
231 | self.contains[varname].update(node.args[1].s.split()) | 268 | self.contains[varname].update(node.args[1].value.split()) |
232 | elif name.endswith(self.getvarflags): | 269 | elif name.endswith(self.getvarflags): |
233 | if isinstance(node.args[1], ast.Str): | 270 | if isinstance(node.args[1], ast.Constant): |
234 | self.references.add('%s[%s]' % (varname, node.args[1].s)) | 271 | self.references.add('%s[%s]' % (varname, node.args[1].value)) |
235 | else: | 272 | else: |
236 | self.warn(node.func, node.args[1]) | 273 | self.warn(node.func, node.args[1]) |
237 | else: | 274 | else: |
@@ -239,8 +276,8 @@ class PythonParser(): | |||
239 | else: | 276 | else: |
240 | self.warn(node.func, node.args[0]) | 277 | self.warn(node.func, node.args[0]) |
241 | elif name and name.endswith(".expand"): | 278 | elif name and name.endswith(".expand"): |
242 | if isinstance(node.args[0], ast.Str): | 279 | if isinstance(node.args[0], ast.Constant): |
243 | value = node.args[0].s | 280 | value = node.args[0].value |
244 | d = bb.data.init() | 281 | d = bb.data.init() |
245 | parser = d.expandWithRefs(value, self.name) | 282 | parser = d.expandWithRefs(value, self.name) |
246 | self.references |= parser.references | 283 | self.references |= parser.references |
@@ -250,8 +287,8 @@ class PythonParser(): | |||
250 | self.contains[varname] = set() | 287 | self.contains[varname] = set() |
251 | self.contains[varname] |= parser.contains[varname] | 288 | self.contains[varname] |= parser.contains[varname] |
252 | elif name in self.execfuncs: | 289 | elif name in self.execfuncs: |
253 | if isinstance(node.args[0], ast.Str): | 290 | if isinstance(node.args[0], ast.Constant): |
254 | self.var_execs.add(node.args[0].s) | 291 | self.var_execs.add(node.args[0].value) |
255 | else: | 292 | else: |
256 | self.warn(node.func, node.args[0]) | 293 | self.warn(node.func, node.args[0]) |
257 | elif name and isinstance(node.func, (ast.Name, ast.Attribute)): | 294 | elif name and isinstance(node.func, (ast.Name, ast.Attribute)): |
@@ -276,16 +313,24 @@ class PythonParser(): | |||
276 | self.contains = {} | 313 | self.contains = {} |
277 | self.execs = set() | 314 | self.execs = set() |
278 | self.references = set() | 315 | self.references = set() |
279 | self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, log) | 316 | self._log = log |
317 | # Defer init as expensive | ||
318 | self.log = DummyLogger() | ||
280 | 319 | ||
281 | self.unhandled_message = "in call of %s, argument '%s' is not a string literal" | 320 | self.unhandled_message = "in call of %s, argument '%s' is not a string literal" |
282 | self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) | 321 | self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) |
283 | 322 | ||
284 | def parse_python(self, node, lineno=0, filename="<string>"): | 323 | # For the python module code it is expensive to have the function text so it is |
285 | if not node or not node.strip(): | 324 | # uses a different fixedhash to cache against. We can take the hit on obtaining the |
325 | # text if it isn't in the cache. | ||
326 | def parse_python(self, node, lineno=0, filename="<string>", fixedhash=None): | ||
327 | if not fixedhash and (not node or not node.strip()): | ||
286 | return | 328 | return |
287 | 329 | ||
288 | h = bbhash(str(node)) | 330 | if fixedhash: |
331 | h = fixedhash | ||
332 | else: | ||
333 | h = bbhash(str(node)) | ||
289 | 334 | ||
290 | if h in codeparsercache.pythoncache: | 335 | if h in codeparsercache.pythoncache: |
291 | self.references = set(codeparsercache.pythoncache[h].refs) | 336 | self.references = set(codeparsercache.pythoncache[h].refs) |
@@ -303,6 +348,12 @@ class PythonParser(): | |||
303 | self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) | 348 | self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) |
304 | return | 349 | return |
305 | 350 | ||
351 | if fixedhash and not node: | ||
352 | raise KeyError | ||
353 | |||
354 | # Need to parse so take the hit on the real log buffer | ||
355 | self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log) | ||
356 | |||
306 | # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though | 357 | # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though |
307 | node = "\n" * int(lineno) + node | 358 | node = "\n" * int(lineno) + node |
308 | code = compile(check_indent(str(node)), filename, "exec", | 359 | code = compile(check_indent(str(node)), filename, "exec", |
@@ -321,7 +372,11 @@ class ShellParser(): | |||
321 | self.funcdefs = set() | 372 | self.funcdefs = set() |
322 | self.allexecs = set() | 373 | self.allexecs = set() |
323 | self.execs = set() | 374 | self.execs = set() |
324 | self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log) | 375 | self._name = name |
376 | self._log = log | ||
377 | # Defer init as expensive | ||
378 | self.log = DummyLogger() | ||
379 | |||
325 | self.unhandled_template = "unable to handle non-literal command '%s'" | 380 | self.unhandled_template = "unable to handle non-literal command '%s'" |
326 | self.unhandled_template = "while parsing %s, %s" % (name, self.unhandled_template) | 381 | self.unhandled_template = "while parsing %s, %s" % (name, self.unhandled_template) |
327 | 382 | ||
@@ -340,6 +395,9 @@ class ShellParser(): | |||
340 | self.execs = set(codeparsercache.shellcacheextras[h].execs) | 395 | self.execs = set(codeparsercache.shellcacheextras[h].execs) |
341 | return self.execs | 396 | return self.execs |
342 | 397 | ||
398 | # Need to parse so take the hit on the real log buffer | ||
399 | self.log = BufferedLogger('BitBake.Data.%s' % self._name, logging.DEBUG, self._log) | ||
400 | |||
343 | self._parse_shell(value) | 401 | self._parse_shell(value) |
344 | self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs) | 402 | self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs) |
345 | 403 | ||
@@ -450,7 +508,7 @@ class ShellParser(): | |||
450 | 508 | ||
451 | cmd = word[1] | 509 | cmd = word[1] |
452 | if cmd.startswith("$"): | 510 | if cmd.startswith("$"): |
453 | self.log.debug(1, self.unhandled_template % cmd) | 511 | self.log.debug(self.unhandled_template % cmd) |
454 | elif cmd == "eval": | 512 | elif cmd == "eval": |
455 | command = " ".join(word for _, word in words[1:]) | 513 | command = " ".join(word for _, word in words[1:]) |
456 | self._parse_shell(command) | 514 | self._parse_shell(command) |