diff options
Diffstat (limited to 'bitbake/lib')
-rw-r--r-- | bitbake/lib/bb/codeparser.py | 43 | ||||
-rw-r--r-- | bitbake/lib/bb/data.py | 15 | ||||
-rw-r--r-- | bitbake/lib/bb/parse/ast.py | 12 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/codeparser.py | 14 |
4 files changed, 68 insertions, 16 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py index 9d66d3ae41..c0b1d7966f 100644 --- a/bitbake/lib/bb/codeparser.py +++ b/bitbake/lib/bb/codeparser.py | |||
@@ -27,6 +27,7 @@ import ast | |||
27 | import sys | 27 | import sys |
28 | import codegen | 28 | import codegen |
29 | import logging | 29 | import logging |
30 | import inspect | ||
30 | import bb.pysh as pysh | 31 | import bb.pysh as pysh |
31 | import bb.utils, bb.data | 32 | import bb.utils, bb.data |
32 | import hashlib | 33 | import hashlib |
@@ -58,10 +59,33 @@ def check_indent(codestr): | |||
58 | 59 | ||
59 | return codestr | 60 | return codestr |
60 | 61 | ||
61 | # A custom getstate/setstate using tuples is actually worth 15% cachesize by | 62 | modulecode_deps = {} |
62 | # avoiding duplication of the attribute names! | ||
63 | 63 | ||
64 | def add_module_functions(fn, functions, namespace): | ||
65 | fstat = os.stat(fn) | ||
66 | fixedhash = fn + ":" + str(fstat.st_size) + ":" + str(fstat.st_mtime) | ||
67 | for f in functions: | ||
68 | name = "%s.%s" % (namespace, f) | ||
69 | parser = PythonParser(name, logger) | ||
70 | try: | ||
71 | parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f) | ||
72 | #bb.warn("Cached %s" % f) | ||
73 | except KeyError: | ||
74 | lines, lineno = inspect.getsourcelines(functions[f]) | ||
75 | src = "".join(lines) | ||
76 | parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f) | ||
77 | #bb.warn("Not cached %s" % f) | ||
78 | execs = parser.execs.copy() | ||
79 | # Expand internal module exec references | ||
80 | for e in parser.execs: | ||
81 | if e in functions: | ||
82 | execs.remove(e) | ||
83 | execs.add(namespace + "." + e) | ||
84 | modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy()] | ||
85 | #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, src, parser.references, parser.execs, parser.var_execs, parser.contains)) | ||
64 | 86 | ||
87 | # A custom getstate/setstate using tuples is actually worth 15% cachesize by | ||
88 | # avoiding duplication of the attribute names! | ||
65 | class SetCache(object): | 89 | class SetCache(object): |
66 | def __init__(self): | 90 | def __init__(self): |
67 | self.setcache = {} | 91 | self.setcache = {} |
@@ -289,11 +313,17 @@ class PythonParser(): | |||
289 | self.unhandled_message = "in call of %s, argument '%s' is not a string literal" | 313 | self.unhandled_message = "in call of %s, argument '%s' is not a string literal" |
290 | self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) | 314 | self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) |
291 | 315 | ||
292 | def parse_python(self, node, lineno=0, filename="<string>"): | 316 | # For the python module code it is expensive to have the function text so it is |
293 | if not node or not node.strip(): | 317 | # uses a different fixedhash to cache against. We can take the hit on obtaining the |
318 | # text if it isn't in the cache. | ||
319 | def parse_python(self, node, lineno=0, filename="<string>", fixedhash=None): | ||
320 | if not fixedhash and (not node or not node.strip()): | ||
294 | return | 321 | return |
295 | 322 | ||
296 | h = bbhash(str(node)) | 323 | if fixedhash: |
324 | h = fixedhash | ||
325 | else: | ||
326 | h = bbhash(str(node)) | ||
297 | 327 | ||
298 | if h in codeparsercache.pythoncache: | 328 | if h in codeparsercache.pythoncache: |
299 | self.references = set(codeparsercache.pythoncache[h].refs) | 329 | self.references = set(codeparsercache.pythoncache[h].refs) |
@@ -311,6 +341,9 @@ class PythonParser(): | |||
311 | self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) | 341 | self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) |
312 | return | 342 | return |
313 | 343 | ||
344 | if fixedhash and not node: | ||
345 | raise KeyError | ||
346 | |||
314 | # Need to parse so take the hit on the real log buffer | 347 | # Need to parse so take the hit on the real log buffer |
315 | self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log) | 348 | self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log) |
316 | 349 | ||
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py index bfaa0410ea..841369699e 100644 --- a/bitbake/lib/bb/data.py +++ b/bitbake/lib/bb/data.py | |||
@@ -261,7 +261,7 @@ def emit_func_python(func, o=sys.__stdout__, d = init()): | |||
261 | newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split()) | 261 | newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split()) |
262 | newdeps -= seen | 262 | newdeps -= seen |
263 | 263 | ||
264 | def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d): | 264 | def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d): |
265 | def handle_contains(value, contains, exclusions, d): | 265 | def handle_contains(value, contains, exclusions, d): |
266 | newvalue = [] | 266 | newvalue = [] |
267 | if value: | 267 | if value: |
@@ -289,6 +289,12 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d): | |||
289 | 289 | ||
290 | deps = set() | 290 | deps = set() |
291 | try: | 291 | try: |
292 | if key in mod_funcs: | ||
293 | exclusions = set() | ||
294 | moddep = bb.codeparser.modulecode_deps[key] | ||
295 | value = handle_contains("", moddep[3], exclusions, d) | ||
296 | return frozenset((moddep[0] | keys & moddep[1]) - ignored_vars), value | ||
297 | |||
292 | if key[-1] == ']': | 298 | if key[-1] == ']': |
293 | vf = key[:-1].split('[') | 299 | vf = key[:-1].split('[') |
294 | if vf[1] == "vardepvalueexclude": | 300 | if vf[1] == "vardepvalueexclude": |
@@ -367,7 +373,8 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d): | |||
367 | 373 | ||
368 | def generate_dependencies(d, ignored_vars): | 374 | def generate_dependencies(d, ignored_vars): |
369 | 375 | ||
370 | keys = set(key for key in d if not key.startswith("__")) | 376 | mod_funcs = set(bb.codeparser.modulecode_deps.keys()) |
377 | keys = set(key for key in d if not key.startswith("__")) | mod_funcs | ||
371 | shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) | 378 | shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) |
372 | varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS') | 379 | varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS') |
373 | 380 | ||
@@ -376,7 +383,7 @@ def generate_dependencies(d, ignored_vars): | |||
376 | 383 | ||
377 | tasklist = d.getVar('__BBTASKS', False) or [] | 384 | tasklist = d.getVar('__BBTASKS', False) or [] |
378 | for task in tasklist: | 385 | for task in tasklist: |
379 | deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, ignored_vars, d) | 386 | deps[task], values[task] = build_dependencies(task, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d) |
380 | newdeps = deps[task] | 387 | newdeps = deps[task] |
381 | seen = set() | 388 | seen = set() |
382 | while newdeps: | 389 | while newdeps: |
@@ -385,7 +392,7 @@ def generate_dependencies(d, ignored_vars): | |||
385 | newdeps = set() | 392 | newdeps = set() |
386 | for dep in nextdeps: | 393 | for dep in nextdeps: |
387 | if dep not in deps: | 394 | if dep not in deps: |
388 | deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, ignored_vars, d) | 395 | deps[dep], values[dep] = build_dependencies(dep, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d) |
389 | newdeps |= deps[dep] | 396 | newdeps |= deps[dep] |
390 | newdeps -= seen | 397 | newdeps -= seen |
391 | #print "For %s: %s" % (task, str(deps[task])) | 398 | #print "For %s: %s" % (task, str(deps[task])) |
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py index 862087c77d..375ba3cb79 100644 --- a/bitbake/lib/bb/parse/ast.py +++ b/bitbake/lib/bb/parse/ast.py | |||
@@ -290,6 +290,18 @@ class PyLibNode(AstNode): | |||
290 | toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", []) | 290 | toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", []) |
291 | for i in toimport: | 291 | for i in toimport: |
292 | bb.utils._context[self.namespace] = __import__(self.namespace + "." + i) | 292 | bb.utils._context[self.namespace] = __import__(self.namespace + "." + i) |
293 | mod = getattr(bb.utils._context[self.namespace], i) | ||
294 | fn = getattr(mod, "__file__") | ||
295 | funcs = {} | ||
296 | for f in dir(mod): | ||
297 | if f.startswith("_"): | ||
298 | continue | ||
299 | fcall = getattr(mod, f) | ||
300 | if not callable(fcall): | ||
301 | continue | ||
302 | funcs[f] = fcall | ||
303 | bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i)) | ||
304 | |||
293 | except AttributeError as e: | 305 | except AttributeError as e: |
294 | bb.error("Error importing OE modules: %s" % str(e)) | 306 | bb.error("Error importing OE modules: %s" % str(e)) |
295 | 307 | ||
diff --git a/bitbake/lib/bb/tests/codeparser.py b/bitbake/lib/bb/tests/codeparser.py index 71ed382ab8..a508f23bcb 100644 --- a/bitbake/lib/bb/tests/codeparser.py +++ b/bitbake/lib/bb/tests/codeparser.py | |||
@@ -318,7 +318,7 @@ d.getVar(a(), False) | |||
318 | "filename": "example.bb", | 318 | "filename": "example.bb", |
319 | }) | 319 | }) |
320 | 320 | ||
321 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) | 321 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d) |
322 | 322 | ||
323 | self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) | 323 | self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) |
324 | 324 | ||
@@ -365,7 +365,7 @@ esac | |||
365 | self.d.setVarFlags("FOO", {"func": True}) | 365 | self.d.setVarFlags("FOO", {"func": True}) |
366 | self.setEmptyVars(execs) | 366 | self.setEmptyVars(execs) |
367 | 367 | ||
368 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) | 368 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d) |
369 | 369 | ||
370 | self.assertEqual(deps, set(["somevar", "inverted"] + execs)) | 370 | self.assertEqual(deps, set(["somevar", "inverted"] + execs)) |
371 | 371 | ||
@@ -375,7 +375,7 @@ esac | |||
375 | self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") | 375 | self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") |
376 | self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") | 376 | self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") |
377 | 377 | ||
378 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) | 378 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d) |
379 | 379 | ||
380 | self.assertEqual(deps, set(["oe_libinstall"])) | 380 | self.assertEqual(deps, set(["oe_libinstall"])) |
381 | 381 | ||
@@ -384,7 +384,7 @@ esac | |||
384 | self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") | 384 | self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") |
385 | self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") | 385 | self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") |
386 | 386 | ||
387 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) | 387 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d) |
388 | 388 | ||
389 | self.assertEqual(deps, set(["oe_libinstall"])) | 389 | self.assertEqual(deps, set(["oe_libinstall"])) |
390 | 390 | ||
@@ -399,7 +399,7 @@ esac | |||
399 | # Check dependencies | 399 | # Check dependencies |
400 | self.d.setVar('ANOTHERVAR', expr) | 400 | self.d.setVar('ANOTHERVAR', expr) |
401 | self.d.setVar('TESTVAR', 'anothervalue testval testval2') | 401 | self.d.setVar('TESTVAR', 'anothervalue testval testval2') |
402 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d) | 402 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d) |
403 | self.assertEqual(sorted(values.splitlines()), | 403 | self.assertEqual(sorted(values.splitlines()), |
404 | sorted([expr, | 404 | sorted([expr, |
405 | 'TESTVAR{anothervalue} = Set', | 405 | 'TESTVAR{anothervalue} = Set', |
@@ -418,14 +418,14 @@ esac | |||
418 | self.d.setVar('ANOTHERVAR', varval) | 418 | self.d.setVar('ANOTHERVAR', varval) |
419 | self.d.setVar('TESTVAR', 'anothervalue testval testval2') | 419 | self.d.setVar('TESTVAR', 'anothervalue testval testval2') |
420 | self.d.setVar('TESTVAR2', 'testval3') | 420 | self.d.setVar('TESTVAR2', 'testval3') |
421 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(["TESTVAR"]), self.d) | 421 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d) |
422 | self.assertEqual(sorted(values.splitlines()), sorted([varval])) | 422 | self.assertEqual(sorted(values.splitlines()), sorted([varval])) |
423 | self.assertEqual(deps, set(["TESTVAR2"])) | 423 | self.assertEqual(deps, set(["TESTVAR2"])) |
424 | self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) | 424 | self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) |
425 | 425 | ||
426 | # Check the vardepsexclude flag is handled by contains functionality | 426 | # Check the vardepsexclude flag is handled by contains functionality |
427 | self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR') | 427 | self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR') |
428 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d) | 428 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d) |
429 | self.assertEqual(sorted(values.splitlines()), sorted([varval])) | 429 | self.assertEqual(sorted(values.splitlines()), sorted([varval])) |
430 | self.assertEqual(deps, set(["TESTVAR2"])) | 430 | self.assertEqual(deps, set(["TESTVAR2"])) |
431 | self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) | 431 | self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) |