diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-11-27 21:16:16 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2022-12-17 08:52:28 +0000 |
commit | f3bcd3c9a91f6212c30b9c778c11f3c8a9f6f1d4 (patch) | |
tree | 760b8dac27191ea1a625476d84c0651aa2788a5f | |
parent | a225aa3ec4726161172ca4b03d02751b1250e7ae (diff) | |
download | poky-f3bcd3c9a91f6212c30b9c778c11f3c8a9f6f1d4.tar.gz |
bitbake: ast/data/codeparser: Add dependencies from python module functions
Moving code into python modules is a very effective way to reduce parsing
time and overhead in recipes. The downside has always been that any
dependency information on which variables those functions access is lost
and the hashes can therefore become less reliable.
This patch adds parsing of the imported module functions and that dependency
information is them injected back into the hash dependency information.
Intermodule function references are resolved to the full function
call names in our module namespace to ensure interfunction dependencies
are correctly handled too.
(Bitbake rev: 605c478ce14cdc3c02d6ef6d57146a76d436a83c)
(Bitbake rev: 91441e157e495b02db44e19e836afad366ee8924)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | bitbake/lib/bb/codeparser.py | 43 | ||||
-rw-r--r-- | bitbake/lib/bb/data.py | 15 | ||||
-rw-r--r-- | bitbake/lib/bb/parse/ast.py | 12 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/codeparser.py | 14 |
4 files changed, 68 insertions, 16 deletions
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py index 9d66d3ae41..c0b1d7966f 100644 --- a/bitbake/lib/bb/codeparser.py +++ b/bitbake/lib/bb/codeparser.py | |||
@@ -27,6 +27,7 @@ import ast | |||
27 | import sys | 27 | import sys |
28 | import codegen | 28 | import codegen |
29 | import logging | 29 | import logging |
30 | import inspect | ||
30 | import bb.pysh as pysh | 31 | import bb.pysh as pysh |
31 | import bb.utils, bb.data | 32 | import bb.utils, bb.data |
32 | import hashlib | 33 | import hashlib |
@@ -58,10 +59,33 @@ def check_indent(codestr): | |||
58 | 59 | ||
59 | return codestr | 60 | return codestr |
60 | 61 | ||
61 | # A custom getstate/setstate using tuples is actually worth 15% cachesize by | 62 | modulecode_deps = {} |
62 | # avoiding duplication of the attribute names! | ||
63 | 63 | ||
64 | def add_module_functions(fn, functions, namespace): | ||
65 | fstat = os.stat(fn) | ||
66 | fixedhash = fn + ":" + str(fstat.st_size) + ":" + str(fstat.st_mtime) | ||
67 | for f in functions: | ||
68 | name = "%s.%s" % (namespace, f) | ||
69 | parser = PythonParser(name, logger) | ||
70 | try: | ||
71 | parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f) | ||
72 | #bb.warn("Cached %s" % f) | ||
73 | except KeyError: | ||
74 | lines, lineno = inspect.getsourcelines(functions[f]) | ||
75 | src = "".join(lines) | ||
76 | parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f) | ||
77 | #bb.warn("Not cached %s" % f) | ||
78 | execs = parser.execs.copy() | ||
79 | # Expand internal module exec references | ||
80 | for e in parser.execs: | ||
81 | if e in functions: | ||
82 | execs.remove(e) | ||
83 | execs.add(namespace + "." + e) | ||
84 | modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy()] | ||
85 | #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, src, parser.references, parser.execs, parser.var_execs, parser.contains)) | ||
64 | 86 | ||
87 | # A custom getstate/setstate using tuples is actually worth 15% cachesize by | ||
88 | # avoiding duplication of the attribute names! | ||
65 | class SetCache(object): | 89 | class SetCache(object): |
66 | def __init__(self): | 90 | def __init__(self): |
67 | self.setcache = {} | 91 | self.setcache = {} |
@@ -289,11 +313,17 @@ class PythonParser(): | |||
289 | self.unhandled_message = "in call of %s, argument '%s' is not a string literal" | 313 | self.unhandled_message = "in call of %s, argument '%s' is not a string literal" |
290 | self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) | 314 | self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message) |
291 | 315 | ||
292 | def parse_python(self, node, lineno=0, filename="<string>"): | 316 | # For the python module code it is expensive to have the function text so it is |
293 | if not node or not node.strip(): | 317 | # uses a different fixedhash to cache against. We can take the hit on obtaining the |
318 | # text if it isn't in the cache. | ||
319 | def parse_python(self, node, lineno=0, filename="<string>", fixedhash=None): | ||
320 | if not fixedhash and (not node or not node.strip()): | ||
294 | return | 321 | return |
295 | 322 | ||
296 | h = bbhash(str(node)) | 323 | if fixedhash: |
324 | h = fixedhash | ||
325 | else: | ||
326 | h = bbhash(str(node)) | ||
297 | 327 | ||
298 | if h in codeparsercache.pythoncache: | 328 | if h in codeparsercache.pythoncache: |
299 | self.references = set(codeparsercache.pythoncache[h].refs) | 329 | self.references = set(codeparsercache.pythoncache[h].refs) |
@@ -311,6 +341,9 @@ class PythonParser(): | |||
311 | self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) | 341 | self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i]) |
312 | return | 342 | return |
313 | 343 | ||
344 | if fixedhash and not node: | ||
345 | raise KeyError | ||
346 | |||
314 | # Need to parse so take the hit on the real log buffer | 347 | # Need to parse so take the hit on the real log buffer |
315 | self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log) | 348 | self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log) |
316 | 349 | ||
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py index bfaa0410ea..841369699e 100644 --- a/bitbake/lib/bb/data.py +++ b/bitbake/lib/bb/data.py | |||
@@ -261,7 +261,7 @@ def emit_func_python(func, o=sys.__stdout__, d = init()): | |||
261 | newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split()) | 261 | newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split()) |
262 | newdeps -= seen | 262 | newdeps -= seen |
263 | 263 | ||
264 | def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d): | 264 | def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d): |
265 | def handle_contains(value, contains, exclusions, d): | 265 | def handle_contains(value, contains, exclusions, d): |
266 | newvalue = [] | 266 | newvalue = [] |
267 | if value: | 267 | if value: |
@@ -289,6 +289,12 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d): | |||
289 | 289 | ||
290 | deps = set() | 290 | deps = set() |
291 | try: | 291 | try: |
292 | if key in mod_funcs: | ||
293 | exclusions = set() | ||
294 | moddep = bb.codeparser.modulecode_deps[key] | ||
295 | value = handle_contains("", moddep[3], exclusions, d) | ||
296 | return frozenset((moddep[0] | keys & moddep[1]) - ignored_vars), value | ||
297 | |||
292 | if key[-1] == ']': | 298 | if key[-1] == ']': |
293 | vf = key[:-1].split('[') | 299 | vf = key[:-1].split('[') |
294 | if vf[1] == "vardepvalueexclude": | 300 | if vf[1] == "vardepvalueexclude": |
@@ -367,7 +373,8 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, ignored_vars, d): | |||
367 | 373 | ||
368 | def generate_dependencies(d, ignored_vars): | 374 | def generate_dependencies(d, ignored_vars): |
369 | 375 | ||
370 | keys = set(key for key in d if not key.startswith("__")) | 376 | mod_funcs = set(bb.codeparser.modulecode_deps.keys()) |
377 | keys = set(key for key in d if not key.startswith("__")) | mod_funcs | ||
371 | shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) | 378 | shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False)) |
372 | varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS') | 379 | varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS') |
373 | 380 | ||
@@ -376,7 +383,7 @@ def generate_dependencies(d, ignored_vars): | |||
376 | 383 | ||
377 | tasklist = d.getVar('__BBTASKS', False) or [] | 384 | tasklist = d.getVar('__BBTASKS', False) or [] |
378 | for task in tasklist: | 385 | for task in tasklist: |
379 | deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, ignored_vars, d) | 386 | deps[task], values[task] = build_dependencies(task, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d) |
380 | newdeps = deps[task] | 387 | newdeps = deps[task] |
381 | seen = set() | 388 | seen = set() |
382 | while newdeps: | 389 | while newdeps: |
@@ -385,7 +392,7 @@ def generate_dependencies(d, ignored_vars): | |||
385 | newdeps = set() | 392 | newdeps = set() |
386 | for dep in nextdeps: | 393 | for dep in nextdeps: |
387 | if dep not in deps: | 394 | if dep not in deps: |
388 | deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, ignored_vars, d) | 395 | deps[dep], values[dep] = build_dependencies(dep, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d) |
389 | newdeps |= deps[dep] | 396 | newdeps |= deps[dep] |
390 | newdeps -= seen | 397 | newdeps -= seen |
391 | #print "For %s: %s" % (task, str(deps[task])) | 398 | #print "For %s: %s" % (task, str(deps[task])) |
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py index 862087c77d..375ba3cb79 100644 --- a/bitbake/lib/bb/parse/ast.py +++ b/bitbake/lib/bb/parse/ast.py | |||
@@ -290,6 +290,18 @@ class PyLibNode(AstNode): | |||
290 | toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", []) | 290 | toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", []) |
291 | for i in toimport: | 291 | for i in toimport: |
292 | bb.utils._context[self.namespace] = __import__(self.namespace + "." + i) | 292 | bb.utils._context[self.namespace] = __import__(self.namespace + "." + i) |
293 | mod = getattr(bb.utils._context[self.namespace], i) | ||
294 | fn = getattr(mod, "__file__") | ||
295 | funcs = {} | ||
296 | for f in dir(mod): | ||
297 | if f.startswith("_"): | ||
298 | continue | ||
299 | fcall = getattr(mod, f) | ||
300 | if not callable(fcall): | ||
301 | continue | ||
302 | funcs[f] = fcall | ||
303 | bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i)) | ||
304 | |||
293 | except AttributeError as e: | 305 | except AttributeError as e: |
294 | bb.error("Error importing OE modules: %s" % str(e)) | 306 | bb.error("Error importing OE modules: %s" % str(e)) |
295 | 307 | ||
diff --git a/bitbake/lib/bb/tests/codeparser.py b/bitbake/lib/bb/tests/codeparser.py index 71ed382ab8..a508f23bcb 100644 --- a/bitbake/lib/bb/tests/codeparser.py +++ b/bitbake/lib/bb/tests/codeparser.py | |||
@@ -318,7 +318,7 @@ d.getVar(a(), False) | |||
318 | "filename": "example.bb", | 318 | "filename": "example.bb", |
319 | }) | 319 | }) |
320 | 320 | ||
321 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) | 321 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d) |
322 | 322 | ||
323 | self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) | 323 | self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"])) |
324 | 324 | ||
@@ -365,7 +365,7 @@ esac | |||
365 | self.d.setVarFlags("FOO", {"func": True}) | 365 | self.d.setVarFlags("FOO", {"func": True}) |
366 | self.setEmptyVars(execs) | 366 | self.setEmptyVars(execs) |
367 | 367 | ||
368 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) | 368 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d) |
369 | 369 | ||
370 | self.assertEqual(deps, set(["somevar", "inverted"] + execs)) | 370 | self.assertEqual(deps, set(["somevar", "inverted"] + execs)) |
371 | 371 | ||
@@ -375,7 +375,7 @@ esac | |||
375 | self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") | 375 | self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") |
376 | self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") | 376 | self.d.setVarFlag("FOO", "vardeps", "oe_libinstall") |
377 | 377 | ||
378 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) | 378 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d) |
379 | 379 | ||
380 | self.assertEqual(deps, set(["oe_libinstall"])) | 380 | self.assertEqual(deps, set(["oe_libinstall"])) |
381 | 381 | ||
@@ -384,7 +384,7 @@ esac | |||
384 | self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") | 384 | self.d.setVar("FOO", "foo=oe_libinstall; eval $foo") |
385 | self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") | 385 | self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}") |
386 | 386 | ||
387 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), self.d) | 387 | deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d) |
388 | 388 | ||
389 | self.assertEqual(deps, set(["oe_libinstall"])) | 389 | self.assertEqual(deps, set(["oe_libinstall"])) |
390 | 390 | ||
@@ -399,7 +399,7 @@ esac | |||
399 | # Check dependencies | 399 | # Check dependencies |
400 | self.d.setVar('ANOTHERVAR', expr) | 400 | self.d.setVar('ANOTHERVAR', expr) |
401 | self.d.setVar('TESTVAR', 'anothervalue testval testval2') | 401 | self.d.setVar('TESTVAR', 'anothervalue testval testval2') |
402 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d) | 402 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d) |
403 | self.assertEqual(sorted(values.splitlines()), | 403 | self.assertEqual(sorted(values.splitlines()), |
404 | sorted([expr, | 404 | sorted([expr, |
405 | 'TESTVAR{anothervalue} = Set', | 405 | 'TESTVAR{anothervalue} = Set', |
@@ -418,14 +418,14 @@ esac | |||
418 | self.d.setVar('ANOTHERVAR', varval) | 418 | self.d.setVar('ANOTHERVAR', varval) |
419 | self.d.setVar('TESTVAR', 'anothervalue testval testval2') | 419 | self.d.setVar('TESTVAR', 'anothervalue testval testval2') |
420 | self.d.setVar('TESTVAR2', 'testval3') | 420 | self.d.setVar('TESTVAR2', 'testval3') |
421 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(["TESTVAR"]), self.d) | 421 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d) |
422 | self.assertEqual(sorted(values.splitlines()), sorted([varval])) | 422 | self.assertEqual(sorted(values.splitlines()), sorted([varval])) |
423 | self.assertEqual(deps, set(["TESTVAR2"])) | 423 | self.assertEqual(deps, set(["TESTVAR2"])) |
424 | self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) | 424 | self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) |
425 | 425 | ||
426 | # Check the vardepsexclude flag is handled by contains functionality | 426 | # Check the vardepsexclude flag is handled by contains functionality |
427 | self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR') | 427 | self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR') |
428 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), self.d) | 428 | deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d) |
429 | self.assertEqual(sorted(values.splitlines()), sorted([varval])) | 429 | self.assertEqual(sorted(values.splitlines()), sorted([varval])) |
430 | self.assertEqual(deps, set(["TESTVAR2"])) | 430 | self.assertEqual(deps, set(["TESTVAR2"])) |
431 | self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) | 431 | self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue']) |