diff options
Diffstat (limited to 'bitbake/lib/bb/data_smart.py')
-rw-r--r-- | bitbake/lib/bb/data_smart.py | 268 |
1 files changed, 180 insertions, 88 deletions
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index 2328c334ac..0128a5bb17 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py | |||
@@ -16,8 +16,11 @@ BitBake build tools. | |||
16 | # | 16 | # |
17 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | 17 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig |
18 | 18 | ||
19 | import copy, re, sys, traceback | 19 | import builtins |
20 | from collections import MutableMapping | 20 | import copy |
21 | import re | ||
22 | import sys | ||
23 | from collections.abc import MutableMapping | ||
21 | import logging | 24 | import logging |
22 | import hashlib | 25 | import hashlib |
23 | import bb, bb.codeparser | 26 | import bb, bb.codeparser |
@@ -26,13 +29,25 @@ from bb.COW import COWDictBase | |||
26 | 29 | ||
27 | logger = logging.getLogger("BitBake.Data") | 30 | logger = logging.getLogger("BitBake.Data") |
28 | 31 | ||
29 | __setvar_keyword__ = ["_append", "_prepend", "_remove"] | 32 | __setvar_keyword__ = [":append", ":prepend", ":remove"] |
30 | __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$') | 33 | __setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$') |
31 | __expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}") | 34 | __expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}") |
32 | __expand_python_regexp__ = re.compile(r"\${@.+?}") | 35 | __expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}") |
33 | __whitespace_split__ = re.compile(r'(\s)') | 36 | __whitespace_split__ = re.compile(r'(\s)') |
34 | __override_regexp__ = re.compile(r'[a-z0-9]+') | 37 | __override_regexp__ = re.compile(r'[a-z0-9]+') |
35 | 38 | ||
39 | bitbake_renamed_vars = { | ||
40 | "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH", | ||
41 | "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS", | ||
42 | "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS", | ||
43 | "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS", | ||
44 | "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS", | ||
45 | "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS", | ||
46 | "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED", | ||
47 | "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed", | ||
48 | "BB_STAMP_POLICY": "is a deprecated variable and support has been removed", | ||
49 | } | ||
50 | |||
36 | def infer_caller_details(loginfo, parent = False, varval = True): | 51 | def infer_caller_details(loginfo, parent = False, varval = True): |
37 | """Save the caller the trouble of specifying everything.""" | 52 | """Save the caller the trouble of specifying everything.""" |
38 | # Save effort. | 53 | # Save effort. |
@@ -80,10 +95,11 @@ def infer_caller_details(loginfo, parent = False, varval = True): | |||
80 | loginfo['func'] = func | 95 | loginfo['func'] = func |
81 | 96 | ||
82 | class VariableParse: | 97 | class VariableParse: |
83 | def __init__(self, varname, d, val = None): | 98 | def __init__(self, varname, d, unexpanded_value = None, val = None): |
84 | self.varname = varname | 99 | self.varname = varname |
85 | self.d = d | 100 | self.d = d |
86 | self.value = val | 101 | self.value = val |
102 | self.unexpanded_value = unexpanded_value | ||
87 | 103 | ||
88 | self.references = set() | 104 | self.references = set() |
89 | self.execs = set() | 105 | self.execs = set() |
@@ -107,6 +123,11 @@ class VariableParse: | |||
107 | else: | 123 | else: |
108 | code = match.group()[3:-1] | 124 | code = match.group()[3:-1] |
109 | 125 | ||
126 | # Do not run code that contains one or more unexpanded variables | ||
127 | # instead return the code with the characters we removed put back | ||
128 | if __expand_var_regexp__.findall(code): | ||
129 | return "${@" + code + "}" | ||
130 | |||
110 | if self.varname: | 131 | if self.varname: |
111 | varname = 'Var <%s>' % self.varname | 132 | varname = 'Var <%s>' % self.varname |
112 | else: | 133 | else: |
@@ -132,16 +153,21 @@ class VariableParse: | |||
132 | value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) | 153 | value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) |
133 | return str(value) | 154 | return str(value) |
134 | 155 | ||
135 | |||
136 | class DataContext(dict): | 156 | class DataContext(dict): |
157 | excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe']) | ||
158 | |||
137 | def __init__(self, metadata, **kwargs): | 159 | def __init__(self, metadata, **kwargs): |
138 | self.metadata = metadata | 160 | self.metadata = metadata |
139 | dict.__init__(self, **kwargs) | 161 | dict.__init__(self, **kwargs) |
140 | self['d'] = metadata | 162 | self['d'] = metadata |
163 | self.context = set(bb.utils.get_context()) | ||
141 | 164 | ||
142 | def __missing__(self, key): | 165 | def __missing__(self, key): |
166 | if key in self.excluded or key in self.context: | ||
167 | raise KeyError(key) | ||
168 | |||
143 | value = self.metadata.getVar(key) | 169 | value = self.metadata.getVar(key) |
144 | if value is None or self.metadata.getVarFlag(key, 'func', False): | 170 | if value is None: |
145 | raise KeyError(key) | 171 | raise KeyError(key) |
146 | else: | 172 | else: |
147 | return value | 173 | return value |
@@ -151,6 +177,7 @@ class ExpansionError(Exception): | |||
151 | self.expression = expression | 177 | self.expression = expression |
152 | self.variablename = varname | 178 | self.variablename = varname |
153 | self.exception = exception | 179 | self.exception = exception |
180 | self.varlist = [varname or expression or ""] | ||
154 | if varname: | 181 | if varname: |
155 | if expression: | 182 | if expression: |
156 | self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) | 183 | self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) |
@@ -160,8 +187,14 @@ class ExpansionError(Exception): | |||
160 | self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) | 187 | self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) |
161 | Exception.__init__(self, self.msg) | 188 | Exception.__init__(self, self.msg) |
162 | self.args = (varname, expression, exception) | 189 | self.args = (varname, expression, exception) |
190 | |||
191 | def addVar(self, varname): | ||
192 | if varname: | ||
193 | self.varlist.append(varname) | ||
194 | |||
163 | def __str__(self): | 195 | def __str__(self): |
164 | return self.msg | 196 | chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist) |
197 | return self.msg + chain | ||
165 | 198 | ||
166 | class IncludeHistory(object): | 199 | class IncludeHistory(object): |
167 | def __init__(self, parent = None, filename = '[TOP LEVEL]'): | 200 | def __init__(self, parent = None, filename = '[TOP LEVEL]'): |
@@ -277,7 +310,7 @@ class VariableHistory(object): | |||
277 | for (r, override) in d.overridedata[var]: | 310 | for (r, override) in d.overridedata[var]: |
278 | for event in self.variable(r): | 311 | for event in self.variable(r): |
279 | loginfo = event.copy() | 312 | loginfo = event.copy() |
280 | if 'flag' in loginfo and not loginfo['flag'].startswith("_"): | 313 | if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")): |
281 | continue | 314 | continue |
282 | loginfo['variable'] = var | 315 | loginfo['variable'] = var |
283 | loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) | 316 | loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) |
@@ -329,6 +362,16 @@ class VariableHistory(object): | |||
329 | lines.append(line) | 362 | lines.append(line) |
330 | return lines | 363 | return lines |
331 | 364 | ||
365 | def get_variable_refs(self, var): | ||
366 | """Return a dict of file/line references""" | ||
367 | var_history = self.variable(var) | ||
368 | refs = {} | ||
369 | for event in var_history: | ||
370 | if event['file'] not in refs: | ||
371 | refs[event['file']] = [] | ||
372 | refs[event['file']].append(event['line']) | ||
373 | return refs | ||
374 | |||
332 | def get_variable_items_files(self, var): | 375 | def get_variable_items_files(self, var): |
333 | """ | 376 | """ |
334 | Use variable history to map items added to a list variable and | 377 | Use variable history to map items added to a list variable and |
@@ -342,7 +385,7 @@ class VariableHistory(object): | |||
342 | for event in history: | 385 | for event in history: |
343 | if 'flag' in event: | 386 | if 'flag' in event: |
344 | continue | 387 | continue |
345 | if event['op'] == '_remove': | 388 | if event['op'] == ':remove': |
346 | continue | 389 | continue |
347 | if isset and event['op'] == 'set?': | 390 | if isset and event['op'] == 'set?': |
348 | continue | 391 | continue |
@@ -363,6 +406,23 @@ class VariableHistory(object): | |||
363 | else: | 406 | else: |
364 | self.variables[var] = [] | 407 | self.variables[var] = [] |
365 | 408 | ||
409 | def _print_rename_error(var, loginfo, renamedvars, fullvar=None): | ||
410 | info = "" | ||
411 | if "file" in loginfo: | ||
412 | info = " file: %s" % loginfo["file"] | ||
413 | if "line" in loginfo: | ||
414 | info += " line: %s" % loginfo["line"] | ||
415 | if fullvar and fullvar != var: | ||
416 | info += " referenced as: %s" % fullvar | ||
417 | if info: | ||
418 | info = " (%s)" % info.strip() | ||
419 | renameinfo = renamedvars[var] | ||
420 | if " " in renameinfo: | ||
421 | # A space signals a string to display instead of a rename | ||
422 | bb.erroronce('Variable %s %s%s' % (var, renameinfo, info)) | ||
423 | else: | ||
424 | bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info)) | ||
425 | |||
366 | class DataSmart(MutableMapping): | 426 | class DataSmart(MutableMapping): |
367 | def __init__(self): | 427 | def __init__(self): |
368 | self.dict = {} | 428 | self.dict = {} |
@@ -370,6 +430,8 @@ class DataSmart(MutableMapping): | |||
370 | self.inchistory = IncludeHistory() | 430 | self.inchistory = IncludeHistory() |
371 | self.varhistory = VariableHistory(self) | 431 | self.varhistory = VariableHistory(self) |
372 | self._tracking = False | 432 | self._tracking = False |
433 | self._var_renames = {} | ||
434 | self._var_renames.update(bitbake_renamed_vars) | ||
373 | 435 | ||
374 | self.expand_cache = {} | 436 | self.expand_cache = {} |
375 | 437 | ||
@@ -391,9 +453,9 @@ class DataSmart(MutableMapping): | |||
391 | def expandWithRefs(self, s, varname): | 453 | def expandWithRefs(self, s, varname): |
392 | 454 | ||
393 | if not isinstance(s, str): # sanity check | 455 | if not isinstance(s, str): # sanity check |
394 | return VariableParse(varname, self, s) | 456 | return VariableParse(varname, self, s, s) |
395 | 457 | ||
396 | varparse = VariableParse(varname, self) | 458 | varparse = VariableParse(varname, self, s) |
397 | 459 | ||
398 | while s.find('${') != -1: | 460 | while s.find('${') != -1: |
399 | olds = s | 461 | olds = s |
@@ -403,14 +465,17 @@ class DataSmart(MutableMapping): | |||
403 | s = __expand_python_regexp__.sub(varparse.python_sub, s) | 465 | s = __expand_python_regexp__.sub(varparse.python_sub, s) |
404 | except SyntaxError as e: | 466 | except SyntaxError as e: |
405 | # Likely unmatched brackets, just don't expand the expression | 467 | # Likely unmatched brackets, just don't expand the expression |
406 | if e.msg != "EOL while scanning string literal": | 468 | if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"): |
407 | raise | 469 | raise |
408 | if s == olds: | 470 | if s == olds: |
409 | break | 471 | break |
410 | except ExpansionError: | 472 | except ExpansionError as e: |
473 | e.addVar(varname) | ||
411 | raise | 474 | raise |
412 | except bb.parse.SkipRecipe: | 475 | except bb.parse.SkipRecipe: |
413 | raise | 476 | raise |
477 | except bb.BBHandledException: | ||
478 | raise | ||
414 | except Exception as exc: | 479 | except Exception as exc: |
415 | tb = sys.exc_info()[2] | 480 | tb = sys.exc_info()[2] |
416 | raise ExpansionError(varname, s, exc).with_traceback(tb) from exc | 481 | raise ExpansionError(varname, s, exc).with_traceback(tb) from exc |
@@ -422,24 +487,19 @@ class DataSmart(MutableMapping): | |||
422 | def expand(self, s, varname = None): | 487 | def expand(self, s, varname = None): |
423 | return self.expandWithRefs(s, varname).value | 488 | return self.expandWithRefs(s, varname).value |
424 | 489 | ||
425 | def finalize(self, parent = False): | ||
426 | return | ||
427 | |||
428 | def internal_finalize(self, parent = False): | ||
429 | """Performs final steps upon the datastore, including application of overrides""" | ||
430 | self.overrides = None | ||
431 | |||
432 | def need_overrides(self): | 490 | def need_overrides(self): |
433 | if self.overrides is not None: | 491 | if self.overrides is not None: |
434 | return | 492 | return |
435 | if self.inoverride: | 493 | if self.inoverride: |
436 | return | 494 | return |
495 | overrride_stack = [] | ||
437 | for count in range(5): | 496 | for count in range(5): |
438 | self.inoverride = True | 497 | self.inoverride = True |
439 | # Can end up here recursively so setup dummy values | 498 | # Can end up here recursively so setup dummy values |
440 | self.overrides = [] | 499 | self.overrides = [] |
441 | self.overridesset = set() | 500 | self.overridesset = set() |
442 | self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] | 501 | self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] |
502 | overrride_stack.append(self.overrides) | ||
443 | self.overridesset = set(self.overrides) | 503 | self.overridesset = set(self.overrides) |
444 | self.inoverride = False | 504 | self.inoverride = False |
445 | self.expand_cache = {} | 505 | self.expand_cache = {} |
@@ -449,7 +509,7 @@ class DataSmart(MutableMapping): | |||
449 | self.overrides = newoverrides | 509 | self.overrides = newoverrides |
450 | self.overridesset = set(self.overrides) | 510 | self.overridesset = set(self.overrides) |
451 | else: | 511 | else: |
452 | bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") | 512 | bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack)) |
453 | 513 | ||
454 | def initVar(self, var): | 514 | def initVar(self, var): |
455 | self.expand_cache = {} | 515 | self.expand_cache = {} |
@@ -460,27 +520,44 @@ class DataSmart(MutableMapping): | |||
460 | dest = self.dict | 520 | dest = self.dict |
461 | while dest: | 521 | while dest: |
462 | if var in dest: | 522 | if var in dest: |
463 | return dest[var], self.overridedata.get(var, None) | 523 | return dest[var] |
464 | 524 | ||
465 | if "_data" not in dest: | 525 | if "_data" not in dest: |
466 | break | 526 | break |
467 | dest = dest["_data"] | 527 | dest = dest["_data"] |
468 | return None, self.overridedata.get(var, None) | 528 | return None |
469 | 529 | ||
470 | def _makeShadowCopy(self, var): | 530 | def _makeShadowCopy(self, var): |
471 | if var in self.dict: | 531 | if var in self.dict: |
472 | return | 532 | return |
473 | 533 | ||
474 | local_var, _ = self._findVar(var) | 534 | local_var = self._findVar(var) |
475 | 535 | ||
476 | if local_var: | 536 | if local_var: |
477 | self.dict[var] = copy.copy(local_var) | 537 | self.dict[var] = copy.copy(local_var) |
478 | else: | 538 | else: |
479 | self.initVar(var) | 539 | self.initVar(var) |
480 | 540 | ||
541 | def hasOverrides(self, var): | ||
542 | return var in self.overridedata | ||
481 | 543 | ||
482 | def setVar(self, var, value, **loginfo): | 544 | def setVar(self, var, value, **loginfo): |
483 | #print("var=" + str(var) + " val=" + str(value)) | 545 | #print("var=" + str(var) + " val=" + str(value)) |
546 | |||
547 | if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var): | ||
548 | info = "%s" % var | ||
549 | if "file" in loginfo: | ||
550 | info += " file: %s" % loginfo["file"] | ||
551 | if "line" in loginfo: | ||
552 | info += " line: %s" % loginfo["line"] | ||
553 | bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info) | ||
554 | |||
555 | shortvar = var.split(":", 1)[0] | ||
556 | if shortvar in self._var_renames: | ||
557 | _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var) | ||
558 | # Mark that we have seen a renamed variable | ||
559 | self.setVar("_FAILPARSINGERRORHANDLED", True) | ||
560 | |||
484 | self.expand_cache = {} | 561 | self.expand_cache = {} |
485 | parsing=False | 562 | parsing=False |
486 | if 'parsing' in loginfo: | 563 | if 'parsing' in loginfo: |
@@ -509,7 +586,7 @@ class DataSmart(MutableMapping): | |||
509 | # pay the cookie monster | 586 | # pay the cookie monster |
510 | 587 | ||
511 | # more cookies for the cookie monster | 588 | # more cookies for the cookie monster |
512 | if '_' in var: | 589 | if ':' in var: |
513 | self._setvar_update_overrides(base, **loginfo) | 590 | self._setvar_update_overrides(base, **loginfo) |
514 | 591 | ||
515 | if base in self.overridevars: | 592 | if base in self.overridevars: |
@@ -520,27 +597,27 @@ class DataSmart(MutableMapping): | |||
520 | self._makeShadowCopy(var) | 597 | self._makeShadowCopy(var) |
521 | 598 | ||
522 | if not parsing: | 599 | if not parsing: |
523 | if "_append" in self.dict[var]: | 600 | if ":append" in self.dict[var]: |
524 | del self.dict[var]["_append"] | 601 | del self.dict[var][":append"] |
525 | if "_prepend" in self.dict[var]: | 602 | if ":prepend" in self.dict[var]: |
526 | del self.dict[var]["_prepend"] | 603 | del self.dict[var][":prepend"] |
527 | if "_remove" in self.dict[var]: | 604 | if ":remove" in self.dict[var]: |
528 | del self.dict[var]["_remove"] | 605 | del self.dict[var][":remove"] |
529 | if var in self.overridedata: | 606 | if var in self.overridedata: |
530 | active = [] | 607 | active = [] |
531 | self.need_overrides() | 608 | self.need_overrides() |
532 | for (r, o) in self.overridedata[var]: | 609 | for (r, o) in self.overridedata[var]: |
533 | if o in self.overridesset: | 610 | if o in self.overridesset: |
534 | active.append(r) | 611 | active.append(r) |
535 | elif "_" in o: | 612 | elif ":" in o: |
536 | if set(o.split("_")).issubset(self.overridesset): | 613 | if set(o.split(":")).issubset(self.overridesset): |
537 | active.append(r) | 614 | active.append(r) |
538 | for a in active: | 615 | for a in active: |
539 | self.delVar(a) | 616 | self.delVar(a) |
540 | del self.overridedata[var] | 617 | del self.overridedata[var] |
541 | 618 | ||
542 | # more cookies for the cookie monster | 619 | # more cookies for the cookie monster |
543 | if '_' in var: | 620 | if ':' in var: |
544 | self._setvar_update_overrides(var, **loginfo) | 621 | self._setvar_update_overrides(var, **loginfo) |
545 | 622 | ||
546 | # setting var | 623 | # setting var |
@@ -562,12 +639,12 @@ class DataSmart(MutableMapping): | |||
562 | nextnew.update(vardata.references) | 639 | nextnew.update(vardata.references) |
563 | nextnew.update(vardata.contains.keys()) | 640 | nextnew.update(vardata.contains.keys()) |
564 | new = nextnew | 641 | new = nextnew |
565 | self.internal_finalize(True) | 642 | self.overrides = None |
566 | 643 | ||
567 | def _setvar_update_overrides(self, var, **loginfo): | 644 | def _setvar_update_overrides(self, var, **loginfo): |
568 | # aka pay the cookie monster | 645 | # aka pay the cookie monster |
569 | override = var[var.rfind('_')+1:] | 646 | override = var[var.rfind(':')+1:] |
570 | shortvar = var[:var.rfind('_')] | 647 | shortvar = var[:var.rfind(':')] |
571 | while override and __override_regexp__.match(override): | 648 | while override and __override_regexp__.match(override): |
572 | if shortvar not in self.overridedata: | 649 | if shortvar not in self.overridedata: |
573 | self.overridedata[shortvar] = [] | 650 | self.overridedata[shortvar] = [] |
@@ -576,9 +653,9 @@ class DataSmart(MutableMapping): | |||
576 | self.overridedata[shortvar] = list(self.overridedata[shortvar]) | 653 | self.overridedata[shortvar] = list(self.overridedata[shortvar]) |
577 | self.overridedata[shortvar].append([var, override]) | 654 | self.overridedata[shortvar].append([var, override]) |
578 | override = None | 655 | override = None |
579 | if "_" in shortvar: | 656 | if ":" in shortvar: |
580 | override = var[shortvar.rfind('_')+1:] | 657 | override = var[shortvar.rfind(':')+1:] |
581 | shortvar = var[:shortvar.rfind('_')] | 658 | shortvar = var[:shortvar.rfind(':')] |
582 | if len(shortvar) == 0: | 659 | if len(shortvar) == 0: |
583 | override = None | 660 | override = None |
584 | 661 | ||
@@ -602,10 +679,11 @@ class DataSmart(MutableMapping): | |||
602 | self.varhistory.record(**loginfo) | 679 | self.varhistory.record(**loginfo) |
603 | self.setVar(newkey, val, ignore=True, parsing=True) | 680 | self.setVar(newkey, val, ignore=True, parsing=True) |
604 | 681 | ||
605 | for i in (__setvar_keyword__): | 682 | srcflags = self.getVarFlags(key, False, True) or {} |
606 | src = self.getVarFlag(key, i, False) | 683 | for i in srcflags: |
607 | if src is None: | 684 | if i not in (__setvar_keyword__): |
608 | continue | 685 | continue |
686 | src = srcflags[i] | ||
609 | 687 | ||
610 | dest = self.getVarFlag(newkey, i, False) or [] | 688 | dest = self.getVarFlag(newkey, i, False) or [] |
611 | dest.extend(src) | 689 | dest.extend(src) |
@@ -617,7 +695,7 @@ class DataSmart(MutableMapping): | |||
617 | self.overridedata[newkey].append([v.replace(key, newkey), o]) | 695 | self.overridedata[newkey].append([v.replace(key, newkey), o]) |
618 | self.renameVar(v, v.replace(key, newkey)) | 696 | self.renameVar(v, v.replace(key, newkey)) |
619 | 697 | ||
620 | if '_' in newkey and val is None: | 698 | if ':' in newkey and val is None: |
621 | self._setvar_update_overrides(newkey, **loginfo) | 699 | self._setvar_update_overrides(newkey, **loginfo) |
622 | 700 | ||
623 | loginfo['variable'] = key | 701 | loginfo['variable'] = key |
@@ -629,12 +707,12 @@ class DataSmart(MutableMapping): | |||
629 | def appendVar(self, var, value, **loginfo): | 707 | def appendVar(self, var, value, **loginfo): |
630 | loginfo['op'] = 'append' | 708 | loginfo['op'] = 'append' |
631 | self.varhistory.record(**loginfo) | 709 | self.varhistory.record(**loginfo) |
632 | self.setVar(var + "_append", value, ignore=True, parsing=True) | 710 | self.setVar(var + ":append", value, ignore=True, parsing=True) |
633 | 711 | ||
634 | def prependVar(self, var, value, **loginfo): | 712 | def prependVar(self, var, value, **loginfo): |
635 | loginfo['op'] = 'prepend' | 713 | loginfo['op'] = 'prepend' |
636 | self.varhistory.record(**loginfo) | 714 | self.varhistory.record(**loginfo) |
637 | self.setVar(var + "_prepend", value, ignore=True, parsing=True) | 715 | self.setVar(var + ":prepend", value, ignore=True, parsing=True) |
638 | 716 | ||
639 | def delVar(self, var, **loginfo): | 717 | def delVar(self, var, **loginfo): |
640 | self.expand_cache = {} | 718 | self.expand_cache = {} |
@@ -645,10 +723,10 @@ class DataSmart(MutableMapping): | |||
645 | self.dict[var] = {} | 723 | self.dict[var] = {} |
646 | if var in self.overridedata: | 724 | if var in self.overridedata: |
647 | del self.overridedata[var] | 725 | del self.overridedata[var] |
648 | if '_' in var: | 726 | if ':' in var: |
649 | override = var[var.rfind('_')+1:] | 727 | override = var[var.rfind(':')+1:] |
650 | shortvar = var[:var.rfind('_')] | 728 | shortvar = var[:var.rfind(':')] |
651 | while override and override.islower(): | 729 | while override and __override_regexp__.match(override): |
652 | try: | 730 | try: |
653 | if shortvar in self.overridedata: | 731 | if shortvar in self.overridedata: |
654 | # Force CoW by recreating the list first | 732 | # Force CoW by recreating the list first |
@@ -657,15 +735,23 @@ class DataSmart(MutableMapping): | |||
657 | except ValueError as e: | 735 | except ValueError as e: |
658 | pass | 736 | pass |
659 | override = None | 737 | override = None |
660 | if "_" in shortvar: | 738 | if ":" in shortvar: |
661 | override = var[shortvar.rfind('_')+1:] | 739 | override = var[shortvar.rfind(':')+1:] |
662 | shortvar = var[:shortvar.rfind('_')] | 740 | shortvar = var[:shortvar.rfind(':')] |
663 | if len(shortvar) == 0: | 741 | if len(shortvar) == 0: |
664 | override = None | 742 | override = None |
665 | 743 | ||
666 | def setVarFlag(self, var, flag, value, **loginfo): | 744 | def setVarFlag(self, var, flag, value, **loginfo): |
667 | self.expand_cache = {} | 745 | self.expand_cache = {} |
668 | 746 | ||
747 | if var == "BB_RENAMED_VARIABLES": | ||
748 | self._var_renames[flag] = value | ||
749 | |||
750 | if var in self._var_renames: | ||
751 | _print_rename_error(var, loginfo, self._var_renames) | ||
752 | # Mark that we have seen a renamed variable | ||
753 | self.setVar("_FAILPARSINGERRORHANDLED", True) | ||
754 | |||
669 | if 'op' not in loginfo: | 755 | if 'op' not in loginfo: |
670 | loginfo['op'] = "set" | 756 | loginfo['op'] = "set" |
671 | loginfo['flag'] = flag | 757 | loginfo['flag'] = flag |
@@ -674,7 +760,7 @@ class DataSmart(MutableMapping): | |||
674 | self._makeShadowCopy(var) | 760 | self._makeShadowCopy(var) |
675 | self.dict[var][flag] = value | 761 | self.dict[var][flag] = value |
676 | 762 | ||
677 | if flag == "_defaultval" and '_' in var: | 763 | if flag == "_defaultval" and ':' in var: |
678 | self._setvar_update_overrides(var, **loginfo) | 764 | self._setvar_update_overrides(var, **loginfo) |
679 | if flag == "_defaultval" and var in self.overridevars: | 765 | if flag == "_defaultval" and var in self.overridevars: |
680 | self._setvar_update_overridevars(var, value) | 766 | self._setvar_update_overridevars(var, value) |
@@ -695,22 +781,27 @@ class DataSmart(MutableMapping): | |||
695 | return None | 781 | return None |
696 | cachename = var + "[" + flag + "]" | 782 | cachename = var + "[" + flag + "]" |
697 | 783 | ||
784 | if not expand and retparser and cachename in self.expand_cache: | ||
785 | return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename] | ||
786 | |||
698 | if expand and cachename in self.expand_cache: | 787 | if expand and cachename in self.expand_cache: |
699 | return self.expand_cache[cachename].value | 788 | return self.expand_cache[cachename].value |
700 | 789 | ||
701 | local_var, overridedata = self._findVar(var) | 790 | local_var = self._findVar(var) |
702 | value = None | 791 | value = None |
703 | removes = set() | 792 | removes = set() |
704 | if flag == "_content" and overridedata is not None and not parsing: | 793 | if flag == "_content" and not parsing: |
794 | overridedata = self.overridedata.get(var, None) | ||
795 | if flag == "_content" and not parsing and overridedata is not None: | ||
705 | match = False | 796 | match = False |
706 | active = {} | 797 | active = {} |
707 | self.need_overrides() | 798 | self.need_overrides() |
708 | for (r, o) in overridedata: | 799 | for (r, o) in overridedata: |
709 | # What about double overrides both with "_" in the name? | 800 | # FIXME What about double overrides both with "_" in the name? |
710 | if o in self.overridesset: | 801 | if o in self.overridesset: |
711 | active[o] = r | 802 | active[o] = r |
712 | elif "_" in o: | 803 | elif ":" in o: |
713 | if set(o.split("_")).issubset(self.overridesset): | 804 | if set(o.split(":")).issubset(self.overridesset): |
714 | active[o] = r | 805 | active[o] = r |
715 | 806 | ||
716 | mod = True | 807 | mod = True |
@@ -718,10 +809,10 @@ class DataSmart(MutableMapping): | |||
718 | mod = False | 809 | mod = False |
719 | for o in self.overrides: | 810 | for o in self.overrides: |
720 | for a in active.copy(): | 811 | for a in active.copy(): |
721 | if a.endswith("_" + o): | 812 | if a.endswith(":" + o): |
722 | t = active[a] | 813 | t = active[a] |
723 | del active[a] | 814 | del active[a] |
724 | active[a.replace("_" + o, "")] = t | 815 | active[a.replace(":" + o, "")] = t |
725 | mod = True | 816 | mod = True |
726 | elif a == o: | 817 | elif a == o: |
727 | match = active[a] | 818 | match = active[a] |
@@ -740,31 +831,31 @@ class DataSmart(MutableMapping): | |||
740 | value = copy.copy(local_var["_defaultval"]) | 831 | value = copy.copy(local_var["_defaultval"]) |
741 | 832 | ||
742 | 833 | ||
743 | if flag == "_content" and local_var is not None and "_append" in local_var and not parsing: | 834 | if flag == "_content" and local_var is not None and ":append" in local_var and not parsing: |
744 | if not value: | ||
745 | value = "" | ||
746 | self.need_overrides() | 835 | self.need_overrides() |
747 | for (r, o) in local_var["_append"]: | 836 | for (r, o) in local_var[":append"]: |
748 | match = True | 837 | match = True |
749 | if o: | 838 | if o: |
750 | for o2 in o.split("_"): | 839 | for o2 in o.split(":"): |
751 | if not o2 in self.overrides: | 840 | if not o2 in self.overrides: |
752 | match = False | 841 | match = False |
753 | if match: | 842 | if match: |
843 | if value is None: | ||
844 | value = "" | ||
754 | value = value + r | 845 | value = value + r |
755 | 846 | ||
756 | if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing: | 847 | if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing: |
757 | if not value: | ||
758 | value = "" | ||
759 | self.need_overrides() | 848 | self.need_overrides() |
760 | for (r, o) in local_var["_prepend"]: | 849 | for (r, o) in local_var[":prepend"]: |
761 | 850 | ||
762 | match = True | 851 | match = True |
763 | if o: | 852 | if o: |
764 | for o2 in o.split("_"): | 853 | for o2 in o.split(":"): |
765 | if not o2 in self.overrides: | 854 | if not o2 in self.overrides: |
766 | match = False | 855 | match = False |
767 | if match: | 856 | if match: |
857 | if value is None: | ||
858 | value = "" | ||
768 | value = r + value | 859 | value = r + value |
769 | 860 | ||
770 | parser = None | 861 | parser = None |
@@ -773,12 +864,12 @@ class DataSmart(MutableMapping): | |||
773 | if expand: | 864 | if expand: |
774 | value = parser.value | 865 | value = parser.value |
775 | 866 | ||
776 | if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing: | 867 | if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing: |
777 | self.need_overrides() | 868 | self.need_overrides() |
778 | for (r, o) in local_var["_remove"]: | 869 | for (r, o) in local_var[":remove"]: |
779 | match = True | 870 | match = True |
780 | if o: | 871 | if o: |
781 | for o2 in o.split("_"): | 872 | for o2 in o.split(":"): |
782 | if not o2 in self.overrides: | 873 | if not o2 in self.overrides: |
783 | match = False | 874 | match = False |
784 | if match: | 875 | if match: |
@@ -791,7 +882,7 @@ class DataSmart(MutableMapping): | |||
791 | expanded_removes[r] = self.expand(r).split() | 882 | expanded_removes[r] = self.expand(r).split() |
792 | 883 | ||
793 | parser.removes = set() | 884 | parser.removes = set() |
794 | val = "" | 885 | val = [] |
795 | for v in __whitespace_split__.split(parser.value): | 886 | for v in __whitespace_split__.split(parser.value): |
796 | skip = False | 887 | skip = False |
797 | for r in removes: | 888 | for r in removes: |
@@ -800,8 +891,8 @@ class DataSmart(MutableMapping): | |||
800 | skip = True | 891 | skip = True |
801 | if skip: | 892 | if skip: |
802 | continue | 893 | continue |
803 | val = val + v | 894 | val.append(v) |
804 | parser.value = val | 895 | parser.value = "".join(val) |
805 | if expand: | 896 | if expand: |
806 | value = parser.value | 897 | value = parser.value |
807 | 898 | ||
@@ -816,7 +907,7 @@ class DataSmart(MutableMapping): | |||
816 | def delVarFlag(self, var, flag, **loginfo): | 907 | def delVarFlag(self, var, flag, **loginfo): |
817 | self.expand_cache = {} | 908 | self.expand_cache = {} |
818 | 909 | ||
819 | local_var, _ = self._findVar(var) | 910 | local_var = self._findVar(var) |
820 | if not local_var: | 911 | if not local_var: |
821 | return | 912 | return |
822 | if not var in self.dict: | 913 | if not var in self.dict: |
@@ -859,12 +950,12 @@ class DataSmart(MutableMapping): | |||
859 | self.dict[var][i] = flags[i] | 950 | self.dict[var][i] = flags[i] |
860 | 951 | ||
861 | def getVarFlags(self, var, expand = False, internalflags=False): | 952 | def getVarFlags(self, var, expand = False, internalflags=False): |
862 | local_var, _ = self._findVar(var) | 953 | local_var = self._findVar(var) |
863 | flags = {} | 954 | flags = {} |
864 | 955 | ||
865 | if local_var: | 956 | if local_var: |
866 | for i in local_var: | 957 | for i in local_var: |
867 | if i.startswith("_") and not internalflags: | 958 | if i.startswith(("_", ":")) and not internalflags: |
868 | continue | 959 | continue |
869 | flags[i] = local_var[i] | 960 | flags[i] = local_var[i] |
870 | if expand and i in expand: | 961 | if expand and i in expand: |
@@ -905,6 +996,7 @@ class DataSmart(MutableMapping): | |||
905 | data.inchistory = self.inchistory.copy() | 996 | data.inchistory = self.inchistory.copy() |
906 | 997 | ||
907 | data._tracking = self._tracking | 998 | data._tracking = self._tracking |
999 | data._var_renames = self._var_renames | ||
908 | 1000 | ||
909 | data.overrides = None | 1001 | data.overrides = None |
910 | data.overridevars = copy.copy(self.overridevars) | 1002 | data.overridevars = copy.copy(self.overridevars) |
@@ -927,7 +1019,7 @@ class DataSmart(MutableMapping): | |||
927 | value = self.getVar(variable, False) | 1019 | value = self.getVar(variable, False) |
928 | for key in keys: | 1020 | for key in keys: |
929 | referrervalue = self.getVar(key, False) | 1021 | referrervalue = self.getVar(key, False) |
930 | if referrervalue and ref in referrervalue: | 1022 | if referrervalue and isinstance(referrervalue, str) and ref in referrervalue: |
931 | self.setVar(key, referrervalue.replace(ref, value)) | 1023 | self.setVar(key, referrervalue.replace(ref, value)) |
932 | 1024 | ||
933 | def localkeys(self): | 1025 | def localkeys(self): |
@@ -962,8 +1054,8 @@ class DataSmart(MutableMapping): | |||
962 | for (r, o) in self.overridedata[var]: | 1054 | for (r, o) in self.overridedata[var]: |
963 | if o in self.overridesset: | 1055 | if o in self.overridesset: |
964 | overrides.add(var) | 1056 | overrides.add(var) |
965 | elif "_" in o: | 1057 | elif ":" in o: |
966 | if set(o.split("_")).issubset(self.overridesset): | 1058 | if set(o.split(":")).issubset(self.overridesset): |
967 | overrides.add(var) | 1059 | overrides.add(var) |
968 | 1060 | ||
969 | for k in keylist(self.dict): | 1061 | for k in keylist(self.dict): |
@@ -993,10 +1085,10 @@ class DataSmart(MutableMapping): | |||
993 | d = self.createCopy() | 1085 | d = self.createCopy() |
994 | bb.data.expandKeys(d) | 1086 | bb.data.expandKeys(d) |
995 | 1087 | ||
996 | config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split()) | 1088 | config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split()) |
997 | keys = set(key for key in iter(d) if not key.startswith("__")) | 1089 | keys = set(key for key in iter(d) if not key.startswith("__")) |
998 | for key in keys: | 1090 | for key in keys: |
999 | if key in config_whitelist: | 1091 | if key in config_ignore_vars: |
1000 | continue | 1092 | continue |
1001 | 1093 | ||
1002 | value = d.getVar(key, False) or "" | 1094 | value = d.getVar(key, False) or "" |