summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/data_smart.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/data_smart.py')
-rw-r--r--bitbake/lib/bb/data_smart.py268
1 files changed, 180 insertions, 88 deletions
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 2328c334ac..0128a5bb17 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -16,8 +16,11 @@ BitBake build tools.
16# 16#
17# Based on functions from the base bb module, Copyright 2003 Holger Schurig 17# Based on functions from the base bb module, Copyright 2003 Holger Schurig
18 18
19import copy, re, sys, traceback 19import builtins
20from collections import MutableMapping 20import copy
21import re
22import sys
23from collections.abc import MutableMapping
21import logging 24import logging
22import hashlib 25import hashlib
23import bb, bb.codeparser 26import bb, bb.codeparser
@@ -26,13 +29,25 @@ from bb.COW import COWDictBase
26 29
27logger = logging.getLogger("BitBake.Data") 30logger = logging.getLogger("BitBake.Data")
28 31
29__setvar_keyword__ = ["_append", "_prepend", "_remove"] 32__setvar_keyword__ = [":append", ":prepend", ":remove"]
30__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$') 33__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
31__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}") 34__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+?}")
32__expand_python_regexp__ = re.compile(r"\${@.+?}") 35__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}")
33__whitespace_split__ = re.compile(r'(\s)') 36__whitespace_split__ = re.compile(r'(\s)')
34__override_regexp__ = re.compile(r'[a-z0-9]+') 37__override_regexp__ = re.compile(r'[a-z0-9]+')
35 38
39bitbake_renamed_vars = {
40 "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH",
41 "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS",
42 "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS",
43 "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS",
44 "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS",
45 "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS",
46 "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED",
47 "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed",
48 "BB_STAMP_POLICY": "is a deprecated variable and support has been removed",
49}
50
36def infer_caller_details(loginfo, parent = False, varval = True): 51def infer_caller_details(loginfo, parent = False, varval = True):
37 """Save the caller the trouble of specifying everything.""" 52 """Save the caller the trouble of specifying everything."""
38 # Save effort. 53 # Save effort.
@@ -80,10 +95,11 @@ def infer_caller_details(loginfo, parent = False, varval = True):
80 loginfo['func'] = func 95 loginfo['func'] = func
81 96
82class VariableParse: 97class VariableParse:
83 def __init__(self, varname, d, val = None): 98 def __init__(self, varname, d, unexpanded_value = None, val = None):
84 self.varname = varname 99 self.varname = varname
85 self.d = d 100 self.d = d
86 self.value = val 101 self.value = val
102 self.unexpanded_value = unexpanded_value
87 103
88 self.references = set() 104 self.references = set()
89 self.execs = set() 105 self.execs = set()
@@ -107,6 +123,11 @@ class VariableParse:
107 else: 123 else:
108 code = match.group()[3:-1] 124 code = match.group()[3:-1]
109 125
126 # Do not run code that contains one or more unexpanded variables
127 # instead return the code with the characters we removed put back
128 if __expand_var_regexp__.findall(code):
129 return "${@" + code + "}"
130
110 if self.varname: 131 if self.varname:
111 varname = 'Var <%s>' % self.varname 132 varname = 'Var <%s>' % self.varname
112 else: 133 else:
@@ -132,16 +153,21 @@ class VariableParse:
132 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d}) 153 value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
133 return str(value) 154 return str(value)
134 155
135
136class DataContext(dict): 156class DataContext(dict):
157 excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe'])
158
137 def __init__(self, metadata, **kwargs): 159 def __init__(self, metadata, **kwargs):
138 self.metadata = metadata 160 self.metadata = metadata
139 dict.__init__(self, **kwargs) 161 dict.__init__(self, **kwargs)
140 self['d'] = metadata 162 self['d'] = metadata
163 self.context = set(bb.utils.get_context())
141 164
142 def __missing__(self, key): 165 def __missing__(self, key):
166 if key in self.excluded or key in self.context:
167 raise KeyError(key)
168
143 value = self.metadata.getVar(key) 169 value = self.metadata.getVar(key)
144 if value is None or self.metadata.getVarFlag(key, 'func', False): 170 if value is None:
145 raise KeyError(key) 171 raise KeyError(key)
146 else: 172 else:
147 return value 173 return value
@@ -151,6 +177,7 @@ class ExpansionError(Exception):
151 self.expression = expression 177 self.expression = expression
152 self.variablename = varname 178 self.variablename = varname
153 self.exception = exception 179 self.exception = exception
180 self.varlist = [varname or expression or ""]
154 if varname: 181 if varname:
155 if expression: 182 if expression:
156 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) 183 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
@@ -160,8 +187,14 @@ class ExpansionError(Exception):
160 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) 187 self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
161 Exception.__init__(self, self.msg) 188 Exception.__init__(self, self.msg)
162 self.args = (varname, expression, exception) 189 self.args = (varname, expression, exception)
190
191 def addVar(self, varname):
192 if varname:
193 self.varlist.append(varname)
194
163 def __str__(self): 195 def __str__(self):
164 return self.msg 196 chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist)
197 return self.msg + chain
165 198
166class IncludeHistory(object): 199class IncludeHistory(object):
167 def __init__(self, parent = None, filename = '[TOP LEVEL]'): 200 def __init__(self, parent = None, filename = '[TOP LEVEL]'):
@@ -277,7 +310,7 @@ class VariableHistory(object):
277 for (r, override) in d.overridedata[var]: 310 for (r, override) in d.overridedata[var]:
278 for event in self.variable(r): 311 for event in self.variable(r):
279 loginfo = event.copy() 312 loginfo = event.copy()
280 if 'flag' in loginfo and not loginfo['flag'].startswith("_"): 313 if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")):
281 continue 314 continue
282 loginfo['variable'] = var 315 loginfo['variable'] = var
283 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op']) 316 loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
@@ -329,6 +362,16 @@ class VariableHistory(object):
329 lines.append(line) 362 lines.append(line)
330 return lines 363 return lines
331 364
365 def get_variable_refs(self, var):
366 """Return a dict of file/line references"""
367 var_history = self.variable(var)
368 refs = {}
369 for event in var_history:
370 if event['file'] not in refs:
371 refs[event['file']] = []
372 refs[event['file']].append(event['line'])
373 return refs
374
332 def get_variable_items_files(self, var): 375 def get_variable_items_files(self, var):
333 """ 376 """
334 Use variable history to map items added to a list variable and 377 Use variable history to map items added to a list variable and
@@ -342,7 +385,7 @@ class VariableHistory(object):
342 for event in history: 385 for event in history:
343 if 'flag' in event: 386 if 'flag' in event:
344 continue 387 continue
345 if event['op'] == '_remove': 388 if event['op'] == ':remove':
346 continue 389 continue
347 if isset and event['op'] == 'set?': 390 if isset and event['op'] == 'set?':
348 continue 391 continue
@@ -363,6 +406,23 @@ class VariableHistory(object):
363 else: 406 else:
364 self.variables[var] = [] 407 self.variables[var] = []
365 408
409def _print_rename_error(var, loginfo, renamedvars, fullvar=None):
410 info = ""
411 if "file" in loginfo:
412 info = " file: %s" % loginfo["file"]
413 if "line" in loginfo:
414 info += " line: %s" % loginfo["line"]
415 if fullvar and fullvar != var:
416 info += " referenced as: %s" % fullvar
417 if info:
418 info = " (%s)" % info.strip()
419 renameinfo = renamedvars[var]
420 if " " in renameinfo:
421 # A space signals a string to display instead of a rename
422 bb.erroronce('Variable %s %s%s' % (var, renameinfo, info))
423 else:
424 bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info))
425
366class DataSmart(MutableMapping): 426class DataSmart(MutableMapping):
367 def __init__(self): 427 def __init__(self):
368 self.dict = {} 428 self.dict = {}
@@ -370,6 +430,8 @@ class DataSmart(MutableMapping):
370 self.inchistory = IncludeHistory() 430 self.inchistory = IncludeHistory()
371 self.varhistory = VariableHistory(self) 431 self.varhistory = VariableHistory(self)
372 self._tracking = False 432 self._tracking = False
433 self._var_renames = {}
434 self._var_renames.update(bitbake_renamed_vars)
373 435
374 self.expand_cache = {} 436 self.expand_cache = {}
375 437
@@ -391,9 +453,9 @@ class DataSmart(MutableMapping):
391 def expandWithRefs(self, s, varname): 453 def expandWithRefs(self, s, varname):
392 454
393 if not isinstance(s, str): # sanity check 455 if not isinstance(s, str): # sanity check
394 return VariableParse(varname, self, s) 456 return VariableParse(varname, self, s, s)
395 457
396 varparse = VariableParse(varname, self) 458 varparse = VariableParse(varname, self, s)
397 459
398 while s.find('${') != -1: 460 while s.find('${') != -1:
399 olds = s 461 olds = s
@@ -403,14 +465,17 @@ class DataSmart(MutableMapping):
403 s = __expand_python_regexp__.sub(varparse.python_sub, s) 465 s = __expand_python_regexp__.sub(varparse.python_sub, s)
404 except SyntaxError as e: 466 except SyntaxError as e:
405 # Likely unmatched brackets, just don't expand the expression 467 # Likely unmatched brackets, just don't expand the expression
406 if e.msg != "EOL while scanning string literal": 468 if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
407 raise 469 raise
408 if s == olds: 470 if s == olds:
409 break 471 break
410 except ExpansionError: 472 except ExpansionError as e:
473 e.addVar(varname)
411 raise 474 raise
412 except bb.parse.SkipRecipe: 475 except bb.parse.SkipRecipe:
413 raise 476 raise
477 except bb.BBHandledException:
478 raise
414 except Exception as exc: 479 except Exception as exc:
415 tb = sys.exc_info()[2] 480 tb = sys.exc_info()[2]
416 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc 481 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
@@ -422,24 +487,19 @@ class DataSmart(MutableMapping):
422 def expand(self, s, varname = None): 487 def expand(self, s, varname = None):
423 return self.expandWithRefs(s, varname).value 488 return self.expandWithRefs(s, varname).value
424 489
425 def finalize(self, parent = False):
426 return
427
428 def internal_finalize(self, parent = False):
429 """Performs final steps upon the datastore, including application of overrides"""
430 self.overrides = None
431
432 def need_overrides(self): 490 def need_overrides(self):
433 if self.overrides is not None: 491 if self.overrides is not None:
434 return 492 return
435 if self.inoverride: 493 if self.inoverride:
436 return 494 return
495 overrride_stack = []
437 for count in range(5): 496 for count in range(5):
438 self.inoverride = True 497 self.inoverride = True
439 # Can end up here recursively so setup dummy values 498 # Can end up here recursively so setup dummy values
440 self.overrides = [] 499 self.overrides = []
441 self.overridesset = set() 500 self.overridesset = set()
442 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or [] 501 self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
502 overrride_stack.append(self.overrides)
443 self.overridesset = set(self.overrides) 503 self.overridesset = set(self.overrides)
444 self.inoverride = False 504 self.inoverride = False
445 self.expand_cache = {} 505 self.expand_cache = {}
@@ -449,7 +509,7 @@ class DataSmart(MutableMapping):
449 self.overrides = newoverrides 509 self.overrides = newoverrides
450 self.overridesset = set(self.overrides) 510 self.overridesset = set(self.overrides)
451 else: 511 else:
452 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.") 512 bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack))
453 513
454 def initVar(self, var): 514 def initVar(self, var):
455 self.expand_cache = {} 515 self.expand_cache = {}
@@ -460,27 +520,44 @@ class DataSmart(MutableMapping):
460 dest = self.dict 520 dest = self.dict
461 while dest: 521 while dest:
462 if var in dest: 522 if var in dest:
463 return dest[var], self.overridedata.get(var, None) 523 return dest[var]
464 524
465 if "_data" not in dest: 525 if "_data" not in dest:
466 break 526 break
467 dest = dest["_data"] 527 dest = dest["_data"]
468 return None, self.overridedata.get(var, None) 528 return None
469 529
470 def _makeShadowCopy(self, var): 530 def _makeShadowCopy(self, var):
471 if var in self.dict: 531 if var in self.dict:
472 return 532 return
473 533
474 local_var, _ = self._findVar(var) 534 local_var = self._findVar(var)
475 535
476 if local_var: 536 if local_var:
477 self.dict[var] = copy.copy(local_var) 537 self.dict[var] = copy.copy(local_var)
478 else: 538 else:
479 self.initVar(var) 539 self.initVar(var)
480 540
541 def hasOverrides(self, var):
542 return var in self.overridedata
481 543
482 def setVar(self, var, value, **loginfo): 544 def setVar(self, var, value, **loginfo):
483 #print("var=" + str(var) + " val=" + str(value)) 545 #print("var=" + str(var) + " val=" + str(value))
546
547 if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var):
548 info = "%s" % var
549 if "file" in loginfo:
550 info += " file: %s" % loginfo["file"]
551 if "line" in loginfo:
552 info += " line: %s" % loginfo["line"]
553 bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
554
555 shortvar = var.split(":", 1)[0]
556 if shortvar in self._var_renames:
557 _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var)
558 # Mark that we have seen a renamed variable
559 self.setVar("_FAILPARSINGERRORHANDLED", True)
560
484 self.expand_cache = {} 561 self.expand_cache = {}
485 parsing=False 562 parsing=False
486 if 'parsing' in loginfo: 563 if 'parsing' in loginfo:
@@ -509,7 +586,7 @@ class DataSmart(MutableMapping):
509 # pay the cookie monster 586 # pay the cookie monster
510 587
511 # more cookies for the cookie monster 588 # more cookies for the cookie monster
512 if '_' in var: 589 if ':' in var:
513 self._setvar_update_overrides(base, **loginfo) 590 self._setvar_update_overrides(base, **loginfo)
514 591
515 if base in self.overridevars: 592 if base in self.overridevars:
@@ -520,27 +597,27 @@ class DataSmart(MutableMapping):
520 self._makeShadowCopy(var) 597 self._makeShadowCopy(var)
521 598
522 if not parsing: 599 if not parsing:
523 if "_append" in self.dict[var]: 600 if ":append" in self.dict[var]:
524 del self.dict[var]["_append"] 601 del self.dict[var][":append"]
525 if "_prepend" in self.dict[var]: 602 if ":prepend" in self.dict[var]:
526 del self.dict[var]["_prepend"] 603 del self.dict[var][":prepend"]
527 if "_remove" in self.dict[var]: 604 if ":remove" in self.dict[var]:
528 del self.dict[var]["_remove"] 605 del self.dict[var][":remove"]
529 if var in self.overridedata: 606 if var in self.overridedata:
530 active = [] 607 active = []
531 self.need_overrides() 608 self.need_overrides()
532 for (r, o) in self.overridedata[var]: 609 for (r, o) in self.overridedata[var]:
533 if o in self.overridesset: 610 if o in self.overridesset:
534 active.append(r) 611 active.append(r)
535 elif "_" in o: 612 elif ":" in o:
536 if set(o.split("_")).issubset(self.overridesset): 613 if set(o.split(":")).issubset(self.overridesset):
537 active.append(r) 614 active.append(r)
538 for a in active: 615 for a in active:
539 self.delVar(a) 616 self.delVar(a)
540 del self.overridedata[var] 617 del self.overridedata[var]
541 618
542 # more cookies for the cookie monster 619 # more cookies for the cookie monster
543 if '_' in var: 620 if ':' in var:
544 self._setvar_update_overrides(var, **loginfo) 621 self._setvar_update_overrides(var, **loginfo)
545 622
546 # setting var 623 # setting var
@@ -562,12 +639,12 @@ class DataSmart(MutableMapping):
562 nextnew.update(vardata.references) 639 nextnew.update(vardata.references)
563 nextnew.update(vardata.contains.keys()) 640 nextnew.update(vardata.contains.keys())
564 new = nextnew 641 new = nextnew
565 self.internal_finalize(True) 642 self.overrides = None
566 643
567 def _setvar_update_overrides(self, var, **loginfo): 644 def _setvar_update_overrides(self, var, **loginfo):
568 # aka pay the cookie monster 645 # aka pay the cookie monster
569 override = var[var.rfind('_')+1:] 646 override = var[var.rfind(':')+1:]
570 shortvar = var[:var.rfind('_')] 647 shortvar = var[:var.rfind(':')]
571 while override and __override_regexp__.match(override): 648 while override and __override_regexp__.match(override):
572 if shortvar not in self.overridedata: 649 if shortvar not in self.overridedata:
573 self.overridedata[shortvar] = [] 650 self.overridedata[shortvar] = []
@@ -576,9 +653,9 @@ class DataSmart(MutableMapping):
576 self.overridedata[shortvar] = list(self.overridedata[shortvar]) 653 self.overridedata[shortvar] = list(self.overridedata[shortvar])
577 self.overridedata[shortvar].append([var, override]) 654 self.overridedata[shortvar].append([var, override])
578 override = None 655 override = None
579 if "_" in shortvar: 656 if ":" in shortvar:
580 override = var[shortvar.rfind('_')+1:] 657 override = var[shortvar.rfind(':')+1:]
581 shortvar = var[:shortvar.rfind('_')] 658 shortvar = var[:shortvar.rfind(':')]
582 if len(shortvar) == 0: 659 if len(shortvar) == 0:
583 override = None 660 override = None
584 661
@@ -602,10 +679,11 @@ class DataSmart(MutableMapping):
602 self.varhistory.record(**loginfo) 679 self.varhistory.record(**loginfo)
603 self.setVar(newkey, val, ignore=True, parsing=True) 680 self.setVar(newkey, val, ignore=True, parsing=True)
604 681
605 for i in (__setvar_keyword__): 682 srcflags = self.getVarFlags(key, False, True) or {}
606 src = self.getVarFlag(key, i, False) 683 for i in srcflags:
607 if src is None: 684 if i not in (__setvar_keyword__):
608 continue 685 continue
686 src = srcflags[i]
609 687
610 dest = self.getVarFlag(newkey, i, False) or [] 688 dest = self.getVarFlag(newkey, i, False) or []
611 dest.extend(src) 689 dest.extend(src)
@@ -617,7 +695,7 @@ class DataSmart(MutableMapping):
617 self.overridedata[newkey].append([v.replace(key, newkey), o]) 695 self.overridedata[newkey].append([v.replace(key, newkey), o])
618 self.renameVar(v, v.replace(key, newkey)) 696 self.renameVar(v, v.replace(key, newkey))
619 697
620 if '_' in newkey and val is None: 698 if ':' in newkey and val is None:
621 self._setvar_update_overrides(newkey, **loginfo) 699 self._setvar_update_overrides(newkey, **loginfo)
622 700
623 loginfo['variable'] = key 701 loginfo['variable'] = key
@@ -629,12 +707,12 @@ class DataSmart(MutableMapping):
629 def appendVar(self, var, value, **loginfo): 707 def appendVar(self, var, value, **loginfo):
630 loginfo['op'] = 'append' 708 loginfo['op'] = 'append'
631 self.varhistory.record(**loginfo) 709 self.varhistory.record(**loginfo)
632 self.setVar(var + "_append", value, ignore=True, parsing=True) 710 self.setVar(var + ":append", value, ignore=True, parsing=True)
633 711
634 def prependVar(self, var, value, **loginfo): 712 def prependVar(self, var, value, **loginfo):
635 loginfo['op'] = 'prepend' 713 loginfo['op'] = 'prepend'
636 self.varhistory.record(**loginfo) 714 self.varhistory.record(**loginfo)
637 self.setVar(var + "_prepend", value, ignore=True, parsing=True) 715 self.setVar(var + ":prepend", value, ignore=True, parsing=True)
638 716
639 def delVar(self, var, **loginfo): 717 def delVar(self, var, **loginfo):
640 self.expand_cache = {} 718 self.expand_cache = {}
@@ -645,10 +723,10 @@ class DataSmart(MutableMapping):
645 self.dict[var] = {} 723 self.dict[var] = {}
646 if var in self.overridedata: 724 if var in self.overridedata:
647 del self.overridedata[var] 725 del self.overridedata[var]
648 if '_' in var: 726 if ':' in var:
649 override = var[var.rfind('_')+1:] 727 override = var[var.rfind(':')+1:]
650 shortvar = var[:var.rfind('_')] 728 shortvar = var[:var.rfind(':')]
651 while override and override.islower(): 729 while override and __override_regexp__.match(override):
652 try: 730 try:
653 if shortvar in self.overridedata: 731 if shortvar in self.overridedata:
654 # Force CoW by recreating the list first 732 # Force CoW by recreating the list first
@@ -657,15 +735,23 @@ class DataSmart(MutableMapping):
657 except ValueError as e: 735 except ValueError as e:
658 pass 736 pass
659 override = None 737 override = None
660 if "_" in shortvar: 738 if ":" in shortvar:
661 override = var[shortvar.rfind('_')+1:] 739 override = var[shortvar.rfind(':')+1:]
662 shortvar = var[:shortvar.rfind('_')] 740 shortvar = var[:shortvar.rfind(':')]
663 if len(shortvar) == 0: 741 if len(shortvar) == 0:
664 override = None 742 override = None
665 743
666 def setVarFlag(self, var, flag, value, **loginfo): 744 def setVarFlag(self, var, flag, value, **loginfo):
667 self.expand_cache = {} 745 self.expand_cache = {}
668 746
747 if var == "BB_RENAMED_VARIABLES":
748 self._var_renames[flag] = value
749
750 if var in self._var_renames:
751 _print_rename_error(var, loginfo, self._var_renames)
752 # Mark that we have seen a renamed variable
753 self.setVar("_FAILPARSINGERRORHANDLED", True)
754
669 if 'op' not in loginfo: 755 if 'op' not in loginfo:
670 loginfo['op'] = "set" 756 loginfo['op'] = "set"
671 loginfo['flag'] = flag 757 loginfo['flag'] = flag
@@ -674,7 +760,7 @@ class DataSmart(MutableMapping):
674 self._makeShadowCopy(var) 760 self._makeShadowCopy(var)
675 self.dict[var][flag] = value 761 self.dict[var][flag] = value
676 762
677 if flag == "_defaultval" and '_' in var: 763 if flag == "_defaultval" and ':' in var:
678 self._setvar_update_overrides(var, **loginfo) 764 self._setvar_update_overrides(var, **loginfo)
679 if flag == "_defaultval" and var in self.overridevars: 765 if flag == "_defaultval" and var in self.overridevars:
680 self._setvar_update_overridevars(var, value) 766 self._setvar_update_overridevars(var, value)
@@ -695,22 +781,27 @@ class DataSmart(MutableMapping):
695 return None 781 return None
696 cachename = var + "[" + flag + "]" 782 cachename = var + "[" + flag + "]"
697 783
784 if not expand and retparser and cachename in self.expand_cache:
785 return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename]
786
698 if expand and cachename in self.expand_cache: 787 if expand and cachename in self.expand_cache:
699 return self.expand_cache[cachename].value 788 return self.expand_cache[cachename].value
700 789
701 local_var, overridedata = self._findVar(var) 790 local_var = self._findVar(var)
702 value = None 791 value = None
703 removes = set() 792 removes = set()
704 if flag == "_content" and overridedata is not None and not parsing: 793 if flag == "_content" and not parsing:
794 overridedata = self.overridedata.get(var, None)
795 if flag == "_content" and not parsing and overridedata is not None:
705 match = False 796 match = False
706 active = {} 797 active = {}
707 self.need_overrides() 798 self.need_overrides()
708 for (r, o) in overridedata: 799 for (r, o) in overridedata:
709 # What about double overrides both with "_" in the name? 800 # FIXME What about double overrides both with "_" in the name?
710 if o in self.overridesset: 801 if o in self.overridesset:
711 active[o] = r 802 active[o] = r
712 elif "_" in o: 803 elif ":" in o:
713 if set(o.split("_")).issubset(self.overridesset): 804 if set(o.split(":")).issubset(self.overridesset):
714 active[o] = r 805 active[o] = r
715 806
716 mod = True 807 mod = True
@@ -718,10 +809,10 @@ class DataSmart(MutableMapping):
718 mod = False 809 mod = False
719 for o in self.overrides: 810 for o in self.overrides:
720 for a in active.copy(): 811 for a in active.copy():
721 if a.endswith("_" + o): 812 if a.endswith(":" + o):
722 t = active[a] 813 t = active[a]
723 del active[a] 814 del active[a]
724 active[a.replace("_" + o, "")] = t 815 active[a.replace(":" + o, "")] = t
725 mod = True 816 mod = True
726 elif a == o: 817 elif a == o:
727 match = active[a] 818 match = active[a]
@@ -740,31 +831,31 @@ class DataSmart(MutableMapping):
740 value = copy.copy(local_var["_defaultval"]) 831 value = copy.copy(local_var["_defaultval"])
741 832
742 833
743 if flag == "_content" and local_var is not None and "_append" in local_var and not parsing: 834 if flag == "_content" and local_var is not None and ":append" in local_var and not parsing:
744 if not value:
745 value = ""
746 self.need_overrides() 835 self.need_overrides()
747 for (r, o) in local_var["_append"]: 836 for (r, o) in local_var[":append"]:
748 match = True 837 match = True
749 if o: 838 if o:
750 for o2 in o.split("_"): 839 for o2 in o.split(":"):
751 if not o2 in self.overrides: 840 if not o2 in self.overrides:
752 match = False 841 match = False
753 if match: 842 if match:
843 if value is None:
844 value = ""
754 value = value + r 845 value = value + r
755 846
756 if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing: 847 if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing:
757 if not value:
758 value = ""
759 self.need_overrides() 848 self.need_overrides()
760 for (r, o) in local_var["_prepend"]: 849 for (r, o) in local_var[":prepend"]:
761 850
762 match = True 851 match = True
763 if o: 852 if o:
764 for o2 in o.split("_"): 853 for o2 in o.split(":"):
765 if not o2 in self.overrides: 854 if not o2 in self.overrides:
766 match = False 855 match = False
767 if match: 856 if match:
857 if value is None:
858 value = ""
768 value = r + value 859 value = r + value
769 860
770 parser = None 861 parser = None
@@ -773,12 +864,12 @@ class DataSmart(MutableMapping):
773 if expand: 864 if expand:
774 value = parser.value 865 value = parser.value
775 866
776 if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing: 867 if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing:
777 self.need_overrides() 868 self.need_overrides()
778 for (r, o) in local_var["_remove"]: 869 for (r, o) in local_var[":remove"]:
779 match = True 870 match = True
780 if o: 871 if o:
781 for o2 in o.split("_"): 872 for o2 in o.split(":"):
782 if not o2 in self.overrides: 873 if not o2 in self.overrides:
783 match = False 874 match = False
784 if match: 875 if match:
@@ -791,7 +882,7 @@ class DataSmart(MutableMapping):
791 expanded_removes[r] = self.expand(r).split() 882 expanded_removes[r] = self.expand(r).split()
792 883
793 parser.removes = set() 884 parser.removes = set()
794 val = "" 885 val = []
795 for v in __whitespace_split__.split(parser.value): 886 for v in __whitespace_split__.split(parser.value):
796 skip = False 887 skip = False
797 for r in removes: 888 for r in removes:
@@ -800,8 +891,8 @@ class DataSmart(MutableMapping):
800 skip = True 891 skip = True
801 if skip: 892 if skip:
802 continue 893 continue
803 val = val + v 894 val.append(v)
804 parser.value = val 895 parser.value = "".join(val)
805 if expand: 896 if expand:
806 value = parser.value 897 value = parser.value
807 898
@@ -816,7 +907,7 @@ class DataSmart(MutableMapping):
816 def delVarFlag(self, var, flag, **loginfo): 907 def delVarFlag(self, var, flag, **loginfo):
817 self.expand_cache = {} 908 self.expand_cache = {}
818 909
819 local_var, _ = self._findVar(var) 910 local_var = self._findVar(var)
820 if not local_var: 911 if not local_var:
821 return 912 return
822 if not var in self.dict: 913 if not var in self.dict:
@@ -859,12 +950,12 @@ class DataSmart(MutableMapping):
859 self.dict[var][i] = flags[i] 950 self.dict[var][i] = flags[i]
860 951
861 def getVarFlags(self, var, expand = False, internalflags=False): 952 def getVarFlags(self, var, expand = False, internalflags=False):
862 local_var, _ = self._findVar(var) 953 local_var = self._findVar(var)
863 flags = {} 954 flags = {}
864 955
865 if local_var: 956 if local_var:
866 for i in local_var: 957 for i in local_var:
867 if i.startswith("_") and not internalflags: 958 if i.startswith(("_", ":")) and not internalflags:
868 continue 959 continue
869 flags[i] = local_var[i] 960 flags[i] = local_var[i]
870 if expand and i in expand: 961 if expand and i in expand:
@@ -905,6 +996,7 @@ class DataSmart(MutableMapping):
905 data.inchistory = self.inchistory.copy() 996 data.inchistory = self.inchistory.copy()
906 997
907 data._tracking = self._tracking 998 data._tracking = self._tracking
999 data._var_renames = self._var_renames
908 1000
909 data.overrides = None 1001 data.overrides = None
910 data.overridevars = copy.copy(self.overridevars) 1002 data.overridevars = copy.copy(self.overridevars)
@@ -927,7 +1019,7 @@ class DataSmart(MutableMapping):
927 value = self.getVar(variable, False) 1019 value = self.getVar(variable, False)
928 for key in keys: 1020 for key in keys:
929 referrervalue = self.getVar(key, False) 1021 referrervalue = self.getVar(key, False)
930 if referrervalue and ref in referrervalue: 1022 if referrervalue and isinstance(referrervalue, str) and ref in referrervalue:
931 self.setVar(key, referrervalue.replace(ref, value)) 1023 self.setVar(key, referrervalue.replace(ref, value))
932 1024
933 def localkeys(self): 1025 def localkeys(self):
@@ -962,8 +1054,8 @@ class DataSmart(MutableMapping):
962 for (r, o) in self.overridedata[var]: 1054 for (r, o) in self.overridedata[var]:
963 if o in self.overridesset: 1055 if o in self.overridesset:
964 overrides.add(var) 1056 overrides.add(var)
965 elif "_" in o: 1057 elif ":" in o:
966 if set(o.split("_")).issubset(self.overridesset): 1058 if set(o.split(":")).issubset(self.overridesset):
967 overrides.add(var) 1059 overrides.add(var)
968 1060
969 for k in keylist(self.dict): 1061 for k in keylist(self.dict):
@@ -993,10 +1085,10 @@ class DataSmart(MutableMapping):
993 d = self.createCopy() 1085 d = self.createCopy()
994 bb.data.expandKeys(d) 1086 bb.data.expandKeys(d)
995 1087
996 config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split()) 1088 config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split())
997 keys = set(key for key in iter(d) if not key.startswith("__")) 1089 keys = set(key for key in iter(d) if not key.startswith("__"))
998 for key in keys: 1090 for key in keys:
999 if key in config_whitelist: 1091 if key in config_ignore_vars:
1000 continue 1092 continue
1001 1093
1002 value = d.getVar(key, False) or "" 1094 value = d.getVar(key, False) or ""