diff options
Diffstat (limited to 'bitbake/lib/bb/data_smart.py')
-rw-r--r-- | bitbake/lib/bb/data_smart.py | 811 |
1 files changed, 811 insertions, 0 deletions
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py new file mode 100644 index 0000000000..d4bb98dd74 --- /dev/null +++ b/bitbake/lib/bb/data_smart.py | |||
@@ -0,0 +1,811 @@ | |||
1 | # ex:ts=4:sw=4:sts=4:et | ||
2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
3 | """ | ||
4 | BitBake Smart Dictionary Implementation | ||
5 | |||
6 | Functions for interacting with the data structure used by the | ||
7 | BitBake build tools. | ||
8 | |||
9 | """ | ||
10 | |||
11 | # Copyright (C) 2003, 2004 Chris Larson | ||
12 | # Copyright (C) 2004, 2005 Seb Frankengul | ||
13 | # Copyright (C) 2005, 2006 Holger Hans Peter Freyther | ||
14 | # Copyright (C) 2005 Uli Luckas | ||
15 | # Copyright (C) 2005 ROAD GmbH | ||
16 | # | ||
17 | # This program is free software; you can redistribute it and/or modify | ||
18 | # it under the terms of the GNU General Public License version 2 as | ||
19 | # published by the Free Software Foundation. | ||
20 | # | ||
21 | # This program is distributed in the hope that it will be useful, | ||
22 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
23 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
24 | # GNU General Public License for more details. | ||
25 | # | ||
26 | # You should have received a copy of the GNU General Public License along | ||
27 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
28 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
29 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
30 | |||
31 | import copy, re, sys, traceback | ||
32 | from collections import MutableMapping | ||
33 | import logging | ||
34 | import hashlib | ||
35 | import bb, bb.codeparser | ||
36 | from bb import utils | ||
37 | from bb.COW import COWDictBase | ||
38 | |||
39 | logger = logging.getLogger("BitBake.Data") | ||
40 | |||
41 | __setvar_keyword__ = ["_append", "_prepend", "_remove"] | ||
42 | __setvar_regexp__ = re.compile('(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>.*))?$') | ||
43 | __expand_var_regexp__ = re.compile(r"\${[^{}@\n\t ]+}") | ||
44 | __expand_python_regexp__ = re.compile(r"\${@.+?}") | ||
45 | |||
46 | def infer_caller_details(loginfo, parent = False, varval = True): | ||
47 | """Save the caller the trouble of specifying everything.""" | ||
48 | # Save effort. | ||
49 | if 'ignore' in loginfo and loginfo['ignore']: | ||
50 | return | ||
51 | # If nothing was provided, mark this as possibly unneeded. | ||
52 | if not loginfo: | ||
53 | loginfo['ignore'] = True | ||
54 | return | ||
55 | # Infer caller's likely values for variable (var) and value (value), | ||
56 | # to reduce clutter in the rest of the code. | ||
57 | if varval and ('variable' not in loginfo or 'detail' not in loginfo): | ||
58 | try: | ||
59 | raise Exception | ||
60 | except Exception: | ||
61 | tb = sys.exc_info()[2] | ||
62 | if parent: | ||
63 | above = tb.tb_frame.f_back.f_back | ||
64 | else: | ||
65 | above = tb.tb_frame.f_back | ||
66 | lcls = above.f_locals.items() | ||
67 | for k, v in lcls: | ||
68 | if k == 'value' and 'detail' not in loginfo: | ||
69 | loginfo['detail'] = v | ||
70 | if k == 'var' and 'variable' not in loginfo: | ||
71 | loginfo['variable'] = v | ||
72 | # Infer file/line/function from traceback | ||
73 | if 'file' not in loginfo: | ||
74 | depth = 3 | ||
75 | if parent: | ||
76 | depth = 4 | ||
77 | file, line, func, text = traceback.extract_stack(limit = depth)[0] | ||
78 | loginfo['file'] = file | ||
79 | loginfo['line'] = line | ||
80 | if func not in loginfo: | ||
81 | loginfo['func'] = func | ||
82 | |||
83 | class VariableParse: | ||
84 | def __init__(self, varname, d, val = None): | ||
85 | self.varname = varname | ||
86 | self.d = d | ||
87 | self.value = val | ||
88 | |||
89 | self.references = set() | ||
90 | self.execs = set() | ||
91 | self.contains = {} | ||
92 | |||
93 | def var_sub(self, match): | ||
94 | key = match.group()[2:-1] | ||
95 | if self.varname and key: | ||
96 | if self.varname == key: | ||
97 | raise Exception("variable %s references itself!" % self.varname) | ||
98 | if key in self.d.expand_cache: | ||
99 | varparse = self.d.expand_cache[key] | ||
100 | var = varparse.value | ||
101 | else: | ||
102 | var = self.d.getVarFlag(key, "_content", True) | ||
103 | self.references.add(key) | ||
104 | if var is not None: | ||
105 | return var | ||
106 | else: | ||
107 | return match.group() | ||
108 | |||
109 | def python_sub(self, match): | ||
110 | code = match.group()[3:-1] | ||
111 | codeobj = compile(code.strip(), self.varname or "<expansion>", "eval") | ||
112 | |||
113 | parser = bb.codeparser.PythonParser(self.varname, logger) | ||
114 | parser.parse_python(code) | ||
115 | if self.varname: | ||
116 | vardeps = self.d.getVarFlag(self.varname, "vardeps", True) | ||
117 | if vardeps is None: | ||
118 | parser.log.flush() | ||
119 | else: | ||
120 | parser.log.flush() | ||
121 | self.references |= parser.references | ||
122 | self.execs |= parser.execs | ||
123 | |||
124 | for k in parser.contains: | ||
125 | if k not in self.contains: | ||
126 | self.contains[k] = parser.contains[k].copy() | ||
127 | else: | ||
128 | self.contains[k].update(parser.contains[k]) | ||
129 | value = utils.better_eval(codeobj, DataContext(self.d)) | ||
130 | return str(value) | ||
131 | |||
132 | |||
133 | class DataContext(dict): | ||
134 | def __init__(self, metadata, **kwargs): | ||
135 | self.metadata = metadata | ||
136 | dict.__init__(self, **kwargs) | ||
137 | self['d'] = metadata | ||
138 | |||
139 | def __missing__(self, key): | ||
140 | value = self.metadata.getVar(key, True) | ||
141 | if value is None or self.metadata.getVarFlag(key, 'func'): | ||
142 | raise KeyError(key) | ||
143 | else: | ||
144 | return value | ||
145 | |||
146 | class ExpansionError(Exception): | ||
147 | def __init__(self, varname, expression, exception): | ||
148 | self.expression = expression | ||
149 | self.variablename = varname | ||
150 | self.exception = exception | ||
151 | if varname: | ||
152 | if expression: | ||
153 | self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception) | ||
154 | else: | ||
155 | self.msg = "Failure expanding variable %s: %s: %s" % (varname, type(exception).__name__, exception) | ||
156 | else: | ||
157 | self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception) | ||
158 | Exception.__init__(self, self.msg) | ||
159 | self.args = (varname, expression, exception) | ||
160 | def __str__(self): | ||
161 | return self.msg | ||
162 | |||
163 | class IncludeHistory(object): | ||
164 | def __init__(self, parent = None, filename = '[TOP LEVEL]'): | ||
165 | self.parent = parent | ||
166 | self.filename = filename | ||
167 | self.children = [] | ||
168 | self.current = self | ||
169 | |||
170 | def copy(self): | ||
171 | new = IncludeHistory(self.parent, self.filename) | ||
172 | for c in self.children: | ||
173 | new.children.append(c) | ||
174 | return new | ||
175 | |||
176 | def include(self, filename): | ||
177 | newfile = IncludeHistory(self.current, filename) | ||
178 | self.current.children.append(newfile) | ||
179 | self.current = newfile | ||
180 | return self | ||
181 | |||
182 | def __enter__(self): | ||
183 | pass | ||
184 | |||
185 | def __exit__(self, a, b, c): | ||
186 | if self.current.parent: | ||
187 | self.current = self.current.parent | ||
188 | else: | ||
189 | bb.warn("Include log: Tried to finish '%s' at top level." % filename) | ||
190 | return False | ||
191 | |||
192 | def emit(self, o, level = 0): | ||
193 | """Emit an include history file, and its children.""" | ||
194 | if level: | ||
195 | spaces = " " * (level - 1) | ||
196 | o.write("# %s%s" % (spaces, self.filename)) | ||
197 | if len(self.children) > 0: | ||
198 | o.write(" includes:") | ||
199 | else: | ||
200 | o.write("#\n# INCLUDE HISTORY:\n#") | ||
201 | level = level + 1 | ||
202 | for child in self.children: | ||
203 | o.write("\n") | ||
204 | child.emit(o, level) | ||
205 | |||
206 | class VariableHistory(object): | ||
207 | def __init__(self, dataroot): | ||
208 | self.dataroot = dataroot | ||
209 | self.variables = COWDictBase.copy() | ||
210 | |||
211 | def copy(self): | ||
212 | new = VariableHistory(self.dataroot) | ||
213 | new.variables = self.variables.copy() | ||
214 | return new | ||
215 | |||
216 | def record(self, *kwonly, **loginfo): | ||
217 | if not self.dataroot._tracking: | ||
218 | return | ||
219 | if len(kwonly) > 0: | ||
220 | raise TypeError | ||
221 | infer_caller_details(loginfo, parent = True) | ||
222 | if 'ignore' in loginfo and loginfo['ignore']: | ||
223 | return | ||
224 | if 'op' not in loginfo or not loginfo['op']: | ||
225 | loginfo['op'] = 'set' | ||
226 | if 'detail' in loginfo: | ||
227 | loginfo['detail'] = str(loginfo['detail']) | ||
228 | if 'variable' not in loginfo or 'file' not in loginfo: | ||
229 | raise ValueError("record() missing variable or file.") | ||
230 | var = loginfo['variable'] | ||
231 | |||
232 | if var not in self.variables: | ||
233 | self.variables[var] = [] | ||
234 | self.variables[var].append(loginfo.copy()) | ||
235 | |||
236 | def variable(self, var): | ||
237 | if var in self.variables: | ||
238 | return self.variables[var] | ||
239 | else: | ||
240 | return [] | ||
241 | |||
242 | def emit(self, var, oval, val, o): | ||
243 | history = self.variable(var) | ||
244 | commentVal = re.sub('\n', '\n#', str(oval)) | ||
245 | if history: | ||
246 | if len(history) == 1: | ||
247 | o.write("#\n# $%s\n" % var) | ||
248 | else: | ||
249 | o.write("#\n# $%s [%d operations]\n" % (var, len(history))) | ||
250 | for event in history: | ||
251 | # o.write("# %s\n" % str(event)) | ||
252 | if 'func' in event: | ||
253 | # If we have a function listed, this is internal | ||
254 | # code, not an operation in a config file, and the | ||
255 | # full path is distracting. | ||
256 | event['file'] = re.sub('.*/', '', event['file']) | ||
257 | display_func = ' [%s]' % event['func'] | ||
258 | else: | ||
259 | display_func = '' | ||
260 | if 'flag' in event: | ||
261 | flag = '[%s] ' % (event['flag']) | ||
262 | else: | ||
263 | flag = '' | ||
264 | o.write("# %s %s:%s%s\n# %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n# ', event['detail']))) | ||
265 | if len(history) > 1: | ||
266 | o.write("# pre-expansion value:\n") | ||
267 | o.write('# "%s"\n' % (commentVal)) | ||
268 | else: | ||
269 | o.write("#\n# $%s\n# [no history recorded]\n#\n" % var) | ||
270 | o.write('# "%s"\n' % (commentVal)) | ||
271 | |||
272 | def get_variable_files(self, var): | ||
273 | """Get the files where operations are made on a variable""" | ||
274 | var_history = self.variable(var) | ||
275 | files = [] | ||
276 | for event in var_history: | ||
277 | files.append(event['file']) | ||
278 | return files | ||
279 | |||
280 | def get_variable_lines(self, var, f): | ||
281 | """Get the line where a operation is made on a variable in file f""" | ||
282 | var_history = self.variable(var) | ||
283 | lines = [] | ||
284 | for event in var_history: | ||
285 | if f== event['file']: | ||
286 | line = event['line'] | ||
287 | lines.append(line) | ||
288 | return lines | ||
289 | |||
290 | def del_var_history(self, var, f=None, line=None): | ||
291 | """If file f and line are not given, the entire history of var is deleted""" | ||
292 | if var in self.variables: | ||
293 | if f and line: | ||
294 | self.variables[var] = [ x for x in self.variables[var] if x['file']!=f and x['line']!=line] | ||
295 | else: | ||
296 | self.variables[var] = [] | ||
297 | |||
298 | class DataSmart(MutableMapping): | ||
299 | def __init__(self, special = COWDictBase.copy(), seen = COWDictBase.copy() ): | ||
300 | self.dict = {} | ||
301 | |||
302 | self.inchistory = IncludeHistory() | ||
303 | self.varhistory = VariableHistory(self) | ||
304 | self._tracking = False | ||
305 | |||
306 | # cookie monster tribute | ||
307 | self._special_values = special | ||
308 | self._seen_overrides = seen | ||
309 | |||
310 | self.expand_cache = {} | ||
311 | |||
312 | def enableTracking(self): | ||
313 | self._tracking = True | ||
314 | |||
315 | def disableTracking(self): | ||
316 | self._tracking = False | ||
317 | |||
318 | def expandWithRefs(self, s, varname): | ||
319 | |||
320 | if not isinstance(s, basestring): # sanity check | ||
321 | return VariableParse(varname, self, s) | ||
322 | |||
323 | if varname and varname in self.expand_cache: | ||
324 | return self.expand_cache[varname] | ||
325 | |||
326 | varparse = VariableParse(varname, self) | ||
327 | |||
328 | while s.find('${') != -1: | ||
329 | olds = s | ||
330 | try: | ||
331 | s = __expand_var_regexp__.sub(varparse.var_sub, s) | ||
332 | s = __expand_python_regexp__.sub(varparse.python_sub, s) | ||
333 | if s == olds: | ||
334 | break | ||
335 | except ExpansionError: | ||
336 | raise | ||
337 | except bb.parse.SkipRecipe: | ||
338 | raise | ||
339 | except Exception as exc: | ||
340 | raise ExpansionError(varname, s, exc) | ||
341 | |||
342 | varparse.value = s | ||
343 | |||
344 | if varname: | ||
345 | self.expand_cache[varname] = varparse | ||
346 | |||
347 | return varparse | ||
348 | |||
349 | def expand(self, s, varname = None): | ||
350 | return self.expandWithRefs(s, varname).value | ||
351 | |||
352 | |||
353 | def finalize(self, parent = False): | ||
354 | """Performs final steps upon the datastore, including application of overrides""" | ||
355 | |||
356 | overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] | ||
357 | finalize_caller = { | ||
358 | 'op': 'finalize', | ||
359 | } | ||
360 | infer_caller_details(finalize_caller, parent = parent, varval = False) | ||
361 | |||
362 | # | ||
363 | # Well let us see what breaks here. We used to iterate | ||
364 | # over each variable and apply the override and then | ||
365 | # do the line expanding. | ||
366 | # If we have bad luck - which we will have - the keys | ||
367 | # where in some order that is so important for this | ||
368 | # method which we don't have anymore. | ||
369 | # Anyway we will fix that and write test cases this | ||
370 | # time. | ||
371 | |||
372 | # | ||
373 | # First we apply all overrides | ||
374 | # Then we will handle _append and _prepend and store the _remove | ||
375 | # information for later. | ||
376 | # | ||
377 | |||
378 | # We only want to report finalization once per variable overridden. | ||
379 | finalizes_reported = {} | ||
380 | |||
381 | for o in overrides: | ||
382 | # calculate '_'+override | ||
383 | l = len(o) + 1 | ||
384 | |||
385 | # see if one should even try | ||
386 | if o not in self._seen_overrides: | ||
387 | continue | ||
388 | |||
389 | vars = self._seen_overrides[o].copy() | ||
390 | for var in vars: | ||
391 | name = var[:-l] | ||
392 | try: | ||
393 | # Report only once, even if multiple changes. | ||
394 | if name not in finalizes_reported: | ||
395 | finalizes_reported[name] = True | ||
396 | finalize_caller['variable'] = name | ||
397 | finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False)) | ||
398 | self.varhistory.record(**finalize_caller) | ||
399 | # Copy history of the override over. | ||
400 | for event in self.varhistory.variable(var): | ||
401 | loginfo = event.copy() | ||
402 | loginfo['variable'] = name | ||
403 | loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op']) | ||
404 | self.varhistory.record(**loginfo) | ||
405 | self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '') | ||
406 | self.delVar(var) | ||
407 | except Exception: | ||
408 | logger.info("Untracked delVar") | ||
409 | |||
410 | # now on to the appends and prepends, and stashing the removes | ||
411 | for op in __setvar_keyword__: | ||
412 | if op in self._special_values: | ||
413 | appends = self._special_values[op] or [] | ||
414 | for append in appends: | ||
415 | keep = [] | ||
416 | for (a, o) in self.getVarFlag(append, op) or []: | ||
417 | match = True | ||
418 | if o: | ||
419 | for o2 in o.split("_"): | ||
420 | if not o2 in overrides: | ||
421 | match = False | ||
422 | if not match: | ||
423 | keep.append((a ,o)) | ||
424 | continue | ||
425 | |||
426 | if op == "_append": | ||
427 | sval = self.getVar(append, False) or "" | ||
428 | sval += a | ||
429 | self.setVar(append, sval) | ||
430 | elif op == "_prepend": | ||
431 | sval = a + (self.getVar(append, False) or "") | ||
432 | self.setVar(append, sval) | ||
433 | elif op == "_remove": | ||
434 | removes = self.getVarFlag(append, "_removeactive", False) or [] | ||
435 | removes.extend(a.split()) | ||
436 | self.setVarFlag(append, "_removeactive", removes, ignore=True) | ||
437 | |||
438 | # We save overrides that may be applied at some later stage | ||
439 | if keep: | ||
440 | self.setVarFlag(append, op, keep, ignore=True) | ||
441 | else: | ||
442 | self.delVarFlag(append, op, ignore=True) | ||
443 | |||
444 | def initVar(self, var): | ||
445 | self.expand_cache = {} | ||
446 | if not var in self.dict: | ||
447 | self.dict[var] = {} | ||
448 | |||
449 | def _findVar(self, var): | ||
450 | dest = self.dict | ||
451 | while dest: | ||
452 | if var in dest: | ||
453 | return dest[var] | ||
454 | |||
455 | if "_data" not in dest: | ||
456 | break | ||
457 | dest = dest["_data"] | ||
458 | |||
459 | def _makeShadowCopy(self, var): | ||
460 | if var in self.dict: | ||
461 | return | ||
462 | |||
463 | local_var = self._findVar(var) | ||
464 | |||
465 | if local_var: | ||
466 | self.dict[var] = copy.copy(local_var) | ||
467 | else: | ||
468 | self.initVar(var) | ||
469 | |||
470 | |||
471 | def setVar(self, var, value, **loginfo): | ||
472 | #print("var=" + str(var) + " val=" + str(value)) | ||
473 | if 'op' not in loginfo: | ||
474 | loginfo['op'] = "set" | ||
475 | self.expand_cache = {} | ||
476 | match = __setvar_regexp__.match(var) | ||
477 | if match and match.group("keyword") in __setvar_keyword__: | ||
478 | base = match.group('base') | ||
479 | keyword = match.group("keyword") | ||
480 | override = match.group('add') | ||
481 | l = self.getVarFlag(base, keyword) or [] | ||
482 | l.append([value, override]) | ||
483 | self.setVarFlag(base, keyword, l, ignore=True) | ||
484 | # And cause that to be recorded: | ||
485 | loginfo['detail'] = value | ||
486 | loginfo['variable'] = base | ||
487 | if override: | ||
488 | loginfo['op'] = '%s[%s]' % (keyword, override) | ||
489 | else: | ||
490 | loginfo['op'] = keyword | ||
491 | self.varhistory.record(**loginfo) | ||
492 | # todo make sure keyword is not __doc__ or __module__ | ||
493 | # pay the cookie monster | ||
494 | try: | ||
495 | self._special_values[keyword].add(base) | ||
496 | except KeyError: | ||
497 | self._special_values[keyword] = set() | ||
498 | self._special_values[keyword].add(base) | ||
499 | |||
500 | return | ||
501 | |||
502 | if not var in self.dict: | ||
503 | self._makeShadowCopy(var) | ||
504 | |||
505 | # more cookies for the cookie monster | ||
506 | if '_' in var: | ||
507 | self._setvar_update_overrides(var) | ||
508 | |||
509 | # setting var | ||
510 | self.dict[var]["_content"] = value | ||
511 | self.varhistory.record(**loginfo) | ||
512 | |||
513 | def _setvar_update_overrides(self, var): | ||
514 | # aka pay the cookie monster | ||
515 | override = var[var.rfind('_')+1:] | ||
516 | shortvar = var[:var.rfind('_')] | ||
517 | while override: | ||
518 | if override not in self._seen_overrides: | ||
519 | self._seen_overrides[override] = set() | ||
520 | self._seen_overrides[override].add( var ) | ||
521 | override = None | ||
522 | if "_" in shortvar: | ||
523 | override = var[shortvar.rfind('_')+1:] | ||
524 | shortvar = var[:shortvar.rfind('_')] | ||
525 | |||
526 | def getVar(self, var, expand=False, noweakdefault=False): | ||
527 | return self.getVarFlag(var, "_content", expand, noweakdefault) | ||
528 | |||
529 | def renameVar(self, key, newkey, **loginfo): | ||
530 | """ | ||
531 | Rename the variable key to newkey | ||
532 | """ | ||
533 | val = self.getVar(key, 0) | ||
534 | if val is not None: | ||
535 | loginfo['variable'] = newkey | ||
536 | loginfo['op'] = 'rename from %s' % key | ||
537 | loginfo['detail'] = val | ||
538 | self.varhistory.record(**loginfo) | ||
539 | self.setVar(newkey, val, ignore=True) | ||
540 | |||
541 | for i in (__setvar_keyword__): | ||
542 | src = self.getVarFlag(key, i) | ||
543 | if src is None: | ||
544 | continue | ||
545 | |||
546 | dest = self.getVarFlag(newkey, i) or [] | ||
547 | dest.extend(src) | ||
548 | self.setVarFlag(newkey, i, dest, ignore=True) | ||
549 | |||
550 | if i in self._special_values and key in self._special_values[i]: | ||
551 | self._special_values[i].remove(key) | ||
552 | self._special_values[i].add(newkey) | ||
553 | |||
554 | loginfo['variable'] = key | ||
555 | loginfo['op'] = 'rename (to)' | ||
556 | loginfo['detail'] = newkey | ||
557 | self.varhistory.record(**loginfo) | ||
558 | self.delVar(key, ignore=True) | ||
559 | |||
560 | def appendVar(self, var, value, **loginfo): | ||
561 | loginfo['op'] = 'append' | ||
562 | self.varhistory.record(**loginfo) | ||
563 | newvalue = (self.getVar(var, False) or "") + value | ||
564 | self.setVar(var, newvalue, ignore=True) | ||
565 | |||
566 | def prependVar(self, var, value, **loginfo): | ||
567 | loginfo['op'] = 'prepend' | ||
568 | self.varhistory.record(**loginfo) | ||
569 | newvalue = value + (self.getVar(var, False) or "") | ||
570 | self.setVar(var, newvalue, ignore=True) | ||
571 | |||
572 | def delVar(self, var, **loginfo): | ||
573 | loginfo['detail'] = "" | ||
574 | loginfo['op'] = 'del' | ||
575 | self.varhistory.record(**loginfo) | ||
576 | self.expand_cache = {} | ||
577 | self.dict[var] = {} | ||
578 | if '_' in var: | ||
579 | override = var[var.rfind('_')+1:] | ||
580 | if override and override in self._seen_overrides and var in self._seen_overrides[override]: | ||
581 | self._seen_overrides[override].remove(var) | ||
582 | |||
583 | def setVarFlag(self, var, flag, value, **loginfo): | ||
584 | if 'op' not in loginfo: | ||
585 | loginfo['op'] = "set" | ||
586 | loginfo['flag'] = flag | ||
587 | self.varhistory.record(**loginfo) | ||
588 | if not var in self.dict: | ||
589 | self._makeShadowCopy(var) | ||
590 | self.dict[var][flag] = value | ||
591 | |||
592 | if flag == "defaultval" and '_' in var: | ||
593 | self._setvar_update_overrides(var) | ||
594 | |||
595 | if flag == "unexport" or flag == "export": | ||
596 | if not "__exportlist" in self.dict: | ||
597 | self._makeShadowCopy("__exportlist") | ||
598 | if not "_content" in self.dict["__exportlist"]: | ||
599 | self.dict["__exportlist"]["_content"] = set() | ||
600 | self.dict["__exportlist"]["_content"].add(var) | ||
601 | |||
602 | def getVarFlag(self, var, flag, expand=False, noweakdefault=False): | ||
603 | local_var = self._findVar(var) | ||
604 | value = None | ||
605 | if local_var is not None: | ||
606 | if flag in local_var: | ||
607 | value = copy.copy(local_var[flag]) | ||
608 | elif flag == "_content" and "defaultval" in local_var and not noweakdefault: | ||
609 | value = copy.copy(local_var["defaultval"]) | ||
610 | if expand and value: | ||
611 | # Only getvar (flag == _content) hits the expand cache | ||
612 | cachename = None | ||
613 | if flag == "_content": | ||
614 | cachename = var | ||
615 | else: | ||
616 | cachename = var + "[" + flag + "]" | ||
617 | value = self.expand(value, cachename) | ||
618 | if value and flag == "_content" and local_var is not None and "_removeactive" in local_var: | ||
619 | removes = [self.expand(r).split() for r in local_var["_removeactive"]] | ||
620 | removes = reduce(lambda a, b: a+b, removes, []) | ||
621 | filtered = filter(lambda v: v not in removes, | ||
622 | value.split()) | ||
623 | value = " ".join(filtered) | ||
624 | if expand: | ||
625 | # We need to ensure the expand cache has the correct value | ||
626 | # flag == "_content" here | ||
627 | self.expand_cache[var].value = value | ||
628 | return value | ||
629 | |||
630 | def delVarFlag(self, var, flag, **loginfo): | ||
631 | local_var = self._findVar(var) | ||
632 | if not local_var: | ||
633 | return | ||
634 | if not var in self.dict: | ||
635 | self._makeShadowCopy(var) | ||
636 | |||
637 | if var in self.dict and flag in self.dict[var]: | ||
638 | loginfo['detail'] = "" | ||
639 | loginfo['op'] = 'delFlag' | ||
640 | loginfo['flag'] = flag | ||
641 | self.varhistory.record(**loginfo) | ||
642 | |||
643 | del self.dict[var][flag] | ||
644 | |||
645 | def appendVarFlag(self, var, flag, value, **loginfo): | ||
646 | loginfo['op'] = 'append' | ||
647 | loginfo['flag'] = flag | ||
648 | self.varhistory.record(**loginfo) | ||
649 | newvalue = (self.getVarFlag(var, flag, False) or "") + value | ||
650 | self.setVarFlag(var, flag, newvalue, ignore=True) | ||
651 | |||
652 | def prependVarFlag(self, var, flag, value, **loginfo): | ||
653 | loginfo['op'] = 'prepend' | ||
654 | loginfo['flag'] = flag | ||
655 | self.varhistory.record(**loginfo) | ||
656 | newvalue = value + (self.getVarFlag(var, flag, False) or "") | ||
657 | self.setVarFlag(var, flag, newvalue, ignore=True) | ||
658 | |||
659 | def setVarFlags(self, var, flags, **loginfo): | ||
660 | infer_caller_details(loginfo) | ||
661 | if not var in self.dict: | ||
662 | self._makeShadowCopy(var) | ||
663 | |||
664 | for i in flags: | ||
665 | if i == "_content": | ||
666 | continue | ||
667 | loginfo['flag'] = i | ||
668 | loginfo['detail'] = flags[i] | ||
669 | self.varhistory.record(**loginfo) | ||
670 | self.dict[var][i] = flags[i] | ||
671 | |||
672 | def getVarFlags(self, var, expand = False, internalflags=False): | ||
673 | local_var = self._findVar(var) | ||
674 | flags = {} | ||
675 | |||
676 | if local_var: | ||
677 | for i in local_var: | ||
678 | if i.startswith("_") and not internalflags: | ||
679 | continue | ||
680 | flags[i] = local_var[i] | ||
681 | if expand and i in expand: | ||
682 | flags[i] = self.expand(flags[i], var + "[" + i + "]") | ||
683 | if len(flags) == 0: | ||
684 | return None | ||
685 | return flags | ||
686 | |||
687 | |||
688 | def delVarFlags(self, var, **loginfo): | ||
689 | if not var in self.dict: | ||
690 | self._makeShadowCopy(var) | ||
691 | |||
692 | if var in self.dict: | ||
693 | content = None | ||
694 | |||
695 | loginfo['op'] = 'delete flags' | ||
696 | self.varhistory.record(**loginfo) | ||
697 | |||
698 | # try to save the content | ||
699 | if "_content" in self.dict[var]: | ||
700 | content = self.dict[var]["_content"] | ||
701 | self.dict[var] = {} | ||
702 | self.dict[var]["_content"] = content | ||
703 | else: | ||
704 | del self.dict[var] | ||
705 | |||
706 | |||
707 | def createCopy(self): | ||
708 | """ | ||
709 | Create a copy of self by setting _data to self | ||
710 | """ | ||
711 | # we really want this to be a DataSmart... | ||
712 | data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) | ||
713 | data.dict["_data"] = self.dict | ||
714 | data.varhistory = self.varhistory.copy() | ||
715 | data.varhistory.datasmart = data | ||
716 | data.inchistory = self.inchistory.copy() | ||
717 | |||
718 | data._tracking = self._tracking | ||
719 | |||
720 | return data | ||
721 | |||
722 | def expandVarref(self, variable, parents=False): | ||
723 | """Find all references to variable in the data and expand it | ||
724 | in place, optionally descending to parent datastores.""" | ||
725 | |||
726 | if parents: | ||
727 | keys = iter(self) | ||
728 | else: | ||
729 | keys = self.localkeys() | ||
730 | |||
731 | ref = '${%s}' % variable | ||
732 | value = self.getVar(variable, False) | ||
733 | for key in keys: | ||
734 | referrervalue = self.getVar(key, False) | ||
735 | if referrervalue and ref in referrervalue: | ||
736 | self.setVar(key, referrervalue.replace(ref, value)) | ||
737 | |||
738 | def localkeys(self): | ||
739 | for key in self.dict: | ||
740 | if key != '_data': | ||
741 | yield key | ||
742 | |||
743 | def __iter__(self): | ||
744 | def keylist(d): | ||
745 | klist = set() | ||
746 | for key in d: | ||
747 | if key == "_data": | ||
748 | continue | ||
749 | if not d[key]: | ||
750 | continue | ||
751 | klist.add(key) | ||
752 | |||
753 | if "_data" in d: | ||
754 | klist |= keylist(d["_data"]) | ||
755 | |||
756 | return klist | ||
757 | |||
758 | for k in keylist(self.dict): | ||
759 | yield k | ||
760 | |||
761 | def __len__(self): | ||
762 | return len(frozenset(self)) | ||
763 | |||
764 | def __getitem__(self, item): | ||
765 | value = self.getVar(item, False) | ||
766 | if value is None: | ||
767 | raise KeyError(item) | ||
768 | else: | ||
769 | return value | ||
770 | |||
771 | def __setitem__(self, var, value): | ||
772 | self.setVar(var, value) | ||
773 | |||
774 | def __delitem__(self, var): | ||
775 | self.delVar(var) | ||
776 | |||
777 | def get_hash(self): | ||
778 | data = {} | ||
779 | d = self.createCopy() | ||
780 | bb.data.expandKeys(d) | ||
781 | bb.data.update_data(d) | ||
782 | |||
783 | config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST", True) or "").split()) | ||
784 | keys = set(key for key in iter(d) if not key.startswith("__")) | ||
785 | for key in keys: | ||
786 | if key in config_whitelist: | ||
787 | continue | ||
788 | |||
789 | value = d.getVar(key, False) or "" | ||
790 | data.update({key:value}) | ||
791 | |||
792 | varflags = d.getVarFlags(key, internalflags = True) | ||
793 | if not varflags: | ||
794 | continue | ||
795 | for f in varflags: | ||
796 | if f == "_content": | ||
797 | continue | ||
798 | data.update({'%s[%s]' % (key, f):varflags[f]}) | ||
799 | |||
800 | for key in ["__BBTASKS", "__BBANONFUNCS", "__BBHANDLERS"]: | ||
801 | bb_list = d.getVar(key, False) or [] | ||
802 | bb_list.sort() | ||
803 | data.update({key:str(bb_list)}) | ||
804 | |||
805 | if key == "__BBANONFUNCS": | ||
806 | for i in bb_list: | ||
807 | value = d.getVar(i, True) or "" | ||
808 | data.update({i:value}) | ||
809 | |||
810 | data_str = str([(k, data[k]) for k in sorted(data.keys())]) | ||
811 | return hashlib.md5(data_str).hexdigest() | ||