summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2015-01-23 23:36:29 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-07-12 22:50:45 +0100
commit577f16040013bc8fe9cea81cd8b40905b4fbccb7 (patch)
tree6cf63cc7cd1f3b993817a50f658404a756c92ef7
parente7ccd9071233d66afb0bc72774b0032fb8229fe4 (diff)
downloadpoky-577f16040013bc8fe9cea81cd8b40905b4fbccb7.tar.gz
bitbake: data_smart: Improve override handling
Rather than the heavy lifting in internal_finalize, move the bulk of the functionality to getVar and rely on a new internal replaces variable to keep track of any mappings that are needed. This removes the need for the COW _special_values cache. This change in functionality also implies we need to track any changes not only to OVERRIDES but also any variable which OVERIDES depends upon. Add code to handle this. Explicitly list FILE as a value OVERRIDES depends upon since it doesn't automatically get detected and is of key importance to OVERRIDES, otherwise PN doesn't update when FILE changes. (Bitbake rev: a6f1377ce3386d274882072d1ae6da3b1834149b) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/data_smart.py156
1 files changed, 60 insertions, 96 deletions
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 77d0c61da0..aefc0b980b 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -296,12 +296,9 @@ class VariableHistory(object):
296 self.variables[var] = [] 296 self.variables[var] = []
297 297
298class DataSmart(MutableMapping): 298class DataSmart(MutableMapping):
299 def __init__(self, special = None, seen = None ): 299 def __init__(self, seen = None):
300 self.dict = {} 300 self.dict = {}
301 self.overrides = []
302 301
303 if special is None:
304 special = COWDictBase.copy()
305 if seen is None: 302 if seen is None:
306 seen = COWDictBase.copy() 303 seen = COWDictBase.copy()
307 304
@@ -310,11 +307,13 @@ class DataSmart(MutableMapping):
310 self._tracking = False 307 self._tracking = False
311 308
312 # cookie monster tribute 309 # cookie monster tribute
313 self._special_values = special
314 self._seen_overrides = seen 310 self._seen_overrides = seen
315 311
316 self.expand_cache = {} 312 self.expand_cache = {}
317 313
314 self.overridevars = set(["OVERRIDES", "FILE"])
315 self.replaces = {}
316
318 def enableTracking(self): 317 def enableTracking(self):
319 self._tracking = True 318 self._tracking = True
320 319
@@ -361,11 +360,7 @@ class DataSmart(MutableMapping):
361 def internal_finalize(self, parent = False): 360 def internal_finalize(self, parent = False):
362 """Performs final steps upon the datastore, including application of overrides""" 361 """Performs final steps upon the datastore, including application of overrides"""
363 362
364 self.overrides = (self.getVar("OVERRIDES", True) or "").split(":") or [] 363 overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
365 finalize_caller = {
366 'op': 'finalize',
367 }
368 infer_caller_details(finalize_caller, parent = parent, varval = False)
369 364
370 # 365 #
371 # Well let us see what breaks here. We used to iterate 366 # Well let us see what breaks here. We used to iterate
@@ -383,10 +378,9 @@ class DataSmart(MutableMapping):
383 # information for later. 378 # information for later.
384 # 379 #
385 380
386 # We only want to report finalization once per variable overridden. 381 self.replaces = {}
387 finalizes_reported = {}
388 382
389 for o in self.overrides: 383 for o in overrides:
390 # calculate '_'+override 384 # calculate '_'+override
391 l = len(o) + 1 385 l = len(o) + 1
392 386
@@ -397,61 +391,7 @@ class DataSmart(MutableMapping):
397 vars = self._seen_overrides[o].copy() 391 vars = self._seen_overrides[o].copy()
398 for var in vars: 392 for var in vars:
399 name = var[:-l] 393 name = var[:-l]
400 try: 394 self.replaces[name] = var
401 # Report only once, even if multiple changes.
402 if name not in finalizes_reported:
403 finalizes_reported[name] = True
404 finalize_caller['variable'] = name
405 finalize_caller['detail'] = 'was: ' + str(self.getVar(name, False))
406 self.varhistory.record(**finalize_caller)
407 # Copy history of the override over.
408 for event in self.varhistory.variable(var):
409 loginfo = event.copy()
410 loginfo['variable'] = name
411 loginfo['op'] = 'override[%s]:%s' % (o, loginfo['op'])
412 self.varhistory.record(**loginfo)
413 self.setVar(name, self.getVar(var, False), op = 'finalize', file = 'override[%s]' % o, line = '')
414 self.delVar(var)
415 except Exception:
416 logger.info("Untracked delVar")
417
418 # now on to the appends and prepends, and stashing the removes
419 for op in __setvar_keyword__:
420 if op in self._special_values:
421 appends = self._special_values[op] or []
422 for append in appends:
423 self.handle_special_values(append, op)
424
425 def handle_special_values(self, append, op):
426 keep = []
427 for (a, o) in self.getVarFlag(append, op) or []:
428 match = True
429 if o:
430 for o2 in o.split("_"):
431 if not o2 in self.overrides:
432 match = False
433 if not match:
434 keep.append((a ,o))
435 continue
436
437 if op == "_append":
438 apps = self.getVarFlag(append, "_appendactive", False) or []
439 apps.extend([a])
440 self.setVarFlag(append, "_appendactive", apps, ignore=True)
441 elif op == "_prepend":
442 prepends = self.getVarFlag(append, "_prependactive", False) or []
443 prepends.extend([a])
444 self.setVarFlag(append, "_prependactive", prepends, ignore=True)
445 elif op == "_remove":
446 removes = self.getVarFlag(append, "_removeactive", False) or []
447 removes.extend(a.split())
448 self.setVarFlag(append, "_removeactive", removes, ignore=True)
449
450 # We save overrides that may be applied at some later stage
451 if keep:
452 self.setVarFlag(append, op, keep, ignore=True)
453 else:
454 self.delVarFlag(append, op, ignore=True)
455 395
456 def initVar(self, var): 396 def initVar(self, var):
457 self.expand_cache = {} 397 self.expand_cache = {}
@@ -503,22 +443,19 @@ class DataSmart(MutableMapping):
503 self.varhistory.record(**loginfo) 443 self.varhistory.record(**loginfo)
504 # todo make sure keyword is not __doc__ or __module__ 444 # todo make sure keyword is not __doc__ or __module__
505 # pay the cookie monster 445 # pay the cookie monster
506 try:
507 self._special_values[keyword].add(base)
508 except KeyError:
509 self._special_values[keyword] = set()
510 self._special_values[keyword].add(base)
511 self.handle_special_values(base, keyword)
512 446
447 if base in self.overridevars:
448 self.overridevars.update(self.expandWithRefs(value, var).references)
449 self.internal_finalize(True)
513 return 450 return
514 451
515 if not var in self.dict: 452 if not var in self.dict:
516 self._makeShadowCopy(var) 453 self._makeShadowCopy(var)
517 454
518 if "_appendactive" in self.dict[var]: 455 if "_append" in self.dict[var]:
519 del self.dict[var]["_appendactive"] 456 del self.dict[var]["_append"]
520 if "_prependactive" in self.dict[var]: 457 if "_prepend" in self.dict[var]:
521 del self.dict[var]["_prependactive"] 458 del self.dict[var]["_prepend"]
522 459
523 # more cookies for the cookie monster 460 # more cookies for the cookie monster
524 if '_' in var: 461 if '_' in var:
@@ -528,7 +465,8 @@ class DataSmart(MutableMapping):
528 self.dict[var]["_content"] = value 465 self.dict[var]["_content"] = value
529 self.varhistory.record(**loginfo) 466 self.varhistory.record(**loginfo)
530 467
531 if var == "OVERRIDES": 468 if var in self.overridevars:
469 self.overridevars.update(self.expandWithRefs(value, var).references)
532 self.internal_finalize(True) 470 self.internal_finalize(True)
533 471
534 def _setvar_update_overrides(self, var): 472 def _setvar_update_overrides(self, var):
@@ -568,10 +506,6 @@ class DataSmart(MutableMapping):
568 dest.extend(src) 506 dest.extend(src)
569 self.setVarFlag(newkey, i, dest, ignore=True) 507 self.setVarFlag(newkey, i, dest, ignore=True)
570 508
571 if i in self._special_values and key in self._special_values[i]:
572 self._special_values[i].remove(key)
573 self._special_values[i].add(newkey)
574
575 loginfo['variable'] = key 509 loginfo['variable'] = key
576 loginfo['op'] = 'rename (to)' 510 loginfo['op'] = 'rename (to)'
577 loginfo['detail'] = newkey 511 loginfo['detail'] = newkey
@@ -623,22 +557,42 @@ class DataSmart(MutableMapping):
623 def getVarFlag(self, var, flag, expand=False, noweakdefault=False): 557 def getVarFlag(self, var, flag, expand=False, noweakdefault=False):
624 local_var = self._findVar(var) 558 local_var = self._findVar(var)
625 value = None 559 value = None
626 if local_var is not None: 560
561 if flag == "_content" and var in self.replaces:
562 value = self.getVar(self.replaces[var])
563
564 if local_var is not None and value is None:
627 if flag in local_var: 565 if flag in local_var:
628 value = copy.copy(local_var[flag]) 566 value = copy.copy(local_var[flag])
629 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault: 567 elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
630 value = copy.copy(local_var["_defaultval"]) 568 value = copy.copy(local_var["_defaultval"])
631 569
632 if flag == "_content" and local_var is not None and "_appendactive" in local_var: 570 if flag == "_content" and local_var is not None and "_append" in local_var:
633 if not value: 571 if not value:
634 value = "" 572 value = ""
635 for r in local_var["_appendactive"]: 573 for (r, o) in local_var["_append"]:
636 value = value + r 574 match = True
637 if flag == "_content" and local_var is not None and "_prependactive" in local_var: 575 if o:
576 overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
577 for o2 in o.split("_"):
578 if not o2 in overrides:
579 match = False
580 if match:
581 value = value + r
582
583 if flag == "_content" and local_var is not None and "_prepend" in local_var:
638 if not value: 584 if not value:
639 value = "" 585 value = ""
640 for r in local_var["_prependactive"]: 586 for (r, o) in local_var["_prepend"]:
641 value = r + value 587 match = True
588 if o:
589 overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
590 for o2 in o.split("_"):
591 if not o2 in overrides:
592 match = False
593 if match:
594 value = r + value
595
642 if expand and value: 596 if expand and value:
643 # Only getvar (flag == _content) hits the expand cache 597 # Only getvar (flag == _content) hits the expand cache
644 cachename = None 598 cachename = None
@@ -647,9 +601,19 @@ class DataSmart(MutableMapping):
647 else: 601 else:
648 cachename = var + "[" + flag + "]" 602 cachename = var + "[" + flag + "]"
649 value = self.expand(value, cachename) 603 value = self.expand(value, cachename)
650 if value and flag == "_content" and local_var is not None and "_removeactive" in local_var: 604
651 removes = [self.expand(r).split() for r in local_var["_removeactive"]] 605 if value and flag == "_content" and local_var is not None and "_remove" in local_var:
652 removes = reduce(lambda a, b: a+b, removes, []) 606 removes = []
607 for (r, o) in local_var["_remove"]:
608 match = True
609 if o:
610 overrides = (self.getVar("OVERRIDES", True) or "").split(":") or []
611 for o2 in o.split("_"):
612 if not o2 in overrides:
613 match = False
614 if match:
615 removes.extend(self.expand(r).split())
616
653 filtered = filter(lambda v: v not in removes, 617 filtered = filter(lambda v: v not in removes,
654 value.split()) 618 value.split())
655 value = " ".join(filtered) 619 value = " ".join(filtered)
@@ -735,21 +699,21 @@ class DataSmart(MutableMapping):
735 else: 699 else:
736 del self.dict[var] 700 del self.dict[var]
737 701
738
739 def createCopy(self): 702 def createCopy(self):
740 """ 703 """
741 Create a copy of self by setting _data to self 704 Create a copy of self by setting _data to self
742 """ 705 """
743 # we really want this to be a DataSmart... 706 # we really want this to be a DataSmart...
744 data = DataSmart(seen=self._seen_overrides.copy(), special=self._special_values.copy()) 707 data = DataSmart(seen=self._seen_overrides.copy())
745 data.dict["_data"] = self.dict 708 data.dict["_data"] = self.dict
746 data.overrides = copy.copy(self.overrides)
747 data.varhistory = self.varhistory.copy() 709 data.varhistory = self.varhistory.copy()
748 data.varhistory.datasmart = data 710 data.varhistory.datasmart = data
749 data.inchistory = self.inchistory.copy() 711 data.inchistory = self.inchistory.copy()
750 712
751 data._tracking = self._tracking 713 data._tracking = self._tracking
752 714
715 data.overridevars = copy.copy(self.overridevars)
716
753 return data 717 return data
754 718
755 def expandVarref(self, variable, parents=False): 719 def expandVarref(self, variable, parents=False):