summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/data.py
diff options
context:
space:
mode:
authorRichard Purdie <richard@openedhand.com>2006-05-09 15:44:08 +0000
committerRichard Purdie <richard@openedhand.com>2006-05-09 15:44:08 +0000
commit27dba1e6247ae48349aee1bce141a9eefaafaad1 (patch)
tree822235005ccbd2707f7874ad680dedc4df36760c /bitbake/lib/bb/data.py
parented234aca98d0867c7b32801fc63820b19cf67df9 (diff)
downloadpoky-27dba1e6247ae48349aee1bce141a9eefaafaad1.tar.gz
Update to bitbake 1.4.2 (latest stable branch release). This includes the caching speedups
git-svn-id: https://svn.o-hand.com/repos/poky/trunk@371 311d38ba-8fff-0310-9ca6-ca027cbcb966
Diffstat (limited to 'bitbake/lib/bb/data.py')
-rw-r--r--bitbake/lib/bb/data.py273
1 files changed, 103 insertions, 170 deletions
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index 56ee977f66..55d1cc9053 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -7,6 +7,18 @@ BitBake 'Data' implementations
7Functions for interacting with the data structure used by the 7Functions for interacting with the data structure used by the
8BitBake build tools. 8BitBake build tools.
9 9
10The expandData and update_data are the most expensive
11operations. At night the cookie monster came by and
12suggested 'give me cookies on setting the variables and
13things will work out'. Taking this suggestion into account
14applying the skills from the not yet passed 'Entwurf und
15Analyse von Algorithmen' lecture and the cookie
16monster seems to be right. We will track setVar more carefully
17to have faster update_data and expandKeys operations.
18
19This is a treade-off between speed and memory again but
20the speed is more critical here.
21
10Copyright (C) 2003, 2004 Chris Larson 22Copyright (C) 2003, 2004 Chris Larson
11Copyright (C) 2005 Holger Hans Peter Freyther 23Copyright (C) 2005 Holger Hans Peter Freyther
12 24
@@ -36,88 +48,15 @@ sys.path.insert(0,path)
36from bb import note, debug, data_smart 48from bb import note, debug, data_smart
37 49
38_dict_type = data_smart.DataSmart 50_dict_type = data_smart.DataSmart
39_dict_p_type = data_smart.DataSmartPackage
40
41class DataDictFull(dict):
42 """
43 This implements our Package Data Storage Interface.
44 setDirty is a no op as all items are held in memory
45 """
46 def setDirty(self, bbfile, data):
47 """
48 No-Op we assume data was manipulated as some sort of
49 reference
50 """
51 if not bbfile in self:
52 raise Exception("File %s was not in dictionary before" % bbfile)
53
54 self[bbfile] = data
55
56class DataDictCache:
57 """
58 Databacked Dictionary implementation
59 """
60 def __init__(self, cache_dir, config):
61 self.cache_dir = cache_dir
62 self.files = []
63 self.dirty = {}
64 self.config = config
65
66 def has_key(self,key):
67 return key in self.files
68
69 def keys(self):
70 return self.files
71
72 def __setitem__(self, key, data):
73 """
74 Add the key to the list of known files and
75 place the data in the cache?
76 """
77 if key in self.files:
78 return
79
80 self.files.append(key)
81
82 def __getitem__(self, key):
83 if not key in self.files:
84 return None
85
86 # if it was dirty we will
87 if key in self.dirty:
88 return self.dirty[key]
89
90 # not cached yet
91 return _dict_p_type(self.cache_dir, key,False,self.config)
92
93 def setDirty(self, bbfile, data):
94 """
95 Only already added items can be declared dirty!!!
96 """
97
98 if not bbfile in self.files:
99 raise Exception("File %s was not in dictionary before" % bbfile)
100
101 self.dirty[bbfile] = data
102
103
104 51
105def init(): 52def init():
106 return _dict_type() 53 return _dict_type()
107 54
108def init_db(cache,name,clean,parent = None): 55def init_db(parent = None):
109 return _dict_p_type(cache,name,clean,parent) 56 if parent:
110 57 return parent.createCopy()
111def init_db_mtime(cache,cache_bbfile): 58 else:
112 return _dict_p_type.mtime(cache,cache_bbfile) 59 return _dict_type()
113
114def pkgdata(use_cache, cache, config = None):
115 """
116 Return some sort of dictionary to lookup parsed dictionaires
117 """
118 if use_cache:
119 return DataDictCache(cache, config)
120 return DataDictFull()
121 60
122def createCopy(source): 61def createCopy(source):
123 """Link the source set to the destination 62 """Link the source set to the destination
@@ -273,6 +212,27 @@ def setData(newData, d):
273 """Sets the data object to the supplied value""" 212 """Sets the data object to the supplied value"""
274 d = newData 213 d = newData
275 214
215
216##
217## Cookie Monsters' query functions
218##
219def _get_override_vars(d, override):
220 """
221 Internal!!!
222
223 Get the Names of Variables that have a specific
224 override. This function returns a iterable
225 Set or an empty list
226 """
227 return []
228
229def _get_var_flags_triple(d):
230 """
231 Internal!!!
232
233 """
234 return []
235
276__expand_var_regexp__ = re.compile(r"\${[^{}]+}") 236__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
277__expand_python_regexp__ = re.compile(r"\${@.+?}") 237__expand_python_regexp__ = re.compile(r"\${@.+?}")
278 238
@@ -303,43 +263,7 @@ def expand(s, d, varname = None):
303 >>> print expand('${SRC_URI}', d) 263 >>> print expand('${SRC_URI}', d)
304 http://somebug.${TARGET_MOO} 264 http://somebug.${TARGET_MOO}
305 """ 265 """
306 def var_sub(match): 266 return d.expand(s, varname)
307 key = match.group()[2:-1]
308 if varname and key:
309 if varname == key:
310 raise Exception("variable %s references itself!" % varname)
311 var = getVar(key, d, 1)
312 if var is not None:
313 return var
314 else:
315 return match.group()
316
317 def python_sub(match):
318 import bb
319 code = match.group()[3:-1]
320 locals()['d'] = d
321 s = eval(code)
322 if type(s) == types.IntType: s = str(s)
323 return s
324
325 if type(s) is not types.StringType: # sanity check
326 return s
327
328 while s.find('$') != -1:
329 olds = s
330 try:
331 s = __expand_var_regexp__.sub(var_sub, s)
332 s = __expand_python_regexp__.sub(python_sub, s)
333 if s == olds: break
334 if type(s) is not types.StringType: # sanity check
335 import bb
336 bb.error('expansion of %s returned non-string %s' % (olds, s))
337 except KeyboardInterrupt:
338 raise
339 except:
340 note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s))
341 raise
342 return s
343 267
344def expandKeys(alterdata, readdata = None): 268def expandKeys(alterdata, readdata = None):
345 if readdata == None: 269 if readdata == None:
@@ -356,7 +280,7 @@ def expandKeys(alterdata, readdata = None):
356# setVarFlags(ekey, copy.copy(getVarFlags(key, readdata)), alterdata) 280# setVarFlags(ekey, copy.copy(getVarFlags(key, readdata)), alterdata)
357 setVar(ekey, val, alterdata) 281 setVar(ekey, val, alterdata)
358 282
359 for i in ('_append', '_prepend', '_delete'): 283 for i in ('_append', '_prepend'):
360 dest = getVarFlag(ekey, i, alterdata) or [] 284 dest = getVarFlag(ekey, i, alterdata) or []
361 src = getVarFlag(key, i, readdata) or [] 285 src = getVarFlag(key, i, readdata) or []
362 dest.extend(src) 286 dest.extend(src)
@@ -507,67 +431,76 @@ def update_data(d):
507 >>> print getVar('TEST', d) 431 >>> print getVar('TEST', d)
508 local 432 local
509 """ 433 """
510
511 debug(2, "update_data()") 434 debug(2, "update_data()")
512 435
513# can't do delete env[...] while iterating over the dictionary, so remember them 436 # now ask the cookie monster for help
514 dodel = [] 437 #print "Cookie Monster"
438 #print "Append/Prepend %s" % d._special_values
439 #print "Overrides %s" % d._seen_overrides
440
515 overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or [] 441 overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or []
516 442
517 def applyOverrides(var, d): 443 #
518 if not overrides: 444 # Well let us see what breaks here. We used to iterate
519 debug(1, "OVERRIDES not defined, nothing to do") 445 # over each variable and apply the override and then
520 return 446 # do the line expanding.
521 val = getVar(var, d) 447 # If we have bad luck - which we will have - the keys
522 for o in overrides: 448 # where in some order that is so important for this
523 if var.endswith("_" + o): 449 # method which we don't have anymore.
524 l = len(o)+1 450 # Anyway we will fix that and write test cases this
525 name = var[:-l] 451 # time.
526 d[name] = d[var] 452
453 #
454 # First we apply all overrides
455 # Then we will handle _append and _prepend
456 #
457
458 for o in overrides:
459 # calculate '_'+override
460 l = len(o)+1
461
462 # see if one should even try
463 if not o in d._seen_overrides:
464 continue
527 465
528 for s in keys(d): 466 vars = d._seen_overrides[o]
529 applyOverrides(s, d) 467 for var in vars:
530 sval = getVar(s, d) or "" 468 name = var[:-l]
531 469 try:
532# Handle line appends: 470 d[name] = d[var]
533 for (a, o) in getVarFlag(s, '_append', d) or []: 471 except:
534 # maybe the OVERRIDE was not yet added so keep the append 472 note ("Untracked delVar")
535 if (o and o in overrides) or not o: 473
536 delVarFlag(s, '_append', d) 474 # now on to the appends and prepends
537 if o: 475 if '_append' in d._special_values:
538 if not o in overrides: 476 appends = d._special_values['_append'] or []
477 for append in appends:
478 for (a, o) in getVarFlag(append, '_append', d) or []:
479 # maybe the OVERRIDE was not yet added so keep the append
480 if (o and o in overrides) or not o:
481 delVarFlag(append, '_append', d)
482 if o and not o in overrides:
539 continue 483 continue
540 sval+=a 484
541 setVar(s, sval, d) 485 sval = getVar(append,d) or ""
542 486 sval+=a
543# Handle line prepends 487 setVar(append, sval, d)
544 for (a, o) in getVarFlag(s, '_prepend', d) or []: 488
545 # maybe the OVERRIDE was not yet added so keep the append 489
546 if (o and o in overrides) or not o: 490 if '_prepend' in d._special_values:
547 delVarFlag(s, '_prepend', d) 491 prepends = d._special_values['_prepend'] or []
548 if o: 492
549 if not o in overrides: 493 for prepend in prepends:
494 for (a, o) in getVarFlag(prepend, '_prepend', d) or []:
495 # maybe the OVERRIDE was not yet added so keep the prepend
496 if (o and o in overrides) or not o:
497 delVarFlag(prepend, '_prepend', d)
498 if o and not o in overrides:
550 continue 499 continue
551 sval=a+sval 500
552 setVar(s, sval, d) 501 sval = a + (getVar(prepend,d) or "")
553 502 setVar(prepend, sval, d)
554# Handle line deletions 503
555 name = s + "_delete"
556 nameval = getVar(name, d)
557 if nameval:
558 sval = getVar(s, d)
559 if sval:
560 new = ''
561 pattern = nameval.replace('\n','').strip()
562 for line in sval.split('\n'):
563 if line.find(pattern) == -1:
564 new = new + '\n' + line
565 setVar(s, new, d)
566 dodel.append(name)
567
568# delete all environment vars no longer needed
569 for s in dodel:
570 delVar(s, d)
571 504
572def inherits_class(klass, d): 505def inherits_class(klass, d):
573 val = getVar('__inherit_cache', d) or "" 506 val = getVar('__inherit_cache', d) or ""