diff options
Diffstat (limited to 'bitbake/lib/bb/data.py')
-rw-r--r-- | bitbake/lib/bb/data.py | 580 |
1 files changed, 580 insertions, 0 deletions
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py new file mode 100644 index 0000000000..b7d707a920 --- /dev/null +++ b/bitbake/lib/bb/data.py | |||
@@ -0,0 +1,580 @@ | |||
1 | #!/usr/bin/env python | ||
2 | # ex:ts=4:sw=4:sts=4:et | ||
3 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
4 | """ | ||
5 | BitBake 'Data' implementations | ||
6 | |||
7 | Functions for interacting with the data structure used by the | ||
8 | BitBake build tools. | ||
9 | |||
10 | Copyright (C) 2003, 2004 Chris Larson | ||
11 | Copyright (C) 2005 Holger Hans Peter Freyther | ||
12 | |||
13 | This program is free software; you can redistribute it and/or modify it under | ||
14 | the terms of the GNU General Public License as published by the Free Software | ||
15 | Foundation; either version 2 of the License, or (at your option) any later | ||
16 | version. | ||
17 | |||
18 | This program is distributed in the hope that it will be useful, but WITHOUT | ||
19 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | ||
20 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. | ||
21 | |||
22 | You should have received a copy of the GNU General Public License along with | ||
23 | this program; if not, write to the Free Software Foundation, Inc., 59 Temple | ||
24 | Place, Suite 330, Boston, MA 02111-1307 USA. | ||
25 | |||
26 | Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
27 | """ | ||
28 | |||
29 | import sys, os, re, time, types | ||
30 | if sys.argv[0][-5:] == "pydoc": | ||
31 | path = os.path.dirname(os.path.dirname(sys.argv[1])) | ||
32 | else: | ||
33 | path = os.path.dirname(os.path.dirname(sys.argv[0])) | ||
34 | sys.path.append(path) | ||
35 | |||
36 | from bb import note, debug, data_smart | ||
37 | |||
38 | _dict_type = data_smart.DataSmart | ||
39 | _dict_p_type = data_smart.DataSmartPackage | ||
40 | |||
41 | class DataDictFull(dict): | ||
42 | """ | ||
43 | This implements our Package Data Storage Interface. | ||
44 | setDirty is a no op as all items are held in memory | ||
45 | """ | ||
46 | def setDirty(self, bbfile, data): | ||
47 | """ | ||
48 | No-Op we assume data was manipulated as some sort of | ||
49 | reference | ||
50 | """ | ||
51 | if not bbfile in self: | ||
52 | raise Exception("File %s was not in dictionary before" % bbfile) | ||
53 | |||
54 | self[bbfile] = data | ||
55 | |||
56 | class DataDictCache: | ||
57 | """ | ||
58 | Databacked Dictionary implementation | ||
59 | """ | ||
60 | def __init__(self, cache_dir, config): | ||
61 | self.cache_dir = cache_dir | ||
62 | self.files = [] | ||
63 | self.dirty = {} | ||
64 | self.config = config | ||
65 | |||
66 | def has_key(self,key): | ||
67 | return key in self.files | ||
68 | |||
69 | def keys(self): | ||
70 | return self.files | ||
71 | |||
72 | def __setitem__(self, key, data): | ||
73 | """ | ||
74 | Add the key to the list of known files and | ||
75 | place the data in the cache? | ||
76 | """ | ||
77 | if key in self.files: | ||
78 | return | ||
79 | |||
80 | self.files.append(key) | ||
81 | |||
82 | def __getitem__(self, key): | ||
83 | if not key in self.files: | ||
84 | return None | ||
85 | |||
86 | # if it was dirty we will | ||
87 | if key in self.dirty: | ||
88 | return self.dirty[key] | ||
89 | |||
90 | # not cached yet | ||
91 | return _dict_p_type(self.cache_dir, key,False,self.config) | ||
92 | |||
93 | def setDirty(self, bbfile, data): | ||
94 | """ | ||
95 | Only already added items can be declared dirty!!! | ||
96 | """ | ||
97 | |||
98 | if not bbfile in self.files: | ||
99 | raise Exception("File %s was not in dictionary before" % bbfile) | ||
100 | |||
101 | self.dirty[bbfile] = data | ||
102 | |||
103 | |||
104 | |||
105 | def init(): | ||
106 | return _dict_type() | ||
107 | |||
108 | def init_db(cache,name,clean,parent = None): | ||
109 | return _dict_p_type(cache,name,clean,parent) | ||
110 | |||
111 | def init_db_mtime(cache,cache_bbfile): | ||
112 | return _dict_p_type.mtime(cache,cache_bbfile) | ||
113 | |||
114 | def pkgdata(use_cache, cache, config = None): | ||
115 | """ | ||
116 | Return some sort of dictionary to lookup parsed dictionaires | ||
117 | """ | ||
118 | if use_cache: | ||
119 | return DataDictCache(cache, config) | ||
120 | return DataDictFull() | ||
121 | |||
122 | def createCopy(source): | ||
123 | """Link the source set to the destination | ||
124 | If one does not find the value in the destination set, | ||
125 | search will go on to the source set to get the value. | ||
126 | Value from source are copy-on-write. i.e. any try to | ||
127 | modify one of them will end up putting the modified value | ||
128 | in the destination set. | ||
129 | """ | ||
130 | return source.createCopy() | ||
131 | |||
132 | def initVar(var, d): | ||
133 | """Non-destructive var init for data structure""" | ||
134 | d.initVar(var) | ||
135 | |||
136 | |||
137 | def setVar(var, value, d): | ||
138 | """Set a variable to a given value | ||
139 | |||
140 | Example: | ||
141 | >>> d = init() | ||
142 | >>> setVar('TEST', 'testcontents', d) | ||
143 | >>> print getVar('TEST', d) | ||
144 | testcontents | ||
145 | """ | ||
146 | d.setVar(var,value) | ||
147 | |||
148 | |||
149 | def getVar(var, d, exp = 0): | ||
150 | """Gets the value of a variable | ||
151 | |||
152 | Example: | ||
153 | >>> d = init() | ||
154 | >>> setVar('TEST', 'testcontents', d) | ||
155 | >>> print getVar('TEST', d) | ||
156 | testcontents | ||
157 | """ | ||
158 | return d.getVar(var,exp) | ||
159 | |||
160 | def delVar(var, d): | ||
161 | """Removes a variable from the data set | ||
162 | |||
163 | Example: | ||
164 | >>> d = init() | ||
165 | >>> setVar('TEST', 'testcontents', d) | ||
166 | >>> print getVar('TEST', d) | ||
167 | testcontents | ||
168 | >>> delVar('TEST', d) | ||
169 | >>> print getVar('TEST', d) | ||
170 | None | ||
171 | """ | ||
172 | d.delVar(var) | ||
173 | |||
174 | def setVarFlag(var, flag, flagvalue, d): | ||
175 | """Set a flag for a given variable to a given value | ||
176 | |||
177 | Example: | ||
178 | >>> d = init() | ||
179 | >>> setVarFlag('TEST', 'python', 1, d) | ||
180 | >>> print getVarFlag('TEST', 'python', d) | ||
181 | 1 | ||
182 | """ | ||
183 | d.setVarFlag(var,flag,flagvalue) | ||
184 | |||
185 | def getVarFlag(var, flag, d): | ||
186 | """Gets given flag from given var | ||
187 | |||
188 | Example: | ||
189 | >>> d = init() | ||
190 | >>> setVarFlag('TEST', 'python', 1, d) | ||
191 | >>> print getVarFlag('TEST', 'python', d) | ||
192 | 1 | ||
193 | """ | ||
194 | return d.getVarFlag(var,flag) | ||
195 | |||
196 | def delVarFlag(var, flag, d): | ||
197 | """Removes a given flag from the variable's flags | ||
198 | |||
199 | Example: | ||
200 | >>> d = init() | ||
201 | >>> setVarFlag('TEST', 'testflag', 1, d) | ||
202 | >>> print getVarFlag('TEST', 'testflag', d) | ||
203 | 1 | ||
204 | >>> delVarFlag('TEST', 'testflag', d) | ||
205 | >>> print getVarFlag('TEST', 'testflag', d) | ||
206 | None | ||
207 | |||
208 | """ | ||
209 | d.delVarFlag(var,flag) | ||
210 | |||
211 | def setVarFlags(var, flags, d): | ||
212 | """Set the flags for a given variable | ||
213 | |||
214 | Example: | ||
215 | >>> d = init() | ||
216 | >>> myflags = {} | ||
217 | >>> myflags['test'] = 'blah' | ||
218 | >>> setVarFlags('TEST', myflags, d) | ||
219 | >>> print getVarFlag('TEST', 'test', d) | ||
220 | blah | ||
221 | """ | ||
222 | d.setVarFlags(var,flags) | ||
223 | |||
224 | def getVarFlags(var, d): | ||
225 | """Gets a variable's flags | ||
226 | |||
227 | Example: | ||
228 | >>> d = init() | ||
229 | >>> setVarFlag('TEST', 'test', 'blah', d) | ||
230 | >>> print getVarFlags('TEST', d)['test'] | ||
231 | blah | ||
232 | """ | ||
233 | return d.getVarFlags(var) | ||
234 | |||
235 | def delVarFlags(var, d): | ||
236 | """Removes a variable's flags | ||
237 | |||
238 | Example: | ||
239 | >>> data = init() | ||
240 | >>> setVarFlag('TEST', 'testflag', 1, data) | ||
241 | >>> print getVarFlag('TEST', 'testflag', data) | ||
242 | 1 | ||
243 | >>> delVarFlags('TEST', data) | ||
244 | >>> print getVarFlags('TEST', data) | ||
245 | None | ||
246 | |||
247 | """ | ||
248 | d.delVarFlags(var) | ||
249 | |||
250 | def keys(d): | ||
251 | """Return a list of keys in d | ||
252 | |||
253 | Example: | ||
254 | >>> d = init() | ||
255 | >>> setVar('TEST', 1, d) | ||
256 | >>> setVar('MOO' , 2, d) | ||
257 | >>> setVarFlag('TEST', 'test', 1, d) | ||
258 | >>> keys(d) | ||
259 | ['TEST', 'MOO'] | ||
260 | """ | ||
261 | return d.keys() | ||
262 | |||
263 | def getData(d): | ||
264 | """Returns the data object used""" | ||
265 | return d | ||
266 | |||
267 | def setData(newData, d): | ||
268 | """Sets the data object to the supplied value""" | ||
269 | d = newData | ||
270 | |||
271 | __expand_var_regexp__ = re.compile(r"\${[^{}]+}") | ||
272 | __expand_python_regexp__ = re.compile(r"\${@.+?}") | ||
273 | |||
274 | def expand(s, d, varname = None): | ||
275 | """Variable expansion using the data store. | ||
276 | |||
277 | Example: | ||
278 | Standard expansion: | ||
279 | >>> d = init() | ||
280 | >>> setVar('A', 'sshd', d) | ||
281 | >>> print expand('/usr/bin/${A}', d) | ||
282 | /usr/bin/sshd | ||
283 | |||
284 | Python expansion: | ||
285 | >>> d = init() | ||
286 | >>> print expand('result: ${@37 * 72}', d) | ||
287 | result: 2664 | ||
288 | |||
289 | Shell expansion: | ||
290 | >>> d = init() | ||
291 | >>> print expand('${TARGET_MOO}', d) | ||
292 | ${TARGET_MOO} | ||
293 | >>> setVar('TARGET_MOO', 'yupp', d) | ||
294 | >>> print expand('${TARGET_MOO}',d) | ||
295 | yupp | ||
296 | >>> setVar('SRC_URI', 'http://somebug.${TARGET_MOO}', d) | ||
297 | >>> delVar('TARGET_MOO', d) | ||
298 | >>> print expand('${SRC_URI}', d) | ||
299 | http://somebug.${TARGET_MOO} | ||
300 | """ | ||
301 | def var_sub(match): | ||
302 | key = match.group()[2:-1] | ||
303 | if varname and key: | ||
304 | if varname == key: | ||
305 | raise Exception("variable %s references itself!" % varname) | ||
306 | var = getVar(key, d, 1) | ||
307 | if var is not None: | ||
308 | return var | ||
309 | else: | ||
310 | return match.group() | ||
311 | |||
312 | def python_sub(match): | ||
313 | import bb | ||
314 | code = match.group()[3:-1] | ||
315 | locals()['d'] = d | ||
316 | s = eval(code) | ||
317 | if type(s) == types.IntType: s = str(s) | ||
318 | return s | ||
319 | |||
320 | if type(s) is not types.StringType: # sanity check | ||
321 | return s | ||
322 | |||
323 | while s.find('$') != -1: | ||
324 | olds = s | ||
325 | try: | ||
326 | s = __expand_var_regexp__.sub(var_sub, s) | ||
327 | s = __expand_python_regexp__.sub(python_sub, s) | ||
328 | if s == olds: break | ||
329 | if type(s) is not types.StringType: # sanity check | ||
330 | import bb | ||
331 | bb.error('expansion of %s returned non-string %s' % (olds, s)) | ||
332 | except KeyboardInterrupt: | ||
333 | raise | ||
334 | except: | ||
335 | note("%s:%s while evaluating:\n%s" % (sys.exc_info()[0], sys.exc_info()[1], s)) | ||
336 | raise | ||
337 | return s | ||
338 | |||
339 | def expandKeys(alterdata, readdata = None): | ||
340 | if readdata == None: | ||
341 | readdata = alterdata | ||
342 | |||
343 | for key in keys(alterdata): | ||
344 | ekey = expand(key, readdata) | ||
345 | if key == ekey: | ||
346 | continue | ||
347 | val = getVar(key, alterdata) | ||
348 | if val is None: | ||
349 | continue | ||
350 | # import copy | ||
351 | # setVarFlags(ekey, copy.copy(getVarFlags(key, readdata)), alterdata) | ||
352 | setVar(ekey, val, alterdata) | ||
353 | |||
354 | for i in ('_append', '_prepend', '_delete'): | ||
355 | dest = getVarFlag(ekey, i, alterdata) or [] | ||
356 | src = getVarFlag(key, i, readdata) or [] | ||
357 | dest.extend(src) | ||
358 | setVarFlag(ekey, i, dest, alterdata) | ||
359 | |||
360 | delVar(key, alterdata) | ||
361 | |||
362 | def expandData(alterdata, readdata = None): | ||
363 | """For each variable in alterdata, expand it, and update the var contents. | ||
364 | Replacements use data from readdata. | ||
365 | |||
366 | Example: | ||
367 | >>> a=init() | ||
368 | >>> b=init() | ||
369 | >>> setVar("dlmsg", "dl_dir is ${DL_DIR}", a) | ||
370 | >>> setVar("DL_DIR", "/path/to/whatever", b) | ||
371 | >>> expandData(a, b) | ||
372 | >>> print getVar("dlmsg", a) | ||
373 | dl_dir is /path/to/whatever | ||
374 | """ | ||
375 | if readdata == None: | ||
376 | readdata = alterdata | ||
377 | |||
378 | for key in keys(alterdata): | ||
379 | val = getVar(key, alterdata) | ||
380 | if type(val) is not types.StringType: | ||
381 | continue | ||
382 | expanded = expand(val, readdata) | ||
383 | # print "key is %s, val is %s, expanded is %s" % (key, val, expanded) | ||
384 | if val != expanded: | ||
385 | setVar(key, expanded, alterdata) | ||
386 | |||
387 | import os | ||
388 | |||
389 | def inheritFromOS(d): | ||
390 | """Inherit variables from the environment.""" | ||
391 | # fakeroot needs to be able to set these | ||
392 | non_inherit_vars = [ "LD_LIBRARY_PATH", "LD_PRELOAD" ] | ||
393 | for s in os.environ.keys(): | ||
394 | if not s in non_inherit_vars: | ||
395 | try: | ||
396 | setVar(s, os.environ[s], d) | ||
397 | setVarFlag(s, 'matchesenv', '1', d) | ||
398 | except TypeError: | ||
399 | pass | ||
400 | |||
401 | import sys | ||
402 | |||
403 | def emit_var(var, o=sys.__stdout__, d = init(), all=False): | ||
404 | """Emit a variable to be sourced by a shell.""" | ||
405 | if getVarFlag(var, "python", d): | ||
406 | return 0 | ||
407 | |||
408 | try: | ||
409 | if all: | ||
410 | oval = getVar(var, d, 0) | ||
411 | val = getVar(var, d, 1) | ||
412 | except KeyboardInterrupt: | ||
413 | raise | ||
414 | except: | ||
415 | excname = str(sys.exc_info()[0]) | ||
416 | if excname == "bb.build.FuncFailed": | ||
417 | raise | ||
418 | o.write('# expansion of %s threw %s\n' % (var, excname)) | ||
419 | return 0 | ||
420 | |||
421 | if all: | ||
422 | o.write('# %s=%s\n' % (var, oval)) | ||
423 | |||
424 | if type(val) is not types.StringType: | ||
425 | return 0 | ||
426 | |||
427 | if getVarFlag(var, 'matchesenv', d): | ||
428 | return 0 | ||
429 | |||
430 | if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all: | ||
431 | return 0 | ||
432 | |||
433 | val.rstrip() | ||
434 | if not val: | ||
435 | return 0 | ||
436 | |||
437 | if getVarFlag(var, "func", d): | ||
438 | # NOTE: should probably check for unbalanced {} within the var | ||
439 | o.write("%s() {\n%s\n}\n" % (var, val)) | ||
440 | else: | ||
441 | if getVarFlag(var, "export", d): | ||
442 | o.write('export ') | ||
443 | else: | ||
444 | if not all: | ||
445 | return 0 | ||
446 | # if we're going to output this within doublequotes, | ||
447 | # to a shell, we need to escape the quotes in the var | ||
448 | alter = re.sub('"', '\\"', val.strip()) | ||
449 | o.write('%s="%s"\n' % (var, alter)) | ||
450 | return 1 | ||
451 | |||
452 | |||
453 | def emit_env(o=sys.__stdout__, d = init(), all=False): | ||
454 | """Emits all items in the data store in a format such that it can be sourced by a shell.""" | ||
455 | |||
456 | env = keys(d) | ||
457 | |||
458 | for e in env: | ||
459 | if getVarFlag(e, "func", d): | ||
460 | continue | ||
461 | emit_var(e, o, d, all) and o.write('\n') | ||
462 | |||
463 | for e in env: | ||
464 | if not getVarFlag(e, "func", d): | ||
465 | continue | ||
466 | emit_var(e, o, d) and o.write('\n') | ||
467 | |||
468 | def update_data(d): | ||
469 | """Modifies the environment vars according to local overrides and commands. | ||
470 | Examples: | ||
471 | Appending to a variable: | ||
472 | >>> d = init() | ||
473 | >>> setVar('TEST', 'this is a', d) | ||
474 | >>> setVar('TEST_append', ' test', d) | ||
475 | >>> setVar('TEST_append', ' of the emergency broadcast system.', d) | ||
476 | >>> update_data(d) | ||
477 | >>> print getVar('TEST', d) | ||
478 | this is a test of the emergency broadcast system. | ||
479 | |||
480 | Prepending to a variable: | ||
481 | >>> setVar('TEST', 'virtual/libc', d) | ||
482 | >>> setVar('TEST_prepend', 'virtual/tmake ', d) | ||
483 | >>> setVar('TEST_prepend', 'virtual/patcher ', d) | ||
484 | >>> update_data(d) | ||
485 | >>> print getVar('TEST', d) | ||
486 | virtual/patcher virtual/tmake virtual/libc | ||
487 | |||
488 | Overrides: | ||
489 | >>> setVar('TEST_arm', 'target', d) | ||
490 | >>> setVar('TEST_ramses', 'machine', d) | ||
491 | >>> setVar('TEST_local', 'local', d) | ||
492 | >>> setVar('OVERRIDES', 'arm', d) | ||
493 | |||
494 | >>> setVar('TEST', 'original', d) | ||
495 | >>> update_data(d) | ||
496 | >>> print getVar('TEST', d) | ||
497 | target | ||
498 | |||
499 | >>> setVar('OVERRIDES', 'arm:ramses:local', d) | ||
500 | >>> setVar('TEST', 'original', d) | ||
501 | >>> update_data(d) | ||
502 | >>> print getVar('TEST', d) | ||
503 | local | ||
504 | """ | ||
505 | |||
506 | debug(2, "update_data()") | ||
507 | |||
508 | # can't do delete env[...] while iterating over the dictionary, so remember them | ||
509 | dodel = [] | ||
510 | overrides = (getVar('OVERRIDES', d, 1) or "").split(':') or [] | ||
511 | |||
512 | def applyOverrides(var, d): | ||
513 | if not overrides: | ||
514 | debug(1, "OVERRIDES not defined, nothing to do") | ||
515 | return | ||
516 | val = getVar(var, d) | ||
517 | for o in overrides: | ||
518 | if var.endswith("_" + o): | ||
519 | l = len(o)+1 | ||
520 | name = var[:-l] | ||
521 | d[name] = d[var] | ||
522 | |||
523 | for s in keys(d): | ||
524 | applyOverrides(s, d) | ||
525 | sval = getVar(s, d) or "" | ||
526 | |||
527 | # Handle line appends: | ||
528 | for (a, o) in getVarFlag(s, '_append', d) or []: | ||
529 | # maybe the OVERRIDE was not yet added so keep the append | ||
530 | if (o and o in overrides) or not o: | ||
531 | delVarFlag(s, '_append', d) | ||
532 | if o: | ||
533 | if not o in overrides: | ||
534 | continue | ||
535 | sval+=a | ||
536 | setVar(s, sval, d) | ||
537 | |||
538 | # Handle line prepends | ||
539 | for (a, o) in getVarFlag(s, '_prepend', d) or []: | ||
540 | # maybe the OVERRIDE was not yet added so keep the append | ||
541 | if (o and o in overrides) or not o: | ||
542 | delVarFlag(s, '_prepend', d) | ||
543 | if o: | ||
544 | if not o in overrides: | ||
545 | continue | ||
546 | sval=a+sval | ||
547 | setVar(s, sval, d) | ||
548 | |||
549 | # Handle line deletions | ||
550 | name = s + "_delete" | ||
551 | nameval = getVar(name, d) | ||
552 | if nameval: | ||
553 | sval = getVar(s, d) | ||
554 | if sval: | ||
555 | new = '' | ||
556 | pattern = nameval.replace('\n','').strip() | ||
557 | for line in sval.split('\n'): | ||
558 | if line.find(pattern) == -1: | ||
559 | new = new + '\n' + line | ||
560 | setVar(s, new, d) | ||
561 | dodel.append(name) | ||
562 | |||
563 | # delete all environment vars no longer needed | ||
564 | for s in dodel: | ||
565 | delVar(s, d) | ||
566 | |||
567 | def inherits_class(klass, d): | ||
568 | val = getVar('__inherit_cache', d) or "" | ||
569 | if os.path.join('classes', '%s.bbclass' % klass) in val.split(): | ||
570 | return True | ||
571 | return False | ||
572 | |||
573 | def _test(): | ||
574 | """Start a doctest run on this module""" | ||
575 | import doctest | ||
576 | from bb import data | ||
577 | doctest.testmod(data) | ||
578 | |||
579 | if __name__ == "__main__": | ||
580 | _test() | ||