diff options
author | Tudor Florea <tudor.florea@enea.com> | 2015-10-09 22:59:03 +0200 |
---|---|---|
committer | Tudor Florea <tudor.florea@enea.com> | 2015-10-09 22:59:03 +0200 |
commit | 972dcfcdbfe75dcfeb777150c136576cf1a71e99 (patch) | |
tree | 97a61cd7e293d7ae9d56ef7ed0f81253365bb026 /bitbake/lib/bb/data.py | |
download | poky-972dcfcdbfe75dcfeb777150c136576cf1a71e99.tar.gz |
initial commit for Enea Linux 5.0 arm
Signed-off-by: Tudor Florea <tudor.florea@enea.com>
Diffstat (limited to 'bitbake/lib/bb/data.py')
-rw-r--r-- | bitbake/lib/bb/data.py | 446 |
1 files changed, 446 insertions, 0 deletions
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py new file mode 100644 index 0000000000..82eefef1a6 --- /dev/null +++ b/bitbake/lib/bb/data.py | |||
@@ -0,0 +1,446 @@ | |||
1 | # ex:ts=4:sw=4:sts=4:et | ||
2 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
3 | """ | ||
4 | BitBake 'Data' implementations | ||
5 | |||
6 | Functions for interacting with the data structure used by the | ||
7 | BitBake build tools. | ||
8 | |||
9 | The expandKeys and update_data are the most expensive | ||
10 | operations. At night the cookie monster came by and | ||
11 | suggested 'give me cookies on setting the variables and | ||
12 | things will work out'. Taking this suggestion into account | ||
13 | applying the skills from the not yet passed 'Entwurf und | ||
14 | Analyse von Algorithmen' lecture and the cookie | ||
15 | monster seems to be right. We will track setVar more carefully | ||
16 | to have faster update_data and expandKeys operations. | ||
17 | |||
18 | This is a trade-off between speed and memory again but | ||
19 | the speed is more critical here. | ||
20 | """ | ||
21 | |||
22 | # Copyright (C) 2003, 2004 Chris Larson | ||
23 | # Copyright (C) 2005 Holger Hans Peter Freyther | ||
24 | # | ||
25 | # This program is free software; you can redistribute it and/or modify | ||
26 | # it under the terms of the GNU General Public License version 2 as | ||
27 | # published by the Free Software Foundation. | ||
28 | # | ||
29 | # This program is distributed in the hope that it will be useful, | ||
30 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
31 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
32 | # GNU General Public License for more details. | ||
33 | # | ||
34 | # You should have received a copy of the GNU General Public License along | ||
35 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
36 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
37 | # | ||
38 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
39 | |||
40 | import sys, os, re | ||
41 | if sys.argv[0][-5:] == "pydoc": | ||
42 | path = os.path.dirname(os.path.dirname(sys.argv[1])) | ||
43 | else: | ||
44 | path = os.path.dirname(os.path.dirname(sys.argv[0])) | ||
45 | sys.path.insert(0, path) | ||
46 | from itertools import groupby | ||
47 | |||
48 | from bb import data_smart | ||
49 | from bb import codeparser | ||
50 | import bb | ||
51 | |||
52 | logger = data_smart.logger | ||
53 | _dict_type = data_smart.DataSmart | ||
54 | |||
55 | def init(): | ||
56 | """Return a new object representing the Bitbake data""" | ||
57 | return _dict_type() | ||
58 | |||
59 | def init_db(parent = None): | ||
60 | """Return a new object representing the Bitbake data, | ||
61 | optionally based on an existing object""" | ||
62 | if parent is not None: | ||
63 | return parent.createCopy() | ||
64 | else: | ||
65 | return _dict_type() | ||
66 | |||
67 | def createCopy(source): | ||
68 | """Link the source set to the destination | ||
69 | If one does not find the value in the destination set, | ||
70 | search will go on to the source set to get the value. | ||
71 | Value from source are copy-on-write. i.e. any try to | ||
72 | modify one of them will end up putting the modified value | ||
73 | in the destination set. | ||
74 | """ | ||
75 | return source.createCopy() | ||
76 | |||
77 | def initVar(var, d): | ||
78 | """Non-destructive var init for data structure""" | ||
79 | d.initVar(var) | ||
80 | |||
81 | |||
82 | def setVar(var, value, d): | ||
83 | """Set a variable to a given value""" | ||
84 | d.setVar(var, value) | ||
85 | |||
86 | |||
87 | def getVar(var, d, exp = 0): | ||
88 | """Gets the value of a variable""" | ||
89 | return d.getVar(var, exp) | ||
90 | |||
91 | |||
92 | def renameVar(key, newkey, d): | ||
93 | """Renames a variable from key to newkey""" | ||
94 | d.renameVar(key, newkey) | ||
95 | |||
96 | def delVar(var, d): | ||
97 | """Removes a variable from the data set""" | ||
98 | d.delVar(var) | ||
99 | |||
100 | def appendVar(var, value, d): | ||
101 | """Append additional value to a variable""" | ||
102 | d.appendVar(var, value) | ||
103 | |||
104 | def setVarFlag(var, flag, flagvalue, d): | ||
105 | """Set a flag for a given variable to a given value""" | ||
106 | d.setVarFlag(var, flag, flagvalue) | ||
107 | |||
108 | def getVarFlag(var, flag, d): | ||
109 | """Gets given flag from given var""" | ||
110 | return d.getVarFlag(var, flag) | ||
111 | |||
112 | def delVarFlag(var, flag, d): | ||
113 | """Removes a given flag from the variable's flags""" | ||
114 | d.delVarFlag(var, flag) | ||
115 | |||
116 | def setVarFlags(var, flags, d): | ||
117 | """Set the flags for a given variable | ||
118 | |||
119 | Note: | ||
120 | setVarFlags will not clear previous | ||
121 | flags. Think of this method as | ||
122 | addVarFlags | ||
123 | """ | ||
124 | d.setVarFlags(var, flags) | ||
125 | |||
126 | def getVarFlags(var, d): | ||
127 | """Gets a variable's flags""" | ||
128 | return d.getVarFlags(var) | ||
129 | |||
130 | def delVarFlags(var, d): | ||
131 | """Removes a variable's flags""" | ||
132 | d.delVarFlags(var) | ||
133 | |||
134 | def keys(d): | ||
135 | """Return a list of keys in d""" | ||
136 | return d.keys() | ||
137 | |||
138 | |||
139 | __expand_var_regexp__ = re.compile(r"\${[^{}]+}") | ||
140 | __expand_python_regexp__ = re.compile(r"\${@.+?}") | ||
141 | |||
142 | def expand(s, d, varname = None): | ||
143 | """Variable expansion using the data store""" | ||
144 | return d.expand(s, varname) | ||
145 | |||
146 | def expandKeys(alterdata, readdata = None): | ||
147 | if readdata == None: | ||
148 | readdata = alterdata | ||
149 | |||
150 | todolist = {} | ||
151 | for key in alterdata: | ||
152 | if not '${' in key: | ||
153 | continue | ||
154 | |||
155 | ekey = expand(key, readdata) | ||
156 | if key == ekey: | ||
157 | continue | ||
158 | todolist[key] = ekey | ||
159 | |||
160 | # These two for loops are split for performance to maximise the | ||
161 | # usefulness of the expand cache | ||
162 | |||
163 | for key in todolist: | ||
164 | ekey = todolist[key] | ||
165 | newval = alterdata.getVar(ekey, 0) | ||
166 | if newval: | ||
167 | val = alterdata.getVar(key, 0) | ||
168 | if val is not None and newval is not None: | ||
169 | bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval)) | ||
170 | alterdata.renameVar(key, ekey) | ||
171 | |||
172 | def inheritFromOS(d, savedenv, permitted): | ||
173 | """Inherit variables from the initial environment.""" | ||
174 | exportlist = bb.utils.preserved_envvars_exported() | ||
175 | for s in savedenv.keys(): | ||
176 | if s in permitted: | ||
177 | try: | ||
178 | d.setVar(s, getVar(s, savedenv, True), op = 'from env') | ||
179 | if s in exportlist: | ||
180 | d.setVarFlag(s, "export", True, op = 'auto env export') | ||
181 | except TypeError: | ||
182 | pass | ||
183 | |||
184 | def emit_var(var, o=sys.__stdout__, d = init(), all=False): | ||
185 | """Emit a variable to be sourced by a shell.""" | ||
186 | if getVarFlag(var, "python", d): | ||
187 | return 0 | ||
188 | |||
189 | export = getVarFlag(var, "export", d) | ||
190 | unexport = getVarFlag(var, "unexport", d) | ||
191 | func = getVarFlag(var, "func", d) | ||
192 | if not all and not export and not unexport and not func: | ||
193 | return 0 | ||
194 | |||
195 | try: | ||
196 | if all: | ||
197 | oval = getVar(var, d, 0) | ||
198 | val = getVar(var, d, 1) | ||
199 | except (KeyboardInterrupt, bb.build.FuncFailed): | ||
200 | raise | ||
201 | except Exception as exc: | ||
202 | o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc))) | ||
203 | return 0 | ||
204 | |||
205 | if all: | ||
206 | d.varhistory.emit(var, oval, val, o) | ||
207 | |||
208 | if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all: | ||
209 | return 0 | ||
210 | |||
211 | varExpanded = expand(var, d) | ||
212 | |||
213 | if unexport: | ||
214 | o.write('unset %s\n' % varExpanded) | ||
215 | return 0 | ||
216 | |||
217 | if val is None: | ||
218 | return 0 | ||
219 | |||
220 | val = str(val) | ||
221 | |||
222 | if varExpanded.startswith("BASH_FUNC_"): | ||
223 | varExpanded = varExpanded[10:-2] | ||
224 | val = val[3:] # Strip off "() " | ||
225 | o.write("%s() %s\n" % (varExpanded, val)) | ||
226 | o.write("export -f %s\n" % (varExpanded)) | ||
227 | return 1 | ||
228 | |||
229 | if func: | ||
230 | # NOTE: should probably check for unbalanced {} within the var | ||
231 | o.write("%s() {\n%s\n}\n" % (varExpanded, val)) | ||
232 | return 1 | ||
233 | |||
234 | if export: | ||
235 | o.write('export ') | ||
236 | |||
237 | # if we're going to output this within doublequotes, | ||
238 | # to a shell, we need to escape the quotes in the var | ||
239 | alter = re.sub('"', '\\"', val) | ||
240 | alter = re.sub('\n', ' \\\n', alter) | ||
241 | alter = re.sub('\\$', '\\\\$', alter) | ||
242 | o.write('%s="%s"\n' % (varExpanded, alter)) | ||
243 | return 0 | ||
244 | |||
245 | def emit_env(o=sys.__stdout__, d = init(), all=False): | ||
246 | """Emits all items in the data store in a format such that it can be sourced by a shell.""" | ||
247 | |||
248 | isfunc = lambda key: bool(d.getVarFlag(key, "func")) | ||
249 | keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc) | ||
250 | grouped = groupby(keys, isfunc) | ||
251 | for isfunc, keys in grouped: | ||
252 | for key in keys: | ||
253 | emit_var(key, o, d, all and not isfunc) and o.write('\n') | ||
254 | |||
255 | def exported_keys(d): | ||
256 | return (key for key in d.keys() if not key.startswith('__') and | ||
257 | d.getVarFlag(key, 'export') and | ||
258 | not d.getVarFlag(key, 'unexport')) | ||
259 | |||
260 | def exported_vars(d): | ||
261 | for key in exported_keys(d): | ||
262 | try: | ||
263 | value = d.getVar(key, True) | ||
264 | except Exception: | ||
265 | pass | ||
266 | |||
267 | if value is not None: | ||
268 | yield key, str(value) | ||
269 | |||
270 | def emit_func(func, o=sys.__stdout__, d = init()): | ||
271 | """Emits all items in the data store in a format such that it can be sourced by a shell.""" | ||
272 | |||
273 | keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func")) | ||
274 | for key in keys: | ||
275 | emit_var(key, o, d, False) and o.write('\n') | ||
276 | |||
277 | emit_var(func, o, d, False) and o.write('\n') | ||
278 | newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True)) | ||
279 | newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split()) | ||
280 | seen = set() | ||
281 | while newdeps: | ||
282 | deps = newdeps | ||
283 | seen |= deps | ||
284 | newdeps = set() | ||
285 | for dep in deps: | ||
286 | if d.getVarFlag(dep, "func") and not d.getVarFlag(dep, "python"): | ||
287 | emit_var(dep, o, d, False) and o.write('\n') | ||
288 | newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep, True)) | ||
289 | newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split()) | ||
290 | newdeps -= seen | ||
291 | |||
292 | _functionfmt = """ | ||
293 | def {function}(d): | ||
294 | {body}""" | ||
295 | |||
296 | def emit_func_python(func, o=sys.__stdout__, d = init()): | ||
297 | """Emits all items in the data store in a format such that it can be sourced by a shell.""" | ||
298 | |||
299 | def write_func(func, o, call = False): | ||
300 | body = d.getVar(func, True) | ||
301 | if not body.startswith("def"): | ||
302 | body = _functionfmt.format(function=func, body=body) | ||
303 | |||
304 | o.write(body.strip() + "\n\n") | ||
305 | if call: | ||
306 | o.write(func + "(d)" + "\n\n") | ||
307 | |||
308 | write_func(func, o, True) | ||
309 | pp = bb.codeparser.PythonParser(func, logger) | ||
310 | pp.parse_python(d.getVar(func, True)) | ||
311 | newdeps = pp.execs | ||
312 | newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split()) | ||
313 | seen = set() | ||
314 | while newdeps: | ||
315 | deps = newdeps | ||
316 | seen |= deps | ||
317 | newdeps = set() | ||
318 | for dep in deps: | ||
319 | if d.getVarFlag(dep, "func") and d.getVarFlag(dep, "python"): | ||
320 | write_func(dep, o) | ||
321 | pp = bb.codeparser.PythonParser(dep, logger) | ||
322 | pp.parse_python(d.getVar(dep, True)) | ||
323 | newdeps |= pp.execs | ||
324 | newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split()) | ||
325 | newdeps -= seen | ||
326 | |||
327 | def update_data(d): | ||
328 | """Performs final steps upon the datastore, including application of overrides""" | ||
329 | d.finalize(parent = True) | ||
330 | |||
331 | def build_dependencies(key, keys, shelldeps, varflagsexcl, d): | ||
332 | deps = set() | ||
333 | try: | ||
334 | if key[-1] == ']': | ||
335 | vf = key[:-1].split('[') | ||
336 | value = d.getVarFlag(vf[0], vf[1], False) | ||
337 | parser = d.expandWithRefs(value, key) | ||
338 | deps |= parser.references | ||
339 | deps = deps | (keys & parser.execs) | ||
340 | return deps, value | ||
341 | varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "vardepvalueexclude", "postfuncs", "prefuncs"]) or {} | ||
342 | vardeps = varflags.get("vardeps") | ||
343 | value = d.getVar(key, False) | ||
344 | |||
345 | def handle_contains(value, contains, d): | ||
346 | newvalue = "" | ||
347 | for k in sorted(contains): | ||
348 | l = (d.getVar(k, True) or "").split() | ||
349 | for word in sorted(contains[k]): | ||
350 | if word in l: | ||
351 | newvalue += "\n%s{%s} = Set" % (k, word) | ||
352 | else: | ||
353 | newvalue += "\n%s{%s} = Unset" % (k, word) | ||
354 | if not newvalue: | ||
355 | return value | ||
356 | if not value: | ||
357 | return newvalue | ||
358 | return value + newvalue | ||
359 | |||
360 | if "vardepvalue" in varflags: | ||
361 | value = varflags.get("vardepvalue") | ||
362 | elif varflags.get("func"): | ||
363 | if varflags.get("python"): | ||
364 | parsedvar = d.expandWithRefs(value, key) | ||
365 | parser = bb.codeparser.PythonParser(key, logger) | ||
366 | if parsedvar.value and "\t" in parsedvar.value: | ||
367 | logger.warn("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True))) | ||
368 | parser.parse_python(parsedvar.value) | ||
369 | deps = deps | parser.references | ||
370 | value = handle_contains(value, parser.contains, d) | ||
371 | else: | ||
372 | parsedvar = d.expandWithRefs(value, key) | ||
373 | parser = bb.codeparser.ShellParser(key, logger) | ||
374 | parser.parse_shell(parsedvar.value) | ||
375 | deps = deps | shelldeps | ||
376 | if vardeps is None: | ||
377 | parser.log.flush() | ||
378 | if "prefuncs" in varflags: | ||
379 | deps = deps | set(varflags["prefuncs"].split()) | ||
380 | if "postfuncs" in varflags: | ||
381 | deps = deps | set(varflags["postfuncs"].split()) | ||
382 | deps = deps | parsedvar.references | ||
383 | deps = deps | (keys & parser.execs) | (keys & parsedvar.execs) | ||
384 | value = handle_contains(value, parsedvar.contains, d) | ||
385 | else: | ||
386 | parser = d.expandWithRefs(value, key) | ||
387 | deps |= parser.references | ||
388 | deps = deps | (keys & parser.execs) | ||
389 | value = handle_contains(value, parser.contains, d) | ||
390 | |||
391 | if "vardepvalueexclude" in varflags: | ||
392 | exclude = varflags.get("vardepvalueexclude") | ||
393 | for excl in exclude.split('|'): | ||
394 | if excl: | ||
395 | value = value.replace(excl, '') | ||
396 | |||
397 | # Add varflags, assuming an exclusion list is set | ||
398 | if varflagsexcl: | ||
399 | varfdeps = [] | ||
400 | for f in varflags: | ||
401 | if f not in varflagsexcl: | ||
402 | varfdeps.append('%s[%s]' % (key, f)) | ||
403 | if varfdeps: | ||
404 | deps |= set(varfdeps) | ||
405 | |||
406 | deps |= set((vardeps or "").split()) | ||
407 | deps -= set(varflags.get("vardepsexclude", "").split()) | ||
408 | except Exception as e: | ||
409 | raise bb.data_smart.ExpansionError(key, None, e) | ||
410 | return deps, value | ||
411 | #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs))) | ||
412 | #d.setVarFlag(key, "vardeps", deps) | ||
413 | |||
414 | def generate_dependencies(d): | ||
415 | |||
416 | keys = set(key for key in d if not key.startswith("__")) | ||
417 | shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export") and not d.getVarFlag(key, "unexport")) | ||
418 | varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True) | ||
419 | |||
420 | deps = {} | ||
421 | values = {} | ||
422 | |||
423 | tasklist = d.getVar('__BBTASKS') or [] | ||
424 | for task in tasklist: | ||
425 | deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d) | ||
426 | newdeps = deps[task] | ||
427 | seen = set() | ||
428 | while newdeps: | ||
429 | nextdeps = newdeps | ||
430 | seen |= nextdeps | ||
431 | newdeps = set() | ||
432 | for dep in nextdeps: | ||
433 | if dep not in deps: | ||
434 | deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d) | ||
435 | newdeps |= deps[dep] | ||
436 | newdeps -= seen | ||
437 | #print "For %s: %s" % (task, str(deps[task])) | ||
438 | return tasklist, deps, values | ||
439 | |||
440 | def inherits_class(klass, d): | ||
441 | val = getVar('__inherit_cache', d) or [] | ||
442 | needle = os.path.join('classes', '%s.bbclass' % klass) | ||
443 | for v in val: | ||
444 | if v.endswith(needle): | ||
445 | return True | ||
446 | return False | ||