summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/data.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/data.py')
-rw-r--r--bitbake/lib/bb/data.py403
1 files changed, 403 insertions, 0 deletions
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
new file mode 100644
index 0000000..db938be
--- /dev/null
+++ b/bitbake/lib/bb/data.py
@@ -0,0 +1,403 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3"""
4BitBake 'Data' implementations
5
6Functions for interacting with the data structure used by the
7BitBake build tools.
8
9The expandData and update_data are the most expensive
10operations. At night the cookie monster came by and
11suggested 'give me cookies on setting the variables and
12things will work out'. Taking this suggestion into account
13applying the skills from the not yet passed 'Entwurf und
14Analyse von Algorithmen' lecture and the cookie
15monster seems to be right. We will track setVar more carefully
16to have faster update_data and expandKeys operations.
17
18This is a treade-off between speed and memory again but
19the speed is more critical here.
20"""
21
22# Copyright (C) 2003, 2004 Chris Larson
23# Copyright (C) 2005 Holger Hans Peter Freyther
24#
25# This program is free software; you can redistribute it and/or modify
26# it under the terms of the GNU General Public License version 2 as
27# published by the Free Software Foundation.
28#
29# This program is distributed in the hope that it will be useful,
30# but WITHOUT ANY WARRANTY; without even the implied warranty of
31# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
32# GNU General Public License for more details.
33#
34# You should have received a copy of the GNU General Public License along
35# with this program; if not, write to the Free Software Foundation, Inc.,
36# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
37#
38#Based on functions from the base bb module, Copyright 2003 Holger Schurig
39
40import sys, os, re
41if sys.argv[0][-5:] == "pydoc":
42 path = os.path.dirname(os.path.dirname(sys.argv[1]))
43else:
44 path = os.path.dirname(os.path.dirname(sys.argv[0]))
45sys.path.insert(0, path)
46from itertools import groupby
47
48from bb import data_smart
49from bb import codeparser
50import bb
51
52logger = data_smart.logger
53_dict_type = data_smart.DataSmart
54
55def init():
56 """Return a new object representing the Bitbake data"""
57 return _dict_type()
58
59def init_db(parent = None):
60 """Return a new object representing the Bitbake data,
61 optionally based on an existing object"""
62 if parent is not None:
63 return parent.createCopy()
64 else:
65 return _dict_type()
66
67def createCopy(source):
68 """Link the source set to the destination
69 If one does not find the value in the destination set,
70 search will go on to the source set to get the value.
71 Value from source are copy-on-write. i.e. any try to
72 modify one of them will end up putting the modified value
73 in the destination set.
74 """
75 return source.createCopy()
76
77def initVar(var, d):
78 """Non-destructive var init for data structure"""
79 d.initVar(var)
80
81
82def setVar(var, value, d):
83 """Set a variable to a given value"""
84 d.setVar(var, value)
85
86
87def getVar(var, d, exp = 0):
88 """Gets the value of a variable"""
89 return d.getVar(var, exp)
90
91
92def renameVar(key, newkey, d):
93 """Renames a variable from key to newkey"""
94 d.renameVar(key, newkey)
95
96def delVar(var, d):
97 """Removes a variable from the data set"""
98 d.delVar(var)
99
100def appendVar(var, value, d):
101 """Append additional value to a variable"""
102 d.appendVar(var, value)
103
104def setVarFlag(var, flag, flagvalue, d):
105 """Set a flag for a given variable to a given value"""
106 d.setVarFlag(var, flag, flagvalue)
107
108def getVarFlag(var, flag, d):
109 """Gets given flag from given var"""
110 return d.getVarFlag(var, flag)
111
112def delVarFlag(var, flag, d):
113 """Removes a given flag from the variable's flags"""
114 d.delVarFlag(var, flag)
115
116def setVarFlags(var, flags, d):
117 """Set the flags for a given variable
118
119 Note:
120 setVarFlags will not clear previous
121 flags. Think of this method as
122 addVarFlags
123 """
124 d.setVarFlags(var, flags)
125
126def getVarFlags(var, d):
127 """Gets a variable's flags"""
128 return d.getVarFlags(var)
129
130def delVarFlags(var, d):
131 """Removes a variable's flags"""
132 d.delVarFlags(var)
133
134def keys(d):
135 """Return a list of keys in d"""
136 return d.keys()
137
138
139__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
140__expand_python_regexp__ = re.compile(r"\${@.+?}")
141
142def expand(s, d, varname = None):
143 """Variable expansion using the data store"""
144 return d.expand(s, varname)
145
146def expandKeys(alterdata, readdata = None):
147 if readdata == None:
148 readdata = alterdata
149
150 todolist = {}
151 for key in alterdata:
152 if not '${' in key:
153 continue
154
155 ekey = expand(key, readdata)
156 if key == ekey:
157 continue
158 todolist[key] = ekey
159
160 # These two for loops are split for performance to maximise the
161 # usefulness of the expand cache
162
163 for key in todolist:
164 ekey = todolist[key]
165 newval = alterdata.getVar(ekey, 0)
166 if newval:
167 val = alterdata.getVar(key, 0)
168 if val is not None and newval is not None:
169 bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval))
170 alterdata.renameVar(key, ekey)
171
172def inheritFromOS(d, savedenv, permitted):
173 """Inherit variables from the initial environment."""
174 exportlist = bb.utils.preserved_envvars_exported()
175 for s in savedenv.keys():
176 if s in permitted:
177 try:
178 d.setVar(s, getVar(s, savedenv, True), op = 'from env')
179 if s in exportlist:
180 d.setVarFlag(s, "export", True, op = 'auto env export')
181 except TypeError:
182 pass
183
184def emit_var(var, o=sys.__stdout__, d = init(), all=False):
185 """Emit a variable to be sourced by a shell."""
186 if getVarFlag(var, "python", d):
187 return 0
188
189 export = getVarFlag(var, "export", d)
190 unexport = getVarFlag(var, "unexport", d)
191 func = getVarFlag(var, "func", d)
192 if not all and not export and not unexport and not func:
193 return 0
194
195 try:
196 if all:
197 oval = getVar(var, d, 0)
198 val = getVar(var, d, 1)
199 except (KeyboardInterrupt, bb.build.FuncFailed):
200 raise
201 except Exception as exc:
202 o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
203 return 0
204
205 if all:
206 d.varhistory.emit(var, oval, val, o)
207
208 if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
209 return 0
210
211 varExpanded = expand(var, d)
212
213 if unexport:
214 o.write('unset %s\n' % varExpanded)
215 return 0
216
217 if val is None:
218 return 0
219
220 val = str(val)
221
222 if func:
223 # NOTE: should probably check for unbalanced {} within the var
224 o.write("%s() {\n%s\n}\n" % (varExpanded, val))
225 return 1
226
227 if export:
228 o.write('export ')
229
230 # if we're going to output this within doublequotes,
231 # to a shell, we need to escape the quotes in the var
232 alter = re.sub('"', '\\"', val)
233 alter = re.sub('\n', ' \\\n', alter)
234 o.write('%s="%s"\n' % (varExpanded, alter))
235 return 0
236
237def emit_env(o=sys.__stdout__, d = init(), all=False):
238 """Emits all items in the data store in a format such that it can be sourced by a shell."""
239
240 isfunc = lambda key: bool(d.getVarFlag(key, "func"))
241 keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
242 grouped = groupby(keys, isfunc)
243 for isfunc, keys in grouped:
244 for key in keys:
245 emit_var(key, o, d, all and not isfunc) and o.write('\n')
246
247def exported_keys(d):
248 return (key for key in d.keys() if not key.startswith('__') and
249 d.getVarFlag(key, 'export') and
250 not d.getVarFlag(key, 'unexport'))
251
252def exported_vars(d):
253 for key in exported_keys(d):
254 try:
255 value = d.getVar(key, True)
256 except Exception:
257 pass
258
259 if value is not None:
260 yield key, str(value)
261
262def emit_func(func, o=sys.__stdout__, d = init()):
263 """Emits all items in the data store in a format such that it can be sourced by a shell."""
264
265 keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func"))
266 for key in keys:
267 emit_var(key, o, d, False) and o.write('\n')
268
269 emit_var(func, o, d, False) and o.write('\n')
270 newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
271 newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
272 seen = set()
273 while newdeps:
274 deps = newdeps
275 seen |= deps
276 newdeps = set()
277 for dep in deps:
278 if d.getVarFlag(dep, "func") and not d.getVarFlag(dep, "python"):
279 emit_var(dep, o, d, False) and o.write('\n')
280 newdeps |= bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep, True))
281 newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
282 newdeps -= seen
283
284def update_data(d):
285 """Performs final steps upon the datastore, including application of overrides"""
286 d.finalize(parent = True)
287
288def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
289 deps = set()
290 try:
291 if key[-1] == ']':
292 vf = key[:-1].split('[')
293 value = d.getVarFlag(vf[0], vf[1], False)
294 parser = d.expandWithRefs(value, key)
295 deps |= parser.references
296 deps = deps | (keys & parser.execs)
297 return deps, value
298 varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "vardepvalueexclude", "postfuncs", "prefuncs"]) or {}
299 vardeps = varflags.get("vardeps")
300 value = d.getVar(key, False)
301
302 def handle_contains(value, contains, d):
303 newvalue = ""
304 for k in sorted(contains):
305 l = (d.getVar(k, True) or "").split()
306 for word in sorted(contains[k]):
307 if word in l:
308 newvalue += "\n%s{%s} = Set" % (k, word)
309 else:
310 newvalue += "\n%s{%s} = Unset" % (k, word)
311 if not newvalue:
312 return value
313 if not value:
314 return newvalue
315 return value + newvalue
316
317 if "vardepvalue" in varflags:
318 value = varflags.get("vardepvalue")
319 elif varflags.get("func"):
320 if varflags.get("python"):
321 parsedvar = d.expandWithRefs(value, key)
322 parser = bb.codeparser.PythonParser(key, logger)
323 if parsedvar.value and "\t" in parsedvar.value:
324 logger.warn("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True)))
325 parser.parse_python(parsedvar.value)
326 deps = deps | parser.references
327 value = handle_contains(value, parser.contains, d)
328 else:
329 parsedvar = d.expandWithRefs(value, key)
330 parser = bb.codeparser.ShellParser(key, logger)
331 parser.parse_shell(parsedvar.value)
332 deps = deps | shelldeps
333 if vardeps is None:
334 parser.log.flush()
335 if "prefuncs" in varflags:
336 deps = deps | set(varflags["prefuncs"].split())
337 if "postfuncs" in varflags:
338 deps = deps | set(varflags["postfuncs"].split())
339 deps = deps | parsedvar.references
340 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
341 value = handle_contains(value, parsedvar.contains, d)
342 else:
343 parser = d.expandWithRefs(value, key)
344 deps |= parser.references
345 deps = deps | (keys & parser.execs)
346 value = handle_contains(value, parser.contains, d)
347
348 if "vardepvalueexclude" in varflags:
349 exclude = varflags.get("vardepvalueexclude")
350 for excl in exclude.split('|'):
351 if excl:
352 value = value.replace(excl, '')
353
354 # Add varflags, assuming an exclusion list is set
355 if varflagsexcl:
356 varfdeps = []
357 for f in varflags:
358 if f not in varflagsexcl:
359 varfdeps.append('%s[%s]' % (key, f))
360 if varfdeps:
361 deps |= set(varfdeps)
362
363 deps |= set((vardeps or "").split())
364 deps -= set(varflags.get("vardepsexclude", "").split())
365 except Exception as e:
366 raise bb.data_smart.ExpansionError(key, None, e)
367 return deps, value
368 #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
369 #d.setVarFlag(key, "vardeps", deps)
370
371def generate_dependencies(d):
372
373 keys = set(key for key in d if not key.startswith("__"))
374 shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export") and not d.getVarFlag(key, "unexport"))
375 varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True)
376
377 deps = {}
378 values = {}
379
380 tasklist = d.getVar('__BBTASKS') or []
381 for task in tasklist:
382 deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
383 newdeps = deps[task]
384 seen = set()
385 while newdeps:
386 nextdeps = newdeps
387 seen |= nextdeps
388 newdeps = set()
389 for dep in nextdeps:
390 if dep not in deps:
391 deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
392 newdeps |= deps[dep]
393 newdeps -= seen
394 #print "For %s: %s" % (task, str(deps[task]))
395 return tasklist, deps, values
396
397def inherits_class(klass, d):
398 val = getVar('__inherit_cache', d) or []
399 needle = os.path.join('classes', '%s.bbclass' % klass)
400 for v in val:
401 if v.endswith(needle):
402 return True
403 return False