1 files changed, 446 insertions, 0 deletions

diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
new file mode 100644
index 0000000000..82eefef1a6
--- /dev/null
+++ b/bitbake/lib/bb/data.py
@@ -0,0 +1,446 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Data' implementations
+
+Functions for interacting with the data structure used by the
+BitBake build tools.
+
+The expandKeys and update_data are the most expensive
+operations. At night the cookie monster came by and
+suggested 'give me cookies on setting the variables and
+things will work out'. Taking this suggestion into account
+applying the skills from the not yet passed 'Entwurf und
+Analyse von Algorithmen' lecture and the cookie
+monster seems to be right. We will track setVar more carefully
+to have faster update_data and expandKeys operations.
+
+This is a trade-off between speed and memory again but
+the speed is more critical here.
+"""
+
+# Copyright (C) 2003, 2004  Chris Larson
+# Copyright (C) 2005        Holger Hans Peter Freyther
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import sys, os, re
+if sys.argv[0][-5:] == "pydoc":
+    path = os.path.dirname(os.path.dirname(sys.argv[1]))
+else:
+    path = os.path.dirname(os.path.dirname(sys.argv[0]))
+sys.path.insert(0, path)
+from itertools import groupby
+
+from bb import data_smart
+from bb import codeparser
+import bb
+
+logger = data_smart.logger
+_dict_type = data_smart.DataSmart
+
+def init():
+    """Return a new object representing the Bitbake data"""
+    return _dict_type()
+
+def init_db(parent = None):
+    """Return a new object representing the Bitbake data,
+    optionally based on an existing object"""
+    if parent is not None:
+        return parent.createCopy()
+    else:
+        return _dict_type()
+
+def createCopy(source):
+    """Link the source set to the destination
+    If one does not find the value in the destination set,
+    search will go on to the source set to get the value.
+    Value from source are copy-on-write. i.e. any try to
+    modify one of them will end up putting the modified value
+    in the destination set.
+    """
+    return source.createCopy()
+
+def initVar(var, d):
+    """Non-destructive var init for data structure"""
+    d.initVar(var)
+
+
+def setVar(var, value, d):
+    """Set a variable to a given value"""
+    d.setVar(var, value)
+
+
+def getVar(var, d, exp = 0):
+    """Gets the value of a variable"""
+    return d.getVar(var, exp)
+
+
+def renameVar(key, newkey, d):
+    """Renames a variable from key to newkey"""
+    d.renameVar(key, newkey)
+
+def delVar(var, d):
+    """Removes a variable from the data set"""
+    d.delVar(var)
+
+def appendVar(var, value, d):
+    """Append additional value to a variable"""
+    d.appendVar(var, value)
+
+def setVarFlag(var, flag, flagvalue, d):
+    """Set a flag for a given variable to a given value"""
+    d.setVarFlag(var, flag, flagvalue)
+
+def getVarFlag(var, flag, d):
+    """Gets given flag from given var"""
+    return d.getVarFlag(var, flag)
+
+def delVarFlag(var, flag, d):
+    """Removes a given flag from the variable's flags"""
+    d.delVarFlag(var, flag)
+
+def setVarFlags(var, flags, d):
+    """Set the flags for a given variable
+
+    Note:
+        setVarFlags will not clear previous
+        flags. Think of this method as
+        addVarFlags
+    """
+    d.setVarFlags(var, flags)
+
+def getVarFlags(var, d):
+    """Gets a variable's flags"""
+    return d.getVarFlags(var)
+
+def delVarFlags(var, d):
+    """Removes a variable's flags"""
+    d.delVarFlags(var)
+
+def keys(d):
+    """Return a list of keys in d"""
+    return d.keys()
+
+
+__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
+__expand_python_regexp__ = re.compile(r"\${@.+?}")
+
+def expand(s, d, varname = None):
+    """Variable expansion using the data store"""
+    return d.expand(s, varname)
+
+def expandKeys(alterdata, readdata = None):
+    if readdata == None:
+        readdata = alterdata
+
+    todolist = {}
+    for key in alterdata:
+        if not '${' in key:
+            continue
+
+        ekey = expand(key, readdata)
+        if key == ekey:
+            continue
+        todolist[key] = ekey
+
+    # These two for loops are split for performance to maximise the
+    # usefulness of the expand cache
+
+    for key in todolist:
+        ekey = todolist[key]
+        newval = alterdata.getVar(ekey, 0)
+        if newval:
+            val = alterdata.getVar(key, 0)
+            if val is not None and newval is not None:
+                bb.warn("Variable key %s (%s) replaces original key %s (%s)." % (key, val, ekey, newval))
+        alterdata.renameVar(key, ekey)
+
+def inheritFromOS(d, savedenv, permitted):
+    """Inherit variables from the initial environment."""
+    exportlist = bb.utils.preserved_envvars_exported()
+    for s in savedenv.keys():
+        if s in permitted:
+            try:
+                d.setVar(s, getVar(s, savedenv, True), op = 'from env')
+                if s in exportlist:
+                    d.setVarFlag(s, "export", True, op = 'auto env export')
+            except TypeError:
+                pass
+
+def emit_var(var, o=sys.__stdout__, d = init(), all=False):
+    """Emit a variable to be sourced by a shell."""
+    if getVarFlag(var, "python", d):
+        return 0
+
+    export = getVarFlag(var, "export", d)
+    unexport = getVarFlag(var, "unexport", d)
+    func = getVarFlag(var, "func", d)
+    if not all and not export and not unexport and not func:
+        return 0
+
+    try:
+        if all:
+            oval = getVar(var, d, 0)
+        val = getVar(var, d, 1)
+    except (KeyboardInterrupt, bb.build.FuncFailed):
+        raise
+    except Exception as exc:
+        o.write('# expansion of %s threw %s: %s\n' % (var, exc.__class__.__name__, str(exc)))
+        return 0
+
+    if all:
+        d.varhistory.emit(var, oval, val, o)
+
+    if (var.find("-") != -1 or var.find(".") != -1 or var.find('{') != -1 or var.find('}') != -1 or var.find('+') != -1) and not all:
+        return 0
+
+    varExpanded = expand(var, d)
+
+    if unexport:
+        o.write('unset %s\n' % varExpanded)
+        return 0
+
+    if val is None:
+        return 0
+
+    val = str(val)
+
+    if varExpanded.startswith("BASH_FUNC_"):
+        varExpanded = varExpanded[10:-2]
+        val = val[3:] # Strip off "() "
+        o.write("%s() %s\n" % (varExpanded, val))
+        o.write("export -f %s\n" % (varExpanded))
+        return 1
+
+    if func:
+        # NOTE: should probably check for unbalanced {} within the var
+        o.write("%s() {\n%s\n}\n" % (varExpanded, val))
+        return 1
+
+    if export:
+        o.write('export ')
+
+    # if we're going to output this within doublequotes,
+    # to a shell, we need to escape the quotes in the var
+    alter = re.sub('"', '\\"', val)
+    alter = re.sub('\n', ' \\\n', alter)
+    alter = re.sub('\\$', '\\\\$', alter)
+    o.write('%s="%s"\n' % (varExpanded, alter))
+    return 0
+
+def emit_env(o=sys.__stdout__, d = init(), all=False):
+    """Emits all items in the data store in a format such that it can be sourced by a shell."""
+
+    isfunc = lambda key: bool(d.getVarFlag(key, "func"))
+    keys = sorted((key for key in d.keys() if not key.startswith("__")), key=isfunc)
+    grouped = groupby(keys, isfunc)
+    for isfunc, keys in grouped:
+        for key in keys:
+            emit_var(key, o, d, all and not isfunc) and o.write('\n')
+
+def exported_keys(d):
+    return (key for key in d.keys() if not key.startswith('__') and
+                                      d.getVarFlag(key, 'export') and
+                                      not d.getVarFlag(key, 'unexport'))
+
+def exported_vars(d):
+    for key in exported_keys(d):
+        try:
+            value = d.getVar(key, True)
+        except Exception:
+            pass
+
+        if value is not None:
+            yield key, str(value)
+
+def emit_func(func, o=sys.__stdout__, d = init()):
+    """Emits all items in the data store in a format such that it can be sourced by a shell."""
+
+    keys = (key for key in d.keys() if not key.startswith("__") and not d.getVarFlag(key, "func"))
+    for key in keys:
+        emit_var(key, o, d, False) and o.write('\n')
+
+    emit_var(func, o, d, False) and o.write('\n')
+    newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
+    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    seen = set()
+    while newdeps:
+        deps = newdeps
+        seen |= deps
+        newdeps = set()
+        for dep in deps:
+            if d.getVarFlag(dep, "func") and not d.getVarFlag(dep, "python"):
+               emit_var(dep, o, d, False) and o.write('\n')
+               newdeps |=  bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep, True))
+               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+        newdeps -= seen
+
+_functionfmt = """
+def {function}(d):
+{body}"""
+
+def emit_func_python(func, o=sys.__stdout__, d = init()):
+    """Emits all items in the data store in a format such that it can be sourced by a shell."""
+
+    def write_func(func, o, call = False):
+        body = d.getVar(func, True)
+        if not body.startswith("def"):
+            body = _functionfmt.format(function=func, body=body)
+
+        o.write(body.strip() + "\n\n")
+        if call:
+            o.write(func + "(d)" + "\n\n")
+
+    write_func(func, o, True)
+    pp = bb.codeparser.PythonParser(func, logger)
+    pp.parse_python(d.getVar(func, True))
+    newdeps = pp.execs
+    newdeps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
+    seen = set()
+    while newdeps:
+        deps = newdeps
+        seen |= deps
+        newdeps = set()
+        for dep in deps:
+            if d.getVarFlag(dep, "func") and d.getVarFlag(dep, "python"):
+               write_func(dep, o)
+               pp = bb.codeparser.PythonParser(dep, logger)
+               pp.parse_python(d.getVar(dep, True))
+               newdeps |= pp.execs
+               newdeps |= set((d.getVarFlag(dep, "vardeps", True) or "").split())
+        newdeps -= seen
+
+def update_data(d):
+    """Performs final steps upon the datastore, including application of overrides"""
+    d.finalize(parent = True)
+
+def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
+    deps = set()
+    try:
+        if key[-1] == ']':
+            vf = key[:-1].split('[')
+            value = d.getVarFlag(vf[0], vf[1], False)
+            parser = d.expandWithRefs(value, key)
+            deps |= parser.references
+            deps = deps | (keys & parser.execs)
+            return deps, value
+        varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "vardepvalueexclude", "postfuncs", "prefuncs"]) or {}
+        vardeps = varflags.get("vardeps")
+        value = d.getVar(key, False)
+
+        def handle_contains(value, contains, d):
+            newvalue = ""
+            for k in sorted(contains):
+                l = (d.getVar(k, True) or "").split()
+                for word in sorted(contains[k]):
+                    if word in l:
+                        newvalue += "\n%s{%s} = Set" %  (k, word)
+                    else:
+                        newvalue += "\n%s{%s} = Unset" %  (k, word)
+            if not newvalue:
+                return value
+            if not value:
+                return newvalue
+            return value + newvalue
+
+        if "vardepvalue" in varflags:
+           value = varflags.get("vardepvalue")
+        elif varflags.get("func"):
+            if varflags.get("python"):
+                parsedvar = d.expandWithRefs(value, key)
+                parser = bb.codeparser.PythonParser(key, logger)
+                if parsedvar.value and "\t" in parsedvar.value:
+                    logger.warn("Variable %s contains tabs, please remove these (%s)" % (key, d.getVar("FILE", True)))
+                parser.parse_python(parsedvar.value)
+                deps = deps | parser.references
+                value = handle_contains(value, parser.contains, d)
+            else:
+                parsedvar = d.expandWithRefs(value, key)
+                parser = bb.codeparser.ShellParser(key, logger)
+                parser.parse_shell(parsedvar.value)
+                deps = deps | shelldeps
+            if vardeps is None:
+                parser.log.flush()
+            if "prefuncs" in varflags:
+                deps = deps | set(varflags["prefuncs"].split())
+            if "postfuncs" in varflags:
+                deps = deps | set(varflags["postfuncs"].split())
+            deps = deps | parsedvar.references
+            deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
+            value = handle_contains(value, parsedvar.contains, d)
+        else:
+            parser = d.expandWithRefs(value, key)
+            deps |= parser.references
+            deps = deps | (keys & parser.execs)
+            value = handle_contains(value, parser.contains, d)
+
+        if "vardepvalueexclude" in varflags:
+            exclude = varflags.get("vardepvalueexclude")
+            for excl in exclude.split('|'):
+                if excl:
+                    value = value.replace(excl, '')
+
+        # Add varflags, assuming an exclusion list is set
+        if varflagsexcl:
+            varfdeps = []
+            for f in varflags:
+                if f not in varflagsexcl:
+                    varfdeps.append('%s[%s]' % (key, f))
+            if varfdeps:
+                deps |= set(varfdeps)
+
+        deps |= set((vardeps or "").split())
+        deps -= set(varflags.get("vardepsexclude", "").split())
+    except Exception as e:
+        raise bb.data_smart.ExpansionError(key, None, e)
+    return deps, value
+    #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
+    #d.setVarFlag(key, "vardeps", deps)
+
+def generate_dependencies(d):
+
+    keys = set(key for key in d if not key.startswith("__"))
+    shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export") and not d.getVarFlag(key, "unexport"))
+    varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS', True)
+
+    deps = {}
+    values = {}
+
+    tasklist = d.getVar('__BBTASKS') or []
+    for task in tasklist:
+        deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
+        newdeps = deps[task]
+        seen = set()
+        while newdeps:
+            nextdeps = newdeps
+            seen |= nextdeps
+            newdeps = set()
+            for dep in nextdeps:
+                if dep not in deps:
+                    deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
+                newdeps |=  deps[dep]
+            newdeps -= seen
+        #print "For %s: %s" % (task, str(deps[task]))
+    return tasklist, deps, values
+
+def inherits_class(klass, d):
+    val = getVar('__inherit_cache', d) or []
+    needle = os.path.join('classes', '%s.bbclass' % klass)
+    for v in val:
+        if v.endswith(needle):
+            return True
+    return False