bitbake: lib/bb/utils: fix and extend edit_metadata_file()

Fix several bugs and add some useful enhancements to make this into a
more generic metadata editing function:
* Support modifying function values (name must be specified ending with
  "()")
* Support dropping values by returning None as the new value
* Split out edit_metadata() function to provide same functionality
  on a list/iterable
* Pass operation to callback and allow function to return them
* Pass current output lines to callback so they can be modified
* Fix handling of single-quoted values
* Handle :=, =+, .=, and =. operators
* Support arbitrary indent string
* Support indenting by length of assignment (by specifying -1)
* Fix typo in variablename - intentspc -> indentspc
* Expand function docstring to cover arguments / usage
* Add a parameter to enable matching names with overrides applied
* Add some bitbake-selftest tests

Note that this does change the expected signature of the callback
function. The only known caller is in lib/bb/utils.py itself; I doubt
anyone else has made extensive use of this function yet.

(Bitbake rev: 20059e4d5ab9bf0f32c781ccb208da3c95818018)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 180 insertions, 58 deletions

diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 0db7e56651..988b845a4a 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -963,14 +963,62 @@ def exec_flat_python_func(func, *args, **kwargs):
     bb.utils.better_exec(comp, context, code, '<string>')
     return context['retval']
 
-def edit_metadata_file(meta_file, variables, func):
-    """Edit a recipe or config file and modify one or more specified
-    variable values set in the file using a specified callback function.
-    The file is only written to if the value(s) actually change.
+def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
+    """Edit lines from a recipe or config file and modify one or more
+    specified variable values set in the file using a specified callback
+    function. Lines are expected to have trailing newlines.
+    Parameters:
+        meta_lines: lines from the file; can be a list or an iterable
+            (e.g. file pointer)
+        variables: a list of variable names to look for. Functions
+            may also be specified, but must be specified with '()' at
+            the end of the name. Note that the function doesn't have
+            any intrinsic understanding of _append, _prepend, _remove,
+            or overrides, so these are considered as part of the name.
+            These values go into a regular expression, so regular
+            expression syntax is allowed.
+        varfunc: callback function called for every variable matching
+            one of the entries in the variables parameter. The function
+            should take four arguments:
+                varname: name of variable matched
+                origvalue: current value in file
+                op: the operator (e.g. '+=')
+                newlines: list of lines up to this point. You can use
+                    this to prepend lines before this variable setting
+                    if you wish.
+            and should return a three-element tuple:
+                newvalue: new value to substitute in, or None to drop
+                    the variable setting entirely. (If the removal
+                    results in two consecutive blank lines, one of the
+                    blank lines will also be dropped).
+                newop: the operator to use - if you specify None here,
+                    the original operation will be used.
+                indent: number of spaces to indent multi-line entries,
+                    or -1 to indent up to the level of the assignment
+                    and opening quote, or a string to use as the indent.
+                minbreak: True to allow the first element of a
+                    multi-line value to continue on the same line as
+                    the assignment, False to indent before the first
+                    element.
+        match_overrides: True to match items with _overrides on the end,
+            False otherwise
+    Returns a tuple:
+        updated:
+            True if changes were made, False otherwise.
+        newlines:
+            Lines after processing
     """
+
     var_res = {}
+    if match_overrides:
+        override_re = '(_[a-zA-Z0-9-_$(){}]+)?'
+    else:
+        override_re = ''
     for var in variables:
-        var_res[var] = re.compile(r'^%s[ \t]*[?+]*=' % var)
+        if var.endswith('()'):
+            var_res[var] = re.compile('^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
+        else:
+            var_res[var] = re.compile('^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
 
     updated = False
     varset_start = ''
@@ -978,70 +1026,144 @@ def edit_metadata_file(meta_file, variables, func):
     newlines = []
     in_var = None
     full_value = ''
+    var_end = ''
 
     def handle_var_end():
-        (newvalue, indent, minbreak) = func(in_var, full_value)
-        if newvalue != full_value:
-            if isinstance(newvalue, list):
-                intentspc = ' ' * indent
-                if minbreak:
-                    # First item on first line
-                    if len(newvalue) == 1:
-                        newlines.append('%s "%s"\n' % (varset_start, newvalue[0]))
+        prerun_newlines = newlines[:]
+        op = varset_start[len(in_var):].strip()
+        (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines)
+        changed = (prerun_newlines != newlines)
+
+        if newvalue is None:
+            # Drop the value
+            return True
+        elif newvalue != full_value or (newop not in [None, op]):
+            if newop not in [None, op]:
+                # Callback changed the operator
+                varset_new = "%s %s" % (in_var, newop)
+            else:
+                varset_new = varset_start
+
+            if isinstance(indent, (int, long)):
+                if indent == -1:
+                    indentspc = ' ' * (len(varset_new) + 2)
+                else:
+                    indentspc = ' ' * indent
+            else:
+                indentspc = indent
+            if in_var.endswith('()'):
+                # A function definition
+                if isinstance(newvalue, list):
+                    newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue)))
+                else:
+                    if not newvalue.startswith('\n'):
+                        newvalue = '\n' + newvalue
+                    if not newvalue.endswith('\n'):
+                        newvalue = newvalue + '\n'
+                    newlines.append('%s {%s}\n' % (varset_new, newvalue))
+            else:
+                # Normal variable
+                if isinstance(newvalue, list):
+                    if not newvalue:
+                        # Empty list -> empty string
+                        newlines.append('%s ""\n' % varset_new)
+                    elif minbreak:
+                        # First item on first line
+                        if len(newvalue) == 1:
+                            newlines.append('%s "%s"\n' % (varset_new, newvalue[0]))
+                        else:
+                            newlines.append('%s "%s \\\n' % (varset_new, newvalue[0]))
+                            for item in newvalue[1:]:
+                                newlines.append('%s%s \\\n' % (indentspc, item))
+                            newlines.append('%s"\n' % indentspc)
                     else:
-                        newlines.append('%s "%s\\\n' % (varset_start, newvalue[0]))
-                        for item in newvalue[1:]:
-                            newlines.append('%s%s \\\n' % (intentspc, item))
+                        # No item on first line
+                        newlines.append('%s " \\\n' % varset_new)
+                        for item in newvalue:
+                            newlines.append('%s%s \\\n' % (indentspc, item))
                         newlines.append('%s"\n' % indentspc)
                 else:
-                    # No item on first line
-                    newlines.append('%s " \\\n' % varset_start)
-                    for item in newvalue:
-                        newlines.append('%s%s \\\n' % (intentspc, item))
-                    newlines.append('%s"\n' % intentspc)
-            else:
-                newlines.append('%s "%s"\n' % (varset_start, newvalue))
+                    newlines.append('%s "%s"\n' % (varset_new, newvalue))
             return True
         else:
             # Put the old lines back where they were
             newlines.extend(varlines)
-            return False
+            # If newlines was touched by the function, we'll need to return True
+            return changed
 
-    with open(meta_file, 'r') as f:
-        for line in f:
-            if in_var:
-                value = line.rstrip()
-                varlines.append(line)
-                full_value += value[:-1]
-                if value.endswith('"') or value.endswith("'"):
-                    full_value = full_value[:-1]
-                    if handle_var_end():
-                        updated = True
-                    in_var = None
+    checkspc = False
+
+    for line in meta_lines:
+        if in_var:
+            value = line.rstrip()
+            varlines.append(line)
+            if in_var.endswith('()'):
+                full_value += '\n' + value
             else:
-                matched = False
-                for (varname, var_re) in var_res.iteritems():
-                    if var_re.match(line):
-                        splitvalue = line.split('"', 1)
-                        varset_start = splitvalue[0].rstrip()
-                        value = splitvalue[1].rstrip()
-                        if value.endswith('\\'):
-                            value = value[:-1]
-                        full_value = value
-                        varlines = [line]
-                        in_var = varname
-                        if value.endswith('"') or value.endswith("'"):
-                            full_value = full_value[:-1]
-                            if handle_var_end():
-                                updated = True
-                            in_var = None
-                        matched = True
-                        break
-                if not matched:
-                    newlines.append(line)
+                full_value += value[:-1]
+            if value.endswith(var_end):
+                if in_var.endswith('()'):
+                    if full_value.count('{') - full_value.count('}') >= 0:
+                        continue
+                full_value = full_value[:-1]
+                if handle_var_end():
+                    updated = True
+                    checkspc = True
+                in_var = None
+        else:
+            skip = False
+            for (varname, var_re) in var_res.iteritems():
+                res = var_re.match(line)
+                if res:
+                    isfunc = varname.endswith('()')
+                    if isfunc:
+                        splitvalue = line.split('{', 1)
+                        var_end = '}'
+                    else:
+                        var_end = res.groups()[-1]
+                        splitvalue = line.split(var_end, 1)
+                    varset_start = splitvalue[0].rstrip()
+                    value = splitvalue[1].rstrip()
+                    if not isfunc and value.endswith('\\'):
+                        value = value[:-1]
+                    full_value = value
+                    varlines = [line]
+                    in_var = res.group(1)
+                    if isfunc:
+                        in_var += '()'
+                    if value.endswith(var_end):
+                        full_value = full_value[:-1]
+                        if handle_var_end():
+                            updated = True
+                            checkspc = True
+                        in_var = None
+                    skip = True
+                    break
+            if not skip:
+                if checkspc:
+                    checkspc = False
+                    if newlines[-1] == '\n' and line == '\n':
+                        # Squash blank line if there are two consecutive blanks after a removal
+                        continue
+                newlines.append(line)
+    return (updated, newlines)
+
+
+def edit_metadata_file(meta_file, variables, varfunc):
+    """Edit a recipe or config file and modify one or more specified
+    variable values set in the file using a specified callback function.
+    The file is only written to if the value(s) actually change.
+    This is basically the file version of edit_metadata(), see that
+    function's description for parameter/usage information.
+    Returns True if the file was written to, False otherwise.
+    """
+    with open(meta_file, 'r') as f:
+        (updated, newlines) = edit_metadata(f, variables, varfunc)
     if updated:
         with open(meta_file, 'w') as f:
             f.writelines(newlines)
+    return updated
+
 
 def edit_bblayers_conf(bblayers_conf, add, remove):
     """Edit bblayers.conf, adding and/or removing layers"""
@@ -1070,7 +1192,7 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
     # Need to use a list here because we can't set non-local variables from a callback in python 2.x
     bblayercalls = []
 
-    def handle_bblayers(varname, origvalue):
+    def handle_bblayers(varname, origvalue, op, newlines):
         bblayercalls.append(varname)
         updated = False
         bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
@@ -1094,9 +1216,9 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
                     notadded.append(addlayer)
 
         if updated:
-            return (bblayers, 2, False)
+            return (bblayers, None, 2, False)
         else:
-            return (origvalue, 2, False)
+            return (origvalue, None, 2, False)
 
     edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers)