From aefc80c02ed6d4de5768723d86ce80520387e1d3 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Mon, 18 May 2015 16:08:36 +0100 Subject: bitbake: lib/bb/utils: fix and extend edit_metadata_file() Fix several bugs and add some useful enhancements to make this into a more generic metadata editing function: * Support modifying function values (name must be specified ending with "()") * Support dropping values by returning None as the new value * Split out edit_metadata() function to provide same functionality on a list/iterable * Pass operation to callback and allow function to return them * Pass current output lines to callback so they can be modified * Fix handling of single-quoted values * Handle :=, =+, .=, and =. operators * Support arbitrary indent string * Support indenting by length of assignment (by specifying -1) * Fix typo in variablename - intentspc -> indentspc * Expand function docstring to cover arguments / usage * Add a parameter to enable matching names with overrides applied * Add some bitbake-selftest tests Note that this does change the expected signature of the callback function. The only known caller is in lib/bb/utils.py itself; I doubt anyone else has made extensive use of this function yet. (Bitbake rev: 20059e4d5ab9bf0f32c781ccb208da3c95818018) Signed-off-by: Paul Eggleton Signed-off-by: Richard Purdie --- bitbake/lib/bb/utils.py | 238 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 180 insertions(+), 58 deletions(-) (limited to 'bitbake/lib/bb/utils.py') diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py index 0db7e56651..988b845a4a 100644 --- a/bitbake/lib/bb/utils.py +++ b/bitbake/lib/bb/utils.py @@ -963,14 +963,62 @@ def exec_flat_python_func(func, *args, **kwargs): bb.utils.better_exec(comp, context, code, '') return context['retval'] -def edit_metadata_file(meta_file, variables, func): - """Edit a recipe or config file and modify one or more specified - variable values set in the file using a specified callback function. - The file is only written to if the value(s) actually change. +def edit_metadata(meta_lines, variables, varfunc, match_overrides=False): + """Edit lines from a recipe or config file and modify one or more + specified variable values set in the file using a specified callback + function. Lines are expected to have trailing newlines. + Parameters: + meta_lines: lines from the file; can be a list or an iterable + (e.g. file pointer) + variables: a list of variable names to look for. Functions + may also be specified, but must be specified with '()' at + the end of the name. Note that the function doesn't have + any intrinsic understanding of _append, _prepend, _remove, + or overrides, so these are considered as part of the name. + These values go into a regular expression, so regular + expression syntax is allowed. + varfunc: callback function called for every variable matching + one of the entries in the variables parameter. The function + should take four arguments: + varname: name of variable matched + origvalue: current value in file + op: the operator (e.g. '+=') + newlines: list of lines up to this point. You can use + this to prepend lines before this variable setting + if you wish. + and should return a three-element tuple: + newvalue: new value to substitute in, or None to drop + the variable setting entirely. (If the removal + results in two consecutive blank lines, one of the + blank lines will also be dropped). + newop: the operator to use - if you specify None here, + the original operation will be used. + indent: number of spaces to indent multi-line entries, + or -1 to indent up to the level of the assignment + and opening quote, or a string to use as the indent. + minbreak: True to allow the first element of a + multi-line value to continue on the same line as + the assignment, False to indent before the first + element. + match_overrides: True to match items with _overrides on the end, + False otherwise + Returns a tuple: + updated: + True if changes were made, False otherwise. + newlines: + Lines after processing """ + var_res = {} + if match_overrides: + override_re = '(_[a-zA-Z0-9-_$(){}]+)?' + else: + override_re = '' for var in variables: - var_res[var] = re.compile(r'^%s[ \t]*[?+]*=' % var) + if var.endswith('()'): + var_res[var] = re.compile('^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re)) + else: + var_res[var] = re.compile('^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re)) updated = False varset_start = '' @@ -978,70 +1026,144 @@ def edit_metadata_file(meta_file, variables, func): newlines = [] in_var = None full_value = '' + var_end = '' def handle_var_end(): - (newvalue, indent, minbreak) = func(in_var, full_value) - if newvalue != full_value: - if isinstance(newvalue, list): - intentspc = ' ' * indent - if minbreak: - # First item on first line - if len(newvalue) == 1: - newlines.append('%s "%s"\n' % (varset_start, newvalue[0])) + prerun_newlines = newlines[:] + op = varset_start[len(in_var):].strip() + (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines) + changed = (prerun_newlines != newlines) + + if newvalue is None: + # Drop the value + return True + elif newvalue != full_value or (newop not in [None, op]): + if newop not in [None, op]: + # Callback changed the operator + varset_new = "%s %s" % (in_var, newop) + else: + varset_new = varset_start + + if isinstance(indent, (int, long)): + if indent == -1: + indentspc = ' ' * (len(varset_new) + 2) + else: + indentspc = ' ' * indent + else: + indentspc = indent + if in_var.endswith('()'): + # A function definition + if isinstance(newvalue, list): + newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue))) + else: + if not newvalue.startswith('\n'): + newvalue = '\n' + newvalue + if not newvalue.endswith('\n'): + newvalue = newvalue + '\n' + newlines.append('%s {%s}\n' % (varset_new, newvalue)) + else: + # Normal variable + if isinstance(newvalue, list): + if not newvalue: + # Empty list -> empty string + newlines.append('%s ""\n' % varset_new) + elif minbreak: + # First item on first line + if len(newvalue) == 1: + newlines.append('%s "%s"\n' % (varset_new, newvalue[0])) + else: + newlines.append('%s "%s \\\n' % (varset_new, newvalue[0])) + for item in newvalue[1:]: + newlines.append('%s%s \\\n' % (indentspc, item)) + newlines.append('%s"\n' % indentspc) else: - newlines.append('%s "%s\\\n' % (varset_start, newvalue[0])) - for item in newvalue[1:]: - newlines.append('%s%s \\\n' % (intentspc, item)) + # No item on first line + newlines.append('%s " \\\n' % varset_new) + for item in newvalue: + newlines.append('%s%s \\\n' % (indentspc, item)) newlines.append('%s"\n' % indentspc) else: - # No item on first line - newlines.append('%s " \\\n' % varset_start) - for item in newvalue: - newlines.append('%s%s \\\n' % (intentspc, item)) - newlines.append('%s"\n' % intentspc) - else: - newlines.append('%s "%s"\n' % (varset_start, newvalue)) + newlines.append('%s "%s"\n' % (varset_new, newvalue)) return True else: # Put the old lines back where they were newlines.extend(varlines) - return False + # If newlines was touched by the function, we'll need to return True + return changed - with open(meta_file, 'r') as f: - for line in f: - if in_var: - value = line.rstrip() - varlines.append(line) - full_value += value[:-1] - if value.endswith('"') or value.endswith("'"): - full_value = full_value[:-1] - if handle_var_end(): - updated = True - in_var = None + checkspc = False + + for line in meta_lines: + if in_var: + value = line.rstrip() + varlines.append(line) + if in_var.endswith('()'): + full_value += '\n' + value else: - matched = False - for (varname, var_re) in var_res.iteritems(): - if var_re.match(line): - splitvalue = line.split('"', 1) - varset_start = splitvalue[0].rstrip() - value = splitvalue[1].rstrip() - if value.endswith('\\'): - value = value[:-1] - full_value = value - varlines = [line] - in_var = varname - if value.endswith('"') or value.endswith("'"): - full_value = full_value[:-1] - if handle_var_end(): - updated = True - in_var = None - matched = True - break - if not matched: - newlines.append(line) + full_value += value[:-1] + if value.endswith(var_end): + if in_var.endswith('()'): + if full_value.count('{') - full_value.count('}') >= 0: + continue + full_value = full_value[:-1] + if handle_var_end(): + updated = True + checkspc = True + in_var = None + else: + skip = False + for (varname, var_re) in var_res.iteritems(): + res = var_re.match(line) + if res: + isfunc = varname.endswith('()') + if isfunc: + splitvalue = line.split('{', 1) + var_end = '}' + else: + var_end = res.groups()[-1] + splitvalue = line.split(var_end, 1) + varset_start = splitvalue[0].rstrip() + value = splitvalue[1].rstrip() + if not isfunc and value.endswith('\\'): + value = value[:-1] + full_value = value + varlines = [line] + in_var = res.group(1) + if isfunc: + in_var += '()' + if value.endswith(var_end): + full_value = full_value[:-1] + if handle_var_end(): + updated = True + checkspc = True + in_var = None + skip = True + break + if not skip: + if checkspc: + checkspc = False + if newlines[-1] == '\n' and line == '\n': + # Squash blank line if there are two consecutive blanks after a removal + continue + newlines.append(line) + return (updated, newlines) + + +def edit_metadata_file(meta_file, variables, varfunc): + """Edit a recipe or config file and modify one or more specified + variable values set in the file using a specified callback function. + The file is only written to if the value(s) actually change. + This is basically the file version of edit_metadata(), see that + function's description for parameter/usage information. + Returns True if the file was written to, False otherwise. + """ + with open(meta_file, 'r') as f: + (updated, newlines) = edit_metadata(f, variables, varfunc) if updated: with open(meta_file, 'w') as f: f.writelines(newlines) + return updated + def edit_bblayers_conf(bblayers_conf, add, remove): """Edit bblayers.conf, adding and/or removing layers""" @@ -1070,7 +1192,7 @@ def edit_bblayers_conf(bblayers_conf, add, remove): # Need to use a list here because we can't set non-local variables from a callback in python 2.x bblayercalls = [] - def handle_bblayers(varname, origvalue): + def handle_bblayers(varname, origvalue, op, newlines): bblayercalls.append(varname) updated = False bblayers = [remove_trailing_sep(x) for x in origvalue.split()] @@ -1094,9 +1216,9 @@ def edit_bblayers_conf(bblayers_conf, add, remove): notadded.append(addlayer) if updated: - return (bblayers, 2, False) + return (bblayers, None, 2, False) else: - return (origvalue, 2, False) + return (origvalue, None, 2, False) edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers) -- cgit v1.2.3-54-g00ecf