summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/tests/utils.py271
-rw-r--r--bitbake/lib/bb/utils.py238
2 files changed, 451 insertions, 58 deletions
diff --git a/bitbake/lib/bb/tests/utils.py b/bitbake/lib/bb/tests/utils.py
index 6e09858e51..9171509a62 100644
--- a/bitbake/lib/bb/tests/utils.py
+++ b/bitbake/lib/bb/tests/utils.py
@@ -22,6 +22,7 @@
22import unittest 22import unittest
23import bb 23import bb
24import os 24import os
25import tempfile
25 26
26class VerCmpString(unittest.TestCase): 27class VerCmpString(unittest.TestCase):
27 28
@@ -105,3 +106,273 @@ class Path(unittest.TestCase):
105 for arg1, correctresult in checkitems: 106 for arg1, correctresult in checkitems:
106 result = bb.utils._check_unsafe_delete_path(arg1) 107 result = bb.utils._check_unsafe_delete_path(arg1)
107 self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult)) 108 self.assertEqual(result, correctresult, '_check_unsafe_delete_path("%s") != %s' % (arg1, correctresult))
109
110
111class EditMetadataFile(unittest.TestCase):
112 _origfile = """
113# A comment
114HELLO = "oldvalue"
115
116THIS = "that"
117
118# Another comment
119NOCHANGE = "samevalue"
120OTHER = 'anothervalue'
121
122MULTILINE = "a1 \\
123 a2 \\
124 a3"
125
126MULTILINE2 := " \\
127 b1 \\
128 b2 \\
129 b3 \\
130 "
131
132
133MULTILINE3 = " \\
134 c1 \\
135 c2 \\
136 c3 \\
137"
138
139do_functionname() {
140 command1 ${VAL1} ${VAL2}
141 command2 ${VAL3} ${VAL4}
142}
143"""
144 def _testeditfile(self, varvalues, compareto, dummyvars=None):
145 if dummyvars is None:
146 dummyvars = []
147 with tempfile.NamedTemporaryFile('w', delete=False) as tf:
148 tf.write(self._origfile)
149 tf.close()
150 try:
151 varcalls = []
152 def handle_file(varname, origvalue, op, newlines):
153 self.assertIn(varname, varvalues, 'Callback called for variable %s not in the list!' % varname)
154 self.assertNotIn(varname, dummyvars, 'Callback called for variable %s in dummy list!' % varname)
155 varcalls.append(varname)
156 return varvalues[varname]
157
158 bb.utils.edit_metadata_file(tf.name, varvalues.keys(), handle_file)
159 with open(tf.name) as f:
160 modfile = f.readlines()
161 # Ensure the output matches the expected output
162 self.assertEqual(compareto.splitlines(True), modfile)
163 # Ensure the callback function was called for every variable we asked for
164 # (plus allow testing behaviour when a requested variable is not present)
165 self.assertEqual(sorted(varvalues.keys()), sorted(varcalls + dummyvars))
166 finally:
167 os.remove(tf.name)
168
169
170 def test_edit_metadata_file_nochange(self):
171 # Test file doesn't get modified with nothing to do
172 self._testeditfile({}, self._origfile)
173 # Test file doesn't get modified with only dummy variables
174 self._testeditfile({'DUMMY1': ('should_not_set', None, 0, True),
175 'DUMMY2': ('should_not_set_again', None, 0, True)}, self._origfile, dummyvars=['DUMMY1', 'DUMMY2'])
176 # Test file doesn't get modified with some the same values
177 self._testeditfile({'THIS': ('that', None, 0, True),
178 'OTHER': ('anothervalue', None, 0, True),
179 'MULTILINE3': (' c1 c2 c3', None, 4, False)}, self._origfile)
180
181 def test_edit_metadata_file_1(self):
182
183 newfile1 = """
184# A comment
185HELLO = "newvalue"
186
187THIS = "that"
188
189# Another comment
190NOCHANGE = "samevalue"
191OTHER = 'anothervalue'
192
193MULTILINE = "a1 \\
194 a2 \\
195 a3"
196
197MULTILINE2 := " \\
198 b1 \\
199 b2 \\
200 b3 \\
201 "
202
203
204MULTILINE3 = " \\
205 c1 \\
206 c2 \\
207 c3 \\
208"
209
210do_functionname() {
211 command1 ${VAL1} ${VAL2}
212 command2 ${VAL3} ${VAL4}
213}
214"""
215 self._testeditfile({'HELLO': ('newvalue', None, 4, True)}, newfile1)
216
217
218 def test_edit_metadata_file_2(self):
219
220 newfile2 = """
221# A comment
222HELLO = "oldvalue"
223
224THIS = "that"
225
226# Another comment
227NOCHANGE = "samevalue"
228OTHER = 'anothervalue'
229
230MULTILINE = " \\
231 d1 \\
232 d2 \\
233 d3 \\
234 "
235
236MULTILINE2 := " \\
237 b1 \\
238 b2 \\
239 b3 \\
240 "
241
242
243MULTILINE3 = "nowsingle"
244
245do_functionname() {
246 command1 ${VAL1} ${VAL2}
247 command2 ${VAL3} ${VAL4}
248}
249"""
250 self._testeditfile({'MULTILINE': (['d1','d2','d3'], None, 4, False),
251 'MULTILINE3': ('nowsingle', None, 4, True),
252 'NOTPRESENT': (['a', 'b'], None, 4, False)}, newfile2, dummyvars=['NOTPRESENT'])
253
254
255 def test_edit_metadata_file_3(self):
256
257 newfile3 = """
258# A comment
259HELLO = "oldvalue"
260
261# Another comment
262NOCHANGE = "samevalue"
263OTHER = "yetanothervalue"
264
265MULTILINE = "e1 \\
266 e2 \\
267 e3 \\
268 "
269
270MULTILINE2 := "f1 \\
271\tf2 \\
272\t"
273
274
275MULTILINE3 = " \\
276 c1 \\
277 c2 \\
278 c3 \\
279"
280
281do_functionname() {
282 othercommand_one a b c
283 othercommand_two d e f
284}
285"""
286
287 self._testeditfile({'do_functionname()': (['othercommand_one a b c', 'othercommand_two d e f'], None, 4, False),
288 'MULTILINE2': (['f1', 'f2'], None, '\t', True),
289 'MULTILINE': (['e1', 'e2', 'e3'], None, -1, True),
290 'THIS': (None, None, 0, False),
291 'OTHER': ('yetanothervalue', None, 0, True)}, newfile3)
292
293
294 def test_edit_metadata_file_4(self):
295
296 newfile4 = """
297# A comment
298HELLO = "oldvalue"
299
300THIS = "that"
301
302# Another comment
303OTHER = 'anothervalue'
304
305MULTILINE = "a1 \\
306 a2 \\
307 a3"
308
309MULTILINE2 := " \\
310 b1 \\
311 b2 \\
312 b3 \\
313 "
314
315
316"""
317
318 self._testeditfile({'NOCHANGE': (None, None, 0, False),
319 'MULTILINE3': (None, None, 0, False),
320 'THIS': ('that', None, 0, False),
321 'do_functionname()': (None, None, 0, False)}, newfile4)
322
323
324 def test_edit_metadata(self):
325 newfile5 = """
326# A comment
327HELLO = "hithere"
328
329# A new comment
330THIS += "that"
331
332# Another comment
333NOCHANGE = "samevalue"
334OTHER = 'anothervalue'
335
336MULTILINE = "a1 \\
337 a2 \\
338 a3"
339
340MULTILINE2 := " \\
341 b1 \\
342 b2 \\
343 b3 \\
344 "
345
346
347MULTILINE3 = " \\
348 c1 \\
349 c2 \\
350 c3 \\
351"
352
353NEWVAR = "value"
354
355do_functionname() {
356 command1 ${VAL1} ${VAL2}
357 command2 ${VAL3} ${VAL4}
358}
359"""
360
361
362 def handle_var(varname, origvalue, op, newlines):
363 if varname == 'THIS':
364 newlines.append('# A new comment\n')
365 elif varname == 'do_functionname()':
366 newlines.append('NEWVAR = "value"\n')
367 newlines.append('\n')
368 valueitem = varvalues.get(varname, None)
369 if valueitem:
370 return valueitem
371 else:
372 return (origvalue, op, 0, True)
373
374 varvalues = {'HELLO': ('hithere', None, 0, True), 'THIS': ('that', '+=', 0, True)}
375 varlist = ['HELLO', 'THIS', 'do_functionname()']
376 (updated, newlines) = bb.utils.edit_metadata(self._origfile.splitlines(True), varlist, handle_var)
377 self.assertTrue(updated, 'List should be updated but isn\'t')
378 self.assertEqual(newlines, newfile5.splitlines(True))
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index 0db7e56651..988b845a4a 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -963,14 +963,62 @@ def exec_flat_python_func(func, *args, **kwargs):
963 bb.utils.better_exec(comp, context, code, '<string>') 963 bb.utils.better_exec(comp, context, code, '<string>')
964 return context['retval'] 964 return context['retval']
965 965
966def edit_metadata_file(meta_file, variables, func): 966def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
967 """Edit a recipe or config file and modify one or more specified 967 """Edit lines from a recipe or config file and modify one or more
968 variable values set in the file using a specified callback function. 968 specified variable values set in the file using a specified callback
969 The file is only written to if the value(s) actually change. 969 function. Lines are expected to have trailing newlines.
970 Parameters:
971 meta_lines: lines from the file; can be a list or an iterable
972 (e.g. file pointer)
973 variables: a list of variable names to look for. Functions
974 may also be specified, but must be specified with '()' at
975 the end of the name. Note that the function doesn't have
976 any intrinsic understanding of _append, _prepend, _remove,
977 or overrides, so these are considered as part of the name.
978 These values go into a regular expression, so regular
979 expression syntax is allowed.
980 varfunc: callback function called for every variable matching
981 one of the entries in the variables parameter. The function
982 should take four arguments:
983 varname: name of variable matched
984 origvalue: current value in file
985 op: the operator (e.g. '+=')
986 newlines: list of lines up to this point. You can use
987 this to prepend lines before this variable setting
988 if you wish.
989 and should return a three-element tuple:
990 newvalue: new value to substitute in, or None to drop
991 the variable setting entirely. (If the removal
992 results in two consecutive blank lines, one of the
993 blank lines will also be dropped).
994 newop: the operator to use - if you specify None here,
995 the original operation will be used.
996 indent: number of spaces to indent multi-line entries,
997 or -1 to indent up to the level of the assignment
998 and opening quote, or a string to use as the indent.
999 minbreak: True to allow the first element of a
1000 multi-line value to continue on the same line as
1001 the assignment, False to indent before the first
1002 element.
1003 match_overrides: True to match items with _overrides on the end,
1004 False otherwise
1005 Returns a tuple:
1006 updated:
1007 True if changes were made, False otherwise.
1008 newlines:
1009 Lines after processing
970 """ 1010 """
1011
971 var_res = {} 1012 var_res = {}
1013 if match_overrides:
1014 override_re = '(_[a-zA-Z0-9-_$(){}]+)?'
1015 else:
1016 override_re = ''
972 for var in variables: 1017 for var in variables:
973 var_res[var] = re.compile(r'^%s[ \t]*[?+]*=' % var) 1018 if var.endswith('()'):
1019 var_res[var] = re.compile('^(%s%s)[ \\t]*\([ \\t]*\)[ \\t]*{' % (var[:-2].rstrip(), override_re))
1020 else:
1021 var_res[var] = re.compile('^(%s%s)[ \\t]*[?+:.]*=[+.]*[ \\t]*(["\'])' % (var, override_re))
974 1022
975 updated = False 1023 updated = False
976 varset_start = '' 1024 varset_start = ''
@@ -978,70 +1026,144 @@ def edit_metadata_file(meta_file, variables, func):
978 newlines = [] 1026 newlines = []
979 in_var = None 1027 in_var = None
980 full_value = '' 1028 full_value = ''
1029 var_end = ''
981 1030
982 def handle_var_end(): 1031 def handle_var_end():
983 (newvalue, indent, minbreak) = func(in_var, full_value) 1032 prerun_newlines = newlines[:]
984 if newvalue != full_value: 1033 op = varset_start[len(in_var):].strip()
985 if isinstance(newvalue, list): 1034 (newvalue, newop, indent, minbreak) = varfunc(in_var, full_value, op, newlines)
986 intentspc = ' ' * indent 1035 changed = (prerun_newlines != newlines)
987 if minbreak: 1036
988 # First item on first line 1037 if newvalue is None:
989 if len(newvalue) == 1: 1038 # Drop the value
990 newlines.append('%s "%s"\n' % (varset_start, newvalue[0])) 1039 return True
1040 elif newvalue != full_value or (newop not in [None, op]):
1041 if newop not in [None, op]:
1042 # Callback changed the operator
1043 varset_new = "%s %s" % (in_var, newop)
1044 else:
1045 varset_new = varset_start
1046
1047 if isinstance(indent, (int, long)):
1048 if indent == -1:
1049 indentspc = ' ' * (len(varset_new) + 2)
1050 else:
1051 indentspc = ' ' * indent
1052 else:
1053 indentspc = indent
1054 if in_var.endswith('()'):
1055 # A function definition
1056 if isinstance(newvalue, list):
1057 newlines.append('%s {\n%s%s\n}\n' % (varset_new, indentspc, ('\n%s' % indentspc).join(newvalue)))
1058 else:
1059 if not newvalue.startswith('\n'):
1060 newvalue = '\n' + newvalue
1061 if not newvalue.endswith('\n'):
1062 newvalue = newvalue + '\n'
1063 newlines.append('%s {%s}\n' % (varset_new, newvalue))
1064 else:
1065 # Normal variable
1066 if isinstance(newvalue, list):
1067 if not newvalue:
1068 # Empty list -> empty string
1069 newlines.append('%s ""\n' % varset_new)
1070 elif minbreak:
1071 # First item on first line
1072 if len(newvalue) == 1:
1073 newlines.append('%s "%s"\n' % (varset_new, newvalue[0]))
1074 else:
1075 newlines.append('%s "%s \\\n' % (varset_new, newvalue[0]))
1076 for item in newvalue[1:]:
1077 newlines.append('%s%s \\\n' % (indentspc, item))
1078 newlines.append('%s"\n' % indentspc)
991 else: 1079 else:
992 newlines.append('%s "%s\\\n' % (varset_start, newvalue[0])) 1080 # No item on first line
993 for item in newvalue[1:]: 1081 newlines.append('%s " \\\n' % varset_new)
994 newlines.append('%s%s \\\n' % (intentspc, item)) 1082 for item in newvalue:
1083 newlines.append('%s%s \\\n' % (indentspc, item))
995 newlines.append('%s"\n' % indentspc) 1084 newlines.append('%s"\n' % indentspc)
996 else: 1085 else:
997 # No item on first line 1086 newlines.append('%s "%s"\n' % (varset_new, newvalue))
998 newlines.append('%s " \\\n' % varset_start)
999 for item in newvalue:
1000 newlines.append('%s%s \\\n' % (intentspc, item))
1001 newlines.append('%s"\n' % intentspc)
1002 else:
1003 newlines.append('%s "%s"\n' % (varset_start, newvalue))
1004 return True 1087 return True
1005 else: 1088 else:
1006 # Put the old lines back where they were 1089 # Put the old lines back where they were
1007 newlines.extend(varlines) 1090 newlines.extend(varlines)
1008 return False 1091 # If newlines was touched by the function, we'll need to return True
1092 return changed
1009 1093
1010 with open(meta_file, 'r') as f: 1094 checkspc = False
1011 for line in f: 1095
1012 if in_var: 1096 for line in meta_lines:
1013 value = line.rstrip() 1097 if in_var:
1014 varlines.append(line) 1098 value = line.rstrip()
1015 full_value += value[:-1] 1099 varlines.append(line)
1016 if value.endswith('"') or value.endswith("'"): 1100 if in_var.endswith('()'):
1017 full_value = full_value[:-1] 1101 full_value += '\n' + value
1018 if handle_var_end():
1019 updated = True
1020 in_var = None
1021 else: 1102 else:
1022 matched = False 1103 full_value += value[:-1]
1023 for (varname, var_re) in var_res.iteritems(): 1104 if value.endswith(var_end):
1024 if var_re.match(line): 1105 if in_var.endswith('()'):
1025 splitvalue = line.split('"', 1) 1106 if full_value.count('{') - full_value.count('}') >= 0:
1026 varset_start = splitvalue[0].rstrip() 1107 continue
1027 value = splitvalue[1].rstrip() 1108 full_value = full_value[:-1]
1028 if value.endswith('\\'): 1109 if handle_var_end():
1029 value = value[:-1] 1110 updated = True
1030 full_value = value 1111 checkspc = True
1031 varlines = [line] 1112 in_var = None
1032 in_var = varname 1113 else:
1033 if value.endswith('"') or value.endswith("'"): 1114 skip = False
1034 full_value = full_value[:-1] 1115 for (varname, var_re) in var_res.iteritems():
1035 if handle_var_end(): 1116 res = var_re.match(line)
1036 updated = True 1117 if res:
1037 in_var = None 1118 isfunc = varname.endswith('()')
1038 matched = True 1119 if isfunc:
1039 break 1120 splitvalue = line.split('{', 1)
1040 if not matched: 1121 var_end = '}'
1041 newlines.append(line) 1122 else:
1123 var_end = res.groups()[-1]
1124 splitvalue = line.split(var_end, 1)
1125 varset_start = splitvalue[0].rstrip()
1126 value = splitvalue[1].rstrip()
1127 if not isfunc and value.endswith('\\'):
1128 value = value[:-1]
1129 full_value = value
1130 varlines = [line]
1131 in_var = res.group(1)
1132 if isfunc:
1133 in_var += '()'
1134 if value.endswith(var_end):
1135 full_value = full_value[:-1]
1136 if handle_var_end():
1137 updated = True
1138 checkspc = True
1139 in_var = None
1140 skip = True
1141 break
1142 if not skip:
1143 if checkspc:
1144 checkspc = False
1145 if newlines[-1] == '\n' and line == '\n':
1146 # Squash blank line if there are two consecutive blanks after a removal
1147 continue
1148 newlines.append(line)
1149 return (updated, newlines)
1150
1151
1152def edit_metadata_file(meta_file, variables, varfunc):
1153 """Edit a recipe or config file and modify one or more specified
1154 variable values set in the file using a specified callback function.
1155 The file is only written to if the value(s) actually change.
1156 This is basically the file version of edit_metadata(), see that
1157 function's description for parameter/usage information.
1158 Returns True if the file was written to, False otherwise.
1159 """
1160 with open(meta_file, 'r') as f:
1161 (updated, newlines) = edit_metadata(f, variables, varfunc)
1042 if updated: 1162 if updated:
1043 with open(meta_file, 'w') as f: 1163 with open(meta_file, 'w') as f:
1044 f.writelines(newlines) 1164 f.writelines(newlines)
1165 return updated
1166
1045 1167
1046def edit_bblayers_conf(bblayers_conf, add, remove): 1168def edit_bblayers_conf(bblayers_conf, add, remove):
1047 """Edit bblayers.conf, adding and/or removing layers""" 1169 """Edit bblayers.conf, adding and/or removing layers"""
@@ -1070,7 +1192,7 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
1070 # Need to use a list here because we can't set non-local variables from a callback in python 2.x 1192 # Need to use a list here because we can't set non-local variables from a callback in python 2.x
1071 bblayercalls = [] 1193 bblayercalls = []
1072 1194
1073 def handle_bblayers(varname, origvalue): 1195 def handle_bblayers(varname, origvalue, op, newlines):
1074 bblayercalls.append(varname) 1196 bblayercalls.append(varname)
1075 updated = False 1197 updated = False
1076 bblayers = [remove_trailing_sep(x) for x in origvalue.split()] 1198 bblayers = [remove_trailing_sep(x) for x in origvalue.split()]
@@ -1094,9 +1216,9 @@ def edit_bblayers_conf(bblayers_conf, add, remove):
1094 notadded.append(addlayer) 1216 notadded.append(addlayer)
1095 1217
1096 if updated: 1218 if updated:
1097 return (bblayers, 2, False) 1219 return (bblayers, None, 2, False)
1098 else: 1220 else:
1099 return (origvalue, 2, False) 1221 return (origvalue, None, 2, False)
1100 1222
1101 edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers) 1223 edit_metadata_file(bblayers_conf, ['BBLAYERS'], handle_bblayers)
1102 1224