diff options
Diffstat (limited to 'scripts/lib/checklayer/__init__.py')
-rw-r--r-- | scripts/lib/checklayer/__init__.py | 392 |
1 files changed, 392 insertions, 0 deletions
diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py new file mode 100644 index 0000000000..6c2b86a79a --- /dev/null +++ b/scripts/lib/checklayer/__init__.py | |||
@@ -0,0 +1,392 @@ | |||
1 | # Yocto Project layer check tool | ||
2 | # | ||
3 | # Copyright (C) 2017 Intel Corporation | ||
4 | # Released under the MIT license (see COPYING.MIT) | ||
5 | |||
6 | import os | ||
7 | import re | ||
8 | import subprocess | ||
9 | from enum import Enum | ||
10 | |||
11 | import bb.tinfoil | ||
12 | |||
13 | class LayerType(Enum): | ||
14 | BSP = 0 | ||
15 | DISTRO = 1 | ||
16 | SOFTWARE = 2 | ||
17 | ERROR_NO_LAYER_CONF = 98 | ||
18 | ERROR_BSP_DISTRO = 99 | ||
19 | |||
20 | def _get_configurations(path): | ||
21 | configs = [] | ||
22 | |||
23 | for f in os.listdir(path): | ||
24 | file_path = os.path.join(path, f) | ||
25 | if os.path.isfile(file_path) and f.endswith('.conf'): | ||
26 | configs.append(f[:-5]) # strip .conf | ||
27 | return configs | ||
28 | |||
29 | def _get_layer_collections(layer_path, lconf=None, data=None): | ||
30 | import bb.parse | ||
31 | import bb.data | ||
32 | |||
33 | if lconf is None: | ||
34 | lconf = os.path.join(layer_path, 'conf', 'layer.conf') | ||
35 | |||
36 | if data is None: | ||
37 | ldata = bb.data.init() | ||
38 | bb.parse.init_parser(ldata) | ||
39 | else: | ||
40 | ldata = data.createCopy() | ||
41 | |||
42 | ldata.setVar('LAYERDIR', layer_path) | ||
43 | try: | ||
44 | ldata = bb.parse.handle(lconf, ldata, include=True) | ||
45 | except BaseException as exc: | ||
46 | raise LayerError(exc) | ||
47 | ldata.expandVarref('LAYERDIR') | ||
48 | |||
49 | collections = (ldata.getVar('BBFILE_COLLECTIONS', True) or '').split() | ||
50 | if not collections: | ||
51 | name = os.path.basename(layer_path) | ||
52 | collections = [name] | ||
53 | |||
54 | collections = {c: {} for c in collections} | ||
55 | for name in collections: | ||
56 | priority = ldata.getVar('BBFILE_PRIORITY_%s' % name, True) | ||
57 | pattern = ldata.getVar('BBFILE_PATTERN_%s' % name, True) | ||
58 | depends = ldata.getVar('LAYERDEPENDS_%s' % name, True) | ||
59 | collections[name]['priority'] = priority | ||
60 | collections[name]['pattern'] = pattern | ||
61 | collections[name]['depends'] = depends | ||
62 | |||
63 | return collections | ||
64 | |||
65 | def _detect_layer(layer_path): | ||
66 | """ | ||
67 | Scans layer directory to detect what type of layer | ||
68 | is BSP, Distro or Software. | ||
69 | |||
70 | Returns a dictionary with layer name, type and path. | ||
71 | """ | ||
72 | |||
73 | layer = {} | ||
74 | layer_name = os.path.basename(layer_path) | ||
75 | |||
76 | layer['name'] = layer_name | ||
77 | layer['path'] = layer_path | ||
78 | layer['conf'] = {} | ||
79 | |||
80 | if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')): | ||
81 | layer['type'] = LayerType.ERROR_NO_LAYER_CONF | ||
82 | return layer | ||
83 | |||
84 | machine_conf = os.path.join(layer_path, 'conf', 'machine') | ||
85 | distro_conf = os.path.join(layer_path, 'conf', 'distro') | ||
86 | |||
87 | is_bsp = False | ||
88 | is_distro = False | ||
89 | |||
90 | if os.path.isdir(machine_conf): | ||
91 | machines = _get_configurations(machine_conf) | ||
92 | if machines: | ||
93 | is_bsp = True | ||
94 | |||
95 | if os.path.isdir(distro_conf): | ||
96 | distros = _get_configurations(distro_conf) | ||
97 | if distros: | ||
98 | is_distro = True | ||
99 | |||
100 | if is_bsp and is_distro: | ||
101 | layer['type'] = LayerType.ERROR_BSP_DISTRO | ||
102 | elif is_bsp: | ||
103 | layer['type'] = LayerType.BSP | ||
104 | layer['conf']['machines'] = machines | ||
105 | elif is_distro: | ||
106 | layer['type'] = LayerType.DISTRO | ||
107 | layer['conf']['distros'] = distros | ||
108 | else: | ||
109 | layer['type'] = LayerType.SOFTWARE | ||
110 | |||
111 | layer['collections'] = _get_layer_collections(layer['path']) | ||
112 | |||
113 | return layer | ||
114 | |||
115 | def detect_layers(layer_directories, no_auto): | ||
116 | layers = [] | ||
117 | |||
118 | for directory in layer_directories: | ||
119 | directory = os.path.realpath(directory) | ||
120 | if directory[-1] == '/': | ||
121 | directory = directory[0:-1] | ||
122 | |||
123 | if no_auto: | ||
124 | conf_dir = os.path.join(directory, 'conf') | ||
125 | if os.path.isdir(conf_dir): | ||
126 | layer = _detect_layer(directory) | ||
127 | if layer: | ||
128 | layers.append(layer) | ||
129 | else: | ||
130 | for root, dirs, files in os.walk(directory): | ||
131 | dir_name = os.path.basename(root) | ||
132 | conf_dir = os.path.join(root, 'conf') | ||
133 | if os.path.isdir(conf_dir): | ||
134 | layer = _detect_layer(root) | ||
135 | if layer: | ||
136 | layers.append(layer) | ||
137 | |||
138 | return layers | ||
139 | |||
140 | def _find_layer_depends(depend, layers): | ||
141 | for layer in layers: | ||
142 | for collection in layer['collections']: | ||
143 | if depend == collection: | ||
144 | return layer | ||
145 | return None | ||
146 | |||
147 | def add_layer_dependencies(bblayersconf, layer, layers, logger): | ||
148 | def recurse_dependencies(depends, layer, layers, logger, ret = []): | ||
149 | logger.debug('Processing dependencies %s for layer %s.' % \ | ||
150 | (depends, layer['name'])) | ||
151 | |||
152 | for depend in depends.split(): | ||
153 | # core (oe-core) is suppose to be provided | ||
154 | if depend == 'core': | ||
155 | continue | ||
156 | |||
157 | layer_depend = _find_layer_depends(depend, layers) | ||
158 | if not layer_depend: | ||
159 | logger.error('Layer %s depends on %s and isn\'t found.' % \ | ||
160 | (layer['name'], depend)) | ||
161 | ret = None | ||
162 | continue | ||
163 | |||
164 | # We keep processing, even if ret is None, this allows us to report | ||
165 | # multiple errors at once | ||
166 | if ret is not None and layer_depend not in ret: | ||
167 | ret.append(layer_depend) | ||
168 | |||
169 | # Recursively process... | ||
170 | if 'collections' not in layer_depend: | ||
171 | continue | ||
172 | |||
173 | for collection in layer_depend['collections']: | ||
174 | collect_deps = layer_depend['collections'][collection]['depends'] | ||
175 | if not collect_deps: | ||
176 | continue | ||
177 | ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret) | ||
178 | |||
179 | return ret | ||
180 | |||
181 | layer_depends = [] | ||
182 | for collection in layer['collections']: | ||
183 | depends = layer['collections'][collection]['depends'] | ||
184 | if not depends: | ||
185 | continue | ||
186 | |||
187 | layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends) | ||
188 | |||
189 | # Note: [] (empty) is allowed, None is not! | ||
190 | if layer_depends is None: | ||
191 | return False | ||
192 | else: | ||
193 | # Don't add a layer that is already present. | ||
194 | added = set() | ||
195 | output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8') | ||
196 | for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE): | ||
197 | added.add(path) | ||
198 | |||
199 | for layer_depend in layer_depends: | ||
200 | name = layer_depend['name'] | ||
201 | path = layer_depend['path'] | ||
202 | if path in added: | ||
203 | continue | ||
204 | else: | ||
205 | added.add(path) | ||
206 | logger.info('Adding layer dependency %s' % name) | ||
207 | with open(bblayersconf, 'a+') as f: | ||
208 | f.write("\nBBLAYERS += \"%s\"\n" % path) | ||
209 | return True | ||
210 | |||
211 | def add_layer(bblayersconf, layer, layers, logger): | ||
212 | logger.info('Adding layer %s' % layer['name']) | ||
213 | with open(bblayersconf, 'a+') as f: | ||
214 | f.write("\nBBLAYERS += \"%s\"\n" % layer['path']) | ||
215 | |||
216 | return True | ||
217 | |||
218 | def check_command(error_msg, cmd): | ||
219 | ''' | ||
220 | Run a command under a shell, capture stdout and stderr in a single stream, | ||
221 | throw an error when command returns non-zero exit code. Returns the output. | ||
222 | ''' | ||
223 | |||
224 | p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | ||
225 | output, _ = p.communicate() | ||
226 | if p.returncode: | ||
227 | msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8')) | ||
228 | raise RuntimeError(msg) | ||
229 | return output | ||
230 | |||
231 | def get_signatures(builddir, failsafe=False, machine=None): | ||
232 | import re | ||
233 | |||
234 | # some recipes needs to be excluded like meta-world-pkgdata | ||
235 | # because a layer can add recipes to a world build so signature | ||
236 | # will be change | ||
237 | exclude_recipes = ('meta-world-pkgdata',) | ||
238 | |||
239 | sigs = {} | ||
240 | tune2tasks = {} | ||
241 | |||
242 | cmd = '' | ||
243 | if machine: | ||
244 | cmd += 'MACHINE=%s ' % machine | ||
245 | cmd += 'bitbake ' | ||
246 | if failsafe: | ||
247 | cmd += '-k ' | ||
248 | cmd += '-S none world' | ||
249 | sigs_file = os.path.join(builddir, 'locked-sigs.inc') | ||
250 | if os.path.exists(sigs_file): | ||
251 | os.unlink(sigs_file) | ||
252 | try: | ||
253 | check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.', | ||
254 | cmd) | ||
255 | except RuntimeError as ex: | ||
256 | if failsafe and os.path.exists(sigs_file): | ||
257 | # Ignore the error here. Most likely some recipes active | ||
258 | # in a world build lack some dependencies. There is a | ||
259 | # separate test_machine_world_build which exposes the | ||
260 | # failure. | ||
261 | pass | ||
262 | else: | ||
263 | raise | ||
264 | |||
265 | sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$") | ||
266 | tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") | ||
267 | current_tune = None | ||
268 | with open(sigs_file, 'r') as f: | ||
269 | for line in f.readlines(): | ||
270 | line = line.strip() | ||
271 | t = tune_regex.search(line) | ||
272 | if t: | ||
273 | current_tune = t.group('tune') | ||
274 | s = sig_regex.match(line) | ||
275 | if s: | ||
276 | exclude = False | ||
277 | for er in exclude_recipes: | ||
278 | (recipe, task) = s.group('task').split(':') | ||
279 | if er == recipe: | ||
280 | exclude = True | ||
281 | break | ||
282 | if exclude: | ||
283 | continue | ||
284 | |||
285 | sigs[s.group('task')] = s.group('hash') | ||
286 | tune2tasks.setdefault(current_tune, []).append(s.group('task')) | ||
287 | |||
288 | if not sigs: | ||
289 | raise RuntimeError('Can\'t load signatures from %s' % sigs_file) | ||
290 | |||
291 | return (sigs, tune2tasks) | ||
292 | |||
293 | def get_depgraph(targets=['world'], failsafe=False): | ||
294 | ''' | ||
295 | Returns the dependency graph for the given target(s). | ||
296 | The dependency graph is taken directly from DepTreeEvent. | ||
297 | ''' | ||
298 | depgraph = None | ||
299 | with bb.tinfoil.Tinfoil() as tinfoil: | ||
300 | tinfoil.prepare(config_only=False) | ||
301 | tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted']) | ||
302 | if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'): | ||
303 | raise RuntimeError('starting generateDepTreeEvent failed') | ||
304 | while True: | ||
305 | event = tinfoil.wait_event(timeout=1000) | ||
306 | if event: | ||
307 | if isinstance(event, bb.command.CommandFailed): | ||
308 | raise RuntimeError('Generating dependency information failed: %s' % event.error) | ||
309 | elif isinstance(event, bb.command.CommandCompleted): | ||
310 | break | ||
311 | elif isinstance(event, bb.event.NoProvider): | ||
312 | if failsafe: | ||
313 | # The event is informational, we will get information about the | ||
314 | # remaining dependencies eventually and thus can ignore this | ||
315 | # here like we do in get_signatures(), if desired. | ||
316 | continue | ||
317 | if event._reasons: | ||
318 | raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons)) | ||
319 | else: | ||
320 | raise RuntimeError('Nothing provides %s.' % (event._item)) | ||
321 | elif isinstance(event, bb.event.DepTreeGenerated): | ||
322 | depgraph = event._depgraph | ||
323 | |||
324 | if depgraph is None: | ||
325 | raise RuntimeError('Could not retrieve the depgraph.') | ||
326 | return depgraph | ||
327 | |||
328 | def compare_signatures(old_sigs, curr_sigs): | ||
329 | ''' | ||
330 | Compares the result of two get_signatures() calls. Returns None if no | ||
331 | problems found, otherwise a string that can be used as additional | ||
332 | explanation in self.fail(). | ||
333 | ''' | ||
334 | # task -> (old signature, new signature) | ||
335 | sig_diff = {} | ||
336 | for task in old_sigs: | ||
337 | if task in curr_sigs and \ | ||
338 | old_sigs[task] != curr_sigs[task]: | ||
339 | sig_diff[task] = (old_sigs[task], curr_sigs[task]) | ||
340 | |||
341 | if not sig_diff: | ||
342 | return None | ||
343 | |||
344 | # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures() | ||
345 | # uses <pn>:<taskname>. Need to convert sometimes. The output follows | ||
346 | # the convention from get_signatures() because that seems closer to | ||
347 | # normal bitbake output. | ||
348 | def sig2graph(task): | ||
349 | pn, taskname = task.rsplit(':', 1) | ||
350 | return pn + '.' + taskname | ||
351 | def graph2sig(task): | ||
352 | pn, taskname = task.rsplit('.', 1) | ||
353 | return pn + ':' + taskname | ||
354 | depgraph = get_depgraph(failsafe=True) | ||
355 | depends = depgraph['tdepends'] | ||
356 | |||
357 | # If a task A has a changed signature, but none of its | ||
358 | # dependencies, then we need to report it because it is | ||
359 | # the one which introduces a change. Any task depending on | ||
360 | # A (directly or indirectly) will also have a changed | ||
361 | # signature, but we don't need to report it. It might have | ||
362 | # its own changes, which will become apparent once the | ||
363 | # issues that we do report are fixed and the test gets run | ||
364 | # again. | ||
365 | sig_diff_filtered = [] | ||
366 | for task, (old_sig, new_sig) in sig_diff.items(): | ||
367 | deps_tainted = False | ||
368 | for dep in depends.get(sig2graph(task), ()): | ||
369 | if graph2sig(dep) in sig_diff: | ||
370 | deps_tainted = True | ||
371 | break | ||
372 | if not deps_tainted: | ||
373 | sig_diff_filtered.append((task, old_sig, new_sig)) | ||
374 | |||
375 | msg = [] | ||
376 | msg.append('%d signatures changed, initial differences (first hash before, second after):' % | ||
377 | len(sig_diff)) | ||
378 | for diff in sorted(sig_diff_filtered): | ||
379 | recipe, taskname = diff[0].rsplit(':', 1) | ||
380 | cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \ | ||
381 | (recipe, taskname, diff[1], diff[2]) | ||
382 | msg.append(' %s: %s -> %s' % diff) | ||
383 | msg.append(' %s' % cmd) | ||
384 | try: | ||
385 | output = check_command('Determining signature difference failed.', | ||
386 | cmd).decode('utf-8') | ||
387 | except RuntimeError as error: | ||
388 | output = str(error) | ||
389 | if output: | ||
390 | msg.extend([' ' + line for line in output.splitlines()]) | ||
391 | msg.append('') | ||
392 | return '\n'.join(msg) | ||