summaryrefslogtreecommitdiffstats
path: root/scripts/lib/checklayer
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2025-11-07 13:31:53 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2025-11-07 13:31:53 +0000
commit8c22ff0d8b70d9b12f0487ef696a7e915b9e3173 (patch)
treeefdc32587159d0050a69009bdf2330a531727d95 /scripts/lib/checklayer
parentd412d2747595c1cc4a5e3ca975e3adc31b2f7891 (diff)
downloadpoky-8c22ff0d8b70d9b12f0487ef696a7e915b9e3173.tar.gz
The poky repository master branch is no longer being updated.
You can either: a) switch to individual clones of bitbake, openembedded-core, meta-yocto and yocto-docs b) use the new bitbake-setup You can find information about either approach in our documentation: https://docs.yoctoproject.org/ Note that "poky" the distro setting is still available in meta-yocto as before and we continue to use and maintain that. Long live Poky! Some further information on the background of this change can be found in: https://lists.openembedded.org/g/openembedded-architecture/message/2179 Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/checklayer')
-rw-r--r--scripts/lib/checklayer/__init__.py466
-rw-r--r--scripts/lib/checklayer/case.py9
-rw-r--r--scripts/lib/checklayer/cases/__init__.py0
-rw-r--r--scripts/lib/checklayer/cases/bsp.py206
-rw-r--r--scripts/lib/checklayer/cases/common.py135
-rw-r--r--scripts/lib/checklayer/cases/distro.py28
-rw-r--r--scripts/lib/checklayer/context.py17
7 files changed, 0 insertions, 861 deletions
diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py
deleted file mode 100644
index 86aadf39a6..0000000000
--- a/scripts/lib/checklayer/__init__.py
+++ /dev/null
@@ -1,466 +0,0 @@
1# Yocto Project layer check tool
2#
3# Copyright (C) 2017 Intel Corporation
4#
5# SPDX-License-Identifier: MIT
6#
7
8import os
9import re
10import subprocess
11from enum import Enum
12
13import bb.tinfoil
14
15class LayerType(Enum):
16 BSP = 0
17 DISTRO = 1
18 SOFTWARE = 2
19 CORE = 3
20 ERROR_NO_LAYER_CONF = 98
21 ERROR_BSP_DISTRO = 99
22
23def _get_configurations(path):
24 configs = []
25
26 for f in os.listdir(path):
27 file_path = os.path.join(path, f)
28 if os.path.isfile(file_path) and f.endswith('.conf'):
29 configs.append(f[:-5]) # strip .conf
30 return configs
31
32def _get_layer_collections(layer_path, lconf=None, data=None):
33 import bb.parse
34 import bb.data
35
36 if lconf is None:
37 lconf = os.path.join(layer_path, 'conf', 'layer.conf')
38
39 if data is None:
40 ldata = bb.data.init()
41 bb.parse.init_parser(ldata)
42 else:
43 ldata = data.createCopy()
44
45 ldata.setVar('LAYERDIR', layer_path)
46 try:
47 ldata = bb.parse.handle(lconf, ldata, include=True, baseconfig=True)
48 except:
49 raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path)
50 ldata.expandVarref('LAYERDIR')
51
52 collections = (ldata.getVar('BBFILE_COLLECTIONS') or '').split()
53 if not collections:
54 name = os.path.basename(layer_path)
55 collections = [name]
56
57 collections = {c: {} for c in collections}
58 for name in collections:
59 priority = ldata.getVar('BBFILE_PRIORITY_%s' % name)
60 pattern = ldata.getVar('BBFILE_PATTERN_%s' % name)
61 depends = ldata.getVar('LAYERDEPENDS_%s' % name)
62 compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name)
63 try:
64 depDict = bb.utils.explode_dep_versions2(depends or "")
65 except bb.utils.VersionStringException as vse:
66 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (name, str(vse)))
67
68 collections[name]['priority'] = priority
69 collections[name]['pattern'] = pattern
70 collections[name]['depends'] = ' '.join(depDict.keys())
71 collections[name]['compat'] = compat
72
73 return collections
74
75def _detect_layer(layer_path):
76 """
77 Scans layer directory to detect what type of layer
78 is BSP, Distro or Software.
79
80 Returns a dictionary with layer name, type and path.
81 """
82
83 layer = {}
84 layer_name = os.path.basename(layer_path)
85
86 layer['name'] = layer_name
87 layer['path'] = layer_path
88 layer['conf'] = {}
89
90 if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')):
91 layer['type'] = LayerType.ERROR_NO_LAYER_CONF
92 return layer
93
94 machine_conf = os.path.join(layer_path, 'conf', 'machine')
95 distro_conf = os.path.join(layer_path, 'conf', 'distro')
96
97 is_bsp = False
98 is_distro = False
99
100 if os.path.isdir(machine_conf):
101 machines = _get_configurations(machine_conf)
102 if machines:
103 is_bsp = True
104
105 if os.path.isdir(distro_conf):
106 distros = _get_configurations(distro_conf)
107 if distros:
108 is_distro = True
109
110 layer['collections'] = _get_layer_collections(layer['path'])
111
112 if layer_name == "meta" and "core" in layer['collections']:
113 layer['type'] = LayerType.CORE
114 layer['conf']['machines'] = machines
115 layer['conf']['distros'] = distros
116 elif is_bsp and is_distro:
117 layer['type'] = LayerType.ERROR_BSP_DISTRO
118 elif is_bsp:
119 layer['type'] = LayerType.BSP
120 layer['conf']['machines'] = machines
121 elif is_distro:
122 layer['type'] = LayerType.DISTRO
123 layer['conf']['distros'] = distros
124 else:
125 layer['type'] = LayerType.SOFTWARE
126
127 return layer
128
129def detect_layers(layer_directories, no_auto):
130 layers = []
131
132 for directory in layer_directories:
133 directory = os.path.realpath(directory)
134 if directory[-1] == '/':
135 directory = directory[0:-1]
136
137 if no_auto:
138 conf_dir = os.path.join(directory, 'conf')
139 if os.path.isdir(conf_dir):
140 layer = _detect_layer(directory)
141 if layer:
142 layers.append(layer)
143 else:
144 for root, dirs, files in os.walk(directory):
145 dir_name = os.path.basename(root)
146 conf_dir = os.path.join(root, 'conf')
147 if os.path.isdir(conf_dir):
148 layer = _detect_layer(root)
149 if layer:
150 layers.append(layer)
151
152 return layers
153
154def _find_layer(depend, layers):
155 for layer in layers:
156 if 'collections' not in layer:
157 continue
158
159 for collection in layer['collections']:
160 if depend == collection:
161 return layer
162 return None
163
164def sanity_check_layers(layers, logger):
165 """
166 Check that we didn't find duplicate collection names, as the layer that will
167 be used is non-deterministic. The precise check is duplicate collections
168 with different patterns, as the same pattern being repeated won't cause
169 problems.
170 """
171 import collections
172
173 passed = True
174 seen = collections.defaultdict(set)
175 for layer in layers:
176 for name, data in layer.get("collections", {}).items():
177 seen[name].add(data["pattern"])
178
179 for name, patterns in seen.items():
180 if len(patterns) > 1:
181 passed = False
182 logger.error("Collection %s found multiple times: %s" % (name, ", ".join(patterns)))
183 return passed
184
185def get_layer_dependencies(layer, layers, logger):
186 def recurse_dependencies(depends, layer, layers, logger, ret = []):
187 logger.debug('Processing dependencies %s for layer %s.' % \
188 (depends, layer['name']))
189
190 for depend in depends.split():
191 # core (oe-core) is suppose to be provided
192 if depend == 'core':
193 continue
194
195 layer_depend = _find_layer(depend, layers)
196 if not layer_depend:
197 logger.error('Layer %s depends on %s and isn\'t found.' % \
198 (layer['name'], depend))
199 ret = None
200 continue
201
202 # We keep processing, even if ret is None, this allows us to report
203 # multiple errors at once
204 if ret is not None and layer_depend not in ret:
205 ret.append(layer_depend)
206 else:
207 # we might have processed this dependency already, in which case
208 # we should not do it again (avoid recursive loop)
209 continue
210
211 # Recursively process...
212 if 'collections' not in layer_depend:
213 continue
214
215 for collection in layer_depend['collections']:
216 collect_deps = layer_depend['collections'][collection]['depends']
217 if not collect_deps:
218 continue
219 ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret)
220
221 return ret
222
223 layer_depends = []
224 for collection in layer['collections']:
225 depends = layer['collections'][collection]['depends']
226 if not depends:
227 continue
228
229 layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
230
231 # Note: [] (empty) is allowed, None is not!
232 return layer_depends
233
234def add_layer_dependencies(bblayersconf, layer, layers, logger):
235
236 layer_depends = get_layer_dependencies(layer, layers, logger)
237 if layer_depends is None:
238 return False
239 else:
240 add_layers(bblayersconf, layer_depends, logger)
241
242 return True
243
244def add_layers(bblayersconf, layers, logger):
245 # Don't add a layer that is already present.
246 added = set()
247 output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8')
248 for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE):
249 added.add(path)
250
251 with open(bblayersconf, 'a+') as f:
252 for layer in layers:
253 logger.info('Adding layer %s' % layer['name'])
254 name = layer['name']
255 path = layer['path']
256 if path in added:
257 logger.info('%s is already in %s' % (name, bblayersconf))
258 else:
259 added.add(path)
260 f.write("\nBBLAYERS += \"%s\"\n" % path)
261 return True
262
263def check_bblayers(bblayersconf, layer_path, logger):
264 '''
265 If layer_path found in BBLAYERS return True
266 '''
267 import bb.parse
268 import bb.data
269
270 ldata = bb.parse.handle(bblayersconf, bb.data.init(), include=True)
271 for bblayer in (ldata.getVar('BBLAYERS') or '').split():
272 if os.path.normpath(bblayer) == os.path.normpath(layer_path):
273 return True
274
275 return False
276
277def check_command(error_msg, cmd, cwd=None):
278 '''
279 Run a command under a shell, capture stdout and stderr in a single stream,
280 throw an error when command returns non-zero exit code. Returns the output.
281 '''
282
283 p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd)
284 output, _ = p.communicate()
285 if p.returncode:
286 msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8'))
287 raise RuntimeError(msg)
288 return output
289
290def get_signatures(builddir, failsafe=False, machine=None, extravars=None):
291 import re
292
293 # some recipes needs to be excluded like meta-world-pkgdata
294 # because a layer can add recipes to a world build so signature
295 # will be change
296 exclude_recipes = ('meta-world-pkgdata',)
297
298 sigs = {}
299 tune2tasks = {}
300
301 cmd = 'BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" '
302 if extravars:
303 cmd += extravars
304 cmd += ' '
305 if machine:
306 cmd += 'MACHINE=%s ' % machine
307 cmd += 'bitbake '
308 if failsafe:
309 cmd += '-k '
310 cmd += '-S lockedsigs world'
311 sigs_file = os.path.join(builddir, 'locked-sigs.inc')
312 if os.path.exists(sigs_file):
313 os.unlink(sigs_file)
314 try:
315 check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.',
316 cmd, builddir)
317 except RuntimeError as ex:
318 if failsafe and os.path.exists(sigs_file):
319 # Ignore the error here. Most likely some recipes active
320 # in a world build lack some dependencies. There is a
321 # separate test_machine_world_build which exposes the
322 # failure.
323 pass
324 else:
325 raise
326
327 sig_regex = re.compile(r"^(?P<task>.*:.*):(?P<hash>.*) .$")
328 tune_regex = re.compile(r"(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
329 current_tune = None
330 with open(sigs_file, 'r') as f:
331 for line in f.readlines():
332 line = line.strip()
333 t = tune_regex.search(line)
334 if t:
335 current_tune = t.group('tune')
336 s = sig_regex.match(line)
337 if s:
338 exclude = False
339 for er in exclude_recipes:
340 (recipe, task) = s.group('task').split(':')
341 if er == recipe:
342 exclude = True
343 break
344 if exclude:
345 continue
346
347 sigs[s.group('task')] = s.group('hash')
348 tune2tasks.setdefault(current_tune, []).append(s.group('task'))
349
350 if not sigs:
351 raise RuntimeError('Can\'t load signatures from %s' % sigs_file)
352
353 return (sigs, tune2tasks)
354
355def get_depgraph(targets=['world'], failsafe=False):
356 '''
357 Returns the dependency graph for the given target(s).
358 The dependency graph is taken directly from DepTreeEvent.
359 '''
360 depgraph = None
361 with bb.tinfoil.Tinfoil() as tinfoil:
362 tinfoil.prepare(config_only=False)
363 tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted'])
364 if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'):
365 raise RuntimeError('starting generateDepTreeEvent failed')
366 while True:
367 event = tinfoil.wait_event(timeout=1000)
368 if event:
369 if isinstance(event, bb.command.CommandFailed):
370 raise RuntimeError('Generating dependency information failed: %s' % event.error)
371 elif isinstance(event, bb.command.CommandCompleted):
372 break
373 elif isinstance(event, bb.event.NoProvider):
374 if failsafe:
375 # The event is informational, we will get information about the
376 # remaining dependencies eventually and thus can ignore this
377 # here like we do in get_signatures(), if desired.
378 continue
379 if event._reasons:
380 raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons))
381 else:
382 raise RuntimeError('Nothing provides %s.' % (event._item))
383 elif isinstance(event, bb.event.DepTreeGenerated):
384 depgraph = event._depgraph
385
386 if depgraph is None:
387 raise RuntimeError('Could not retrieve the depgraph.')
388 return depgraph
389
390def compare_signatures(old_sigs, curr_sigs):
391 '''
392 Compares the result of two get_signatures() calls. Returns None if no
393 problems found, otherwise a string that can be used as additional
394 explanation in self.fail().
395 '''
396 # task -> (old signature, new signature)
397 sig_diff = {}
398 for task in old_sigs:
399 if task in curr_sigs and \
400 old_sigs[task] != curr_sigs[task]:
401 sig_diff[task] = (old_sigs[task], curr_sigs[task])
402
403 if not sig_diff:
404 return None
405
406 # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures()
407 # uses <pn>:<taskname>. Need to convert sometimes. The output follows
408 # the convention from get_signatures() because that seems closer to
409 # normal bitbake output.
410 def sig2graph(task):
411 pn, taskname = task.rsplit(':', 1)
412 return pn + '.' + taskname
413 def graph2sig(task):
414 pn, taskname = task.rsplit('.', 1)
415 return pn + ':' + taskname
416 depgraph = get_depgraph(failsafe=True)
417 depends = depgraph['tdepends']
418
419 # If a task A has a changed signature, but none of its
420 # dependencies, then we need to report it because it is
421 # the one which introduces a change. Any task depending on
422 # A (directly or indirectly) will also have a changed
423 # signature, but we don't need to report it. It might have
424 # its own changes, which will become apparent once the
425 # issues that we do report are fixed and the test gets run
426 # again.
427 sig_diff_filtered = []
428 for task, (old_sig, new_sig) in sig_diff.items():
429 deps_tainted = False
430 for dep in depends.get(sig2graph(task), ()):
431 if graph2sig(dep) in sig_diff:
432 deps_tainted = True
433 break
434 if not deps_tainted:
435 sig_diff_filtered.append((task, old_sig, new_sig))
436
437 msg = []
438 msg.append('%d signatures changed, initial differences (first hash before, second after):' %
439 len(sig_diff))
440 for diff in sorted(sig_diff_filtered):
441 recipe, taskname = diff[0].rsplit(':', 1)
442 cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \
443 (recipe, taskname, diff[1], diff[2])
444 msg.append(' %s: %s -> %s' % diff)
445 msg.append(' %s' % cmd)
446 try:
447 output = check_command('Determining signature difference failed.',
448 cmd).decode('utf-8')
449 except RuntimeError as error:
450 output = str(error)
451 if output:
452 msg.extend([' ' + line for line in output.splitlines()])
453 msg.append('')
454 return '\n'.join(msg)
455
456
457def get_git_toplevel(directory):
458 """
459 Try and find the top of the git repository that directory might be in.
460 Returns the top-level directory, or None.
461 """
462 cmd = ["git", "-C", directory, "rev-parse", "--show-toplevel"]
463 try:
464 return subprocess.check_output(cmd, text=True).strip()
465 except:
466 return None
diff --git a/scripts/lib/checklayer/case.py b/scripts/lib/checklayer/case.py
deleted file mode 100644
index fa9dee384e..0000000000
--- a/scripts/lib/checklayer/case.py
+++ /dev/null
@@ -1,9 +0,0 @@
1# Copyright (C) 2017 Intel Corporation
2#
3# SPDX-License-Identifier: MIT
4#
5
6from oeqa.core.case import OETestCase
7
8class OECheckLayerTestCase(OETestCase):
9 pass
diff --git a/scripts/lib/checklayer/cases/__init__.py b/scripts/lib/checklayer/cases/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
--- a/scripts/lib/checklayer/cases/__init__.py
+++ /dev/null
diff --git a/scripts/lib/checklayer/cases/bsp.py b/scripts/lib/checklayer/cases/bsp.py
deleted file mode 100644
index b76163fb56..0000000000
--- a/scripts/lib/checklayer/cases/bsp.py
+++ /dev/null
@@ -1,206 +0,0 @@
1# Copyright (C) 2017 Intel Corporation
2#
3# SPDX-License-Identifier: MIT
4#
5
6import unittest
7
8from checklayer import LayerType, get_signatures, check_command, get_depgraph
9from checklayer.case import OECheckLayerTestCase
10
11class BSPCheckLayer(OECheckLayerTestCase):
12 @classmethod
13 def setUpClass(self):
14 if self.tc.layer['type'] not in (LayerType.BSP, LayerType.CORE):
15 raise unittest.SkipTest("BSPCheckLayer: Layer %s isn't BSP one." %\
16 self.tc.layer['name'])
17
18 def test_bsp_defines_machines(self):
19 self.assertTrue(self.tc.layer['conf']['machines'],
20 "Layer is BSP but doesn't defines machines.")
21
22 def test_bsp_no_set_machine(self):
23 from oeqa.utils.commands import get_bb_var
24
25 machine = get_bb_var('MACHINE')
26 self.assertEqual(self.td['bbvars']['MACHINE'], machine,
27 msg="Layer %s modified machine %s -> %s" % \
28 (self.tc.layer['name'], self.td['bbvars']['MACHINE'], machine))
29
30
31 def test_machine_world(self):
32 '''
33 "bitbake world" is expected to work regardless which machine is selected.
34 BSP layers sometimes break that by enabling a recipe for a certain machine
35 without checking whether that recipe actually can be built in the current
36 distro configuration (for example, OpenGL might not enabled).
37
38 This test iterates over all machines. It would be nicer to instantiate
39 it once per machine. It merely checks for errors during parse
40 time. It does not actually attempt to build anything.
41 '''
42
43 if not self.td['machines']:
44 self.skipTest('No machines set with --machines.')
45 msg = []
46 for machine in self.td['machines']:
47 # In contrast to test_machine_signatures() below, errors are fatal here.
48 try:
49 get_signatures(self.td['builddir'], failsafe=False, machine=machine)
50 except RuntimeError as ex:
51 msg.append(str(ex))
52 if msg:
53 msg.insert(0, 'The following machines broke a world build:')
54 self.fail('\n'.join(msg))
55
56 def test_machine_signatures(self):
57 '''
58 Selecting a machine may only affect the signature of tasks that are specific
59 to that machine. In other words, when MACHINE=A and MACHINE=B share a recipe
60 foo and the output of foo, then both machine configurations must build foo
61 in exactly the same way. Otherwise it is not possible to use both machines
62 in the same distribution.
63
64 This criteria can only be tested by testing different machines in combination,
65 i.e. one main layer, potentially several additional BSP layers and an explicit
66 choice of machines:
67 yocto-check-layer --additional-layers .../meta-intel --machines intel-corei7-64 imx6slevk -- .../meta-freescale
68 '''
69
70 if not self.td['machines']:
71 self.skipTest('No machines set with --machines.')
72
73 # Collect signatures for all machines that we are testing
74 # and merge that into a hash:
75 # tune -> task -> signature -> list of machines with that combination
76 #
77 # It is an error if any tune/task pair has more than one signature,
78 # because that implies that the machines that caused those different
79 # signatures do not agree on how to execute the task.
80 tunes = {}
81 # Preserve ordering of machines as chosen by the user.
82 for machine in self.td['machines']:
83 curr_sigs, tune2tasks = get_signatures(self.td['builddir'], failsafe=True, machine=machine)
84 # Invert the tune -> [tasks] mapping.
85 tasks2tune = {}
86 for tune, tasks in tune2tasks.items():
87 for task in tasks:
88 tasks2tune[task] = tune
89 for task, sighash in curr_sigs.items():
90 tunes.setdefault(tasks2tune[task], {}).setdefault(task, {}).setdefault(sighash, []).append(machine)
91
92 msg = []
93 pruned = 0
94 last_line_key = None
95 # do_fetch, do_unpack, ..., do_build
96 taskname_list = []
97 if tunes:
98 # The output below is most useful when we start with tasks that are at
99 # the bottom of the dependency chain, i.e. those that run first. If
100 # those tasks differ, the rest also does.
101 #
102 # To get an ordering of tasks, we do a topological sort of the entire
103 # depgraph for the base configuration, then on-the-fly flatten that list by stripping
104 # out the recipe names and removing duplicates. The base configuration
105 # is not necessarily representative, but should be close enough. Tasks
106 # that were not encountered get a default priority.
107 depgraph = get_depgraph()
108 depends = depgraph['tdepends']
109 WHITE = 1
110 GRAY = 2
111 BLACK = 3
112 color = {}
113 found = set()
114 def visit(task):
115 color[task] = GRAY
116 for dep in depends.get(task, ()):
117 if color.setdefault(dep, WHITE) == WHITE:
118 visit(dep)
119 color[task] = BLACK
120 pn, taskname = task.rsplit('.', 1)
121 if taskname not in found:
122 taskname_list.append(taskname)
123 found.add(taskname)
124 for task in depends.keys():
125 if color.setdefault(task, WHITE) == WHITE:
126 visit(task)
127
128 taskname_order = dict([(task, index) for index, task in enumerate(taskname_list) ])
129 def task_key(task):
130 pn, taskname = task.rsplit(':', 1)
131 return (pn, taskname_order.get(taskname, len(taskname_list)), taskname)
132
133 for tune in sorted(tunes.keys()):
134 tasks = tunes[tune]
135 # As for test_signatures it would be nicer to sort tasks
136 # by dependencies here, but that is harder because we have
137 # to report on tasks from different machines, which might
138 # have different dependencies. We resort to pruning the
139 # output by reporting only one task per recipe if the set
140 # of machines matches.
141 #
142 # "bitbake-diffsigs -t -s" is intelligent enough to print
143 # diffs recursively, so often it does not matter that much
144 # if we don't pick the underlying difference
145 # here. However, sometimes recursion fails
146 # (https://bugzilla.yoctoproject.org/show_bug.cgi?id=6428).
147 #
148 # To mitigate that a bit, we use a hard-coded ordering of
149 # tasks that represents how they normally run and prefer
150 # to print the ones that run first.
151 for task in sorted(tasks.keys(), key=task_key):
152 signatures = tasks[task]
153 # do_build can be ignored: it is know to have
154 # different signatures in some cases, for example in
155 # the allarch ca-certificates due to RDEPENDS=openssl.
156 # That particular dependency is marked via
157 # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up
158 # in the sstate signature hash because filtering it
159 # out would be hard and running do_build multiple
160 # times doesn't really matter.
161 if len(signatures.keys()) > 1 and \
162 not task.endswith(':do_build'):
163 # Error!
164 #
165 # Sort signatures by machines, because the hex values don't mean anything.
166 # => all-arch adwaita-icon-theme:do_build: 1234... (beaglebone, qemux86) != abcdf... (qemux86-64)
167 #
168 # Skip the line if it is covered already by the predecessor (same pn, same sets of machines).
169 pn, taskname = task.rsplit(':', 1)
170 next_line_key = (pn, sorted(signatures.values()))
171 if next_line_key != last_line_key:
172 line = ' %s %s: ' % (tune, task)
173 line += ' != '.join(['%s (%s)' % (signature, ', '.join([m for m in signatures[signature]])) for
174 signature in sorted(signatures.keys(), key=lambda s: signatures[s])])
175 last_line_key = next_line_key
176 msg.append(line)
177 # Randomly pick two mismatched signatures and remember how to invoke
178 # bitbake-diffsigs for them.
179 iterator = iter(signatures.items())
180 a = next(iterator)
181 b = next(iterator)
182 diffsig_machines = '(%s) != (%s)' % (', '.join(a[1]), ', '.join(b[1]))
183 diffsig_params = '-t %s %s -s %s %s' % (pn, taskname, a[0], b[0])
184 else:
185 pruned += 1
186
187 if msg:
188 msg.insert(0, 'The machines have conflicting signatures for some shared tasks:')
189 if pruned > 0:
190 msg.append('')
191 msg.append('%d tasks where not listed because some other task of the recipe already differed.' % pruned)
192 msg.append('It is likely that differences from different recipes also have the same root cause.')
193 msg.append('')
194 # Explain how to investigate...
195 msg.append('To investigate, run bitbake-diffsigs -t recipename taskname -s fromsig tosig.')
196 cmd = 'bitbake-diffsigs %s' % diffsig_params
197 msg.append('Example: %s in the last line' % diffsig_machines)
198 msg.append('Command: %s' % cmd)
199 # ... and actually do it automatically for that example, but without aborting
200 # when that fails.
201 try:
202 output = check_command('Comparing signatures failed.', cmd).decode('utf-8')
203 except RuntimeError as ex:
204 output = str(ex)
205 msg.extend([' ' + line for line in output.splitlines()])
206 self.fail('\n'.join(msg))
diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py
deleted file mode 100644
index ddead69a7b..0000000000
--- a/scripts/lib/checklayer/cases/common.py
+++ /dev/null
@@ -1,135 +0,0 @@
1# Copyright (C) 2017 Intel Corporation
2#
3# SPDX-License-Identifier: MIT
4#
5
6import glob
7import os
8import unittest
9import re
10from checklayer import get_signatures, LayerType, check_command, compare_signatures, get_git_toplevel
11from checklayer.case import OECheckLayerTestCase
12
13class CommonCheckLayer(OECheckLayerTestCase):
14 def test_readme(self):
15 if self.tc.layer['type'] == LayerType.CORE:
16 raise unittest.SkipTest("Core layer's README is top level")
17
18 # The top-level README file may have a suffix (like README.rst or README.txt).
19 readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*'))
20 self.assertTrue(len(readme_files) > 0,
21 msg="Layer doesn't contain a README file.")
22
23 # There might be more than one file matching the file pattern above
24 # (for example, README.rst and README-COPYING.rst). The one with the shortest
25 # name is considered the "main" one.
26 readme_file = sorted(readme_files)[0]
27 data = ''
28 with open(readme_file, 'r') as f:
29 data = f.read()
30 self.assertTrue(data,
31 msg="Layer contains a README file but it is empty.")
32
33 # If a layer's README references another README, then the checks below are not valid
34 if re.search('README', data, re.IGNORECASE):
35 return
36
37 self.assertIn('maintainer', data.lower())
38 self.assertIn('patch', data.lower())
39 # Check that there is an email address in the README
40 email_regex = re.compile(r"[^@]+@[^@]+")
41 self.assertTrue(email_regex.match(data))
42
43 def find_file_by_name(self, globs):
44 """
45 Utility function to find a file that matches the specified list of
46 globs, in either the layer directory itself or the repository top-level
47 directory.
48 """
49 directories = [self.tc.layer["path"]]
50 toplevel = get_git_toplevel(directories[0])
51 if toplevel:
52 directories.append(toplevel)
53
54 for path in directories:
55 for name in globs:
56 files = glob.glob(os.path.join(path, name))
57 if files:
58 return sorted(files)[0]
59 return None
60
61 def test_security(self):
62 """
63 Test that the layer has a SECURITY.md (or similar) file, either in the
64 layer itself or at the top of the containing git repository.
65 """
66 if self.tc.layer["type"] == LayerType.CORE:
67 raise unittest.SkipTest("Core layer's SECURITY is top level")
68
69 filename = self.find_file_by_name(("SECURITY", "SECURITY.*"))
70 self.assertTrue(filename, msg="Layer doesn't contain a SECURITY.md file.")
71
72 size = os.path.getsize(filename)
73 self.assertGreater(size, 0, msg=f"{filename} has no content.")
74
75 def test_parse(self):
76 check_command('Layer %s failed to parse.' % self.tc.layer['name'],
77 'bitbake -p')
78
79 def test_show_environment(self):
80 check_command('Layer %s failed to show environment.' % self.tc.layer['name'],
81 'bitbake -e')
82
83 def test_world(self):
84 '''
85 "bitbake world" is expected to work. test_signatures does not cover that
86 because it is more lenient and ignores recipes in a world build that
87 are not actually buildable, so here we fail when "bitbake -S none world"
88 fails.
89 '''
90 get_signatures(self.td['builddir'], failsafe=False)
91
92 def test_world_inherit_class(self):
93 '''
94 This also does "bitbake -S none world" along with inheriting "yocto-check-layer"
95 class, which can do additional per-recipe test cases.
96 '''
97 msg = []
98 try:
99 get_signatures(self.td['builddir'], failsafe=False, machine=None, extravars='BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS INHERIT" INHERIT="yocto-check-layer"')
100 except RuntimeError as ex:
101 msg.append(str(ex))
102 if msg:
103 msg.insert(0, 'Layer %s failed additional checks from yocto-check-layer.bbclass\nSee below log for specific recipe parsing errors:\n' % \
104 self.tc.layer['name'])
105 self.fail('\n'.join(msg))
106
107 def test_patches_upstream_status(self):
108 import sys
109 sys.path.append(os.path.join(sys.path[0], '../../../../meta/lib/'))
110 import oe.qa
111 patches = []
112 for dirpath, dirs, files in os.walk(self.tc.layer['path']):
113 for filename in files:
114 if filename.endswith(".patch"):
115 ppath = os.path.join(dirpath, filename)
116 if oe.qa.check_upstream_status(ppath):
117 patches.append(ppath)
118 self.assertEqual(len(patches), 0 , \
119 msg="Found following patches with malformed or missing upstream status:\n%s" % '\n'.join([str(patch) for patch in patches]))
120
121 def test_signatures(self):
122 if self.tc.layer['type'] == LayerType.SOFTWARE and \
123 not self.tc.test_software_layer_signatures:
124 raise unittest.SkipTest("Not testing for signature changes in a software layer %s." \
125 % self.tc.layer['name'])
126
127 curr_sigs, _ = get_signatures(self.td['builddir'], failsafe=True)
128 msg = compare_signatures(self.td['sigs'], curr_sigs)
129 if msg is not None:
130 self.fail('Adding layer %s changed signatures.\n%s' % (self.tc.layer['name'], msg))
131
132 def test_layerseries_compat(self):
133 for collection_name, collection_data in self.tc.layer['collections'].items():
134 self.assertTrue(collection_data['compat'], "Collection %s from layer %s does not set compatible oe-core versions via LAYERSERIES_COMPAT_collection." \
135 % (collection_name, self.tc.layer['name']))
diff --git a/scripts/lib/checklayer/cases/distro.py b/scripts/lib/checklayer/cases/distro.py
deleted file mode 100644
index a35332451c..0000000000
--- a/scripts/lib/checklayer/cases/distro.py
+++ /dev/null
@@ -1,28 +0,0 @@
1# Copyright (C) 2017 Intel Corporation
2#
3# SPDX-License-Identifier: MIT
4#
5
6import unittest
7
8from checklayer import LayerType
9from checklayer.case import OECheckLayerTestCase
10
11class DistroCheckLayer(OECheckLayerTestCase):
12 @classmethod
13 def setUpClass(self):
14 if self.tc.layer['type'] not in (LayerType.DISTRO, LayerType.CORE):
15 raise unittest.SkipTest("DistroCheckLayer: Layer %s isn't Distro one." %\
16 self.tc.layer['name'])
17
18 def test_distro_defines_distros(self):
19 self.assertTrue(self.tc.layer['conf']['distros'],
20 "Layer is BSP but doesn't defines machines.")
21
22 def test_distro_no_set_distros(self):
23 from oeqa.utils.commands import get_bb_var
24
25 distro = get_bb_var('DISTRO')
26 self.assertEqual(self.td['bbvars']['DISTRO'], distro,
27 msg="Layer %s modified distro %s -> %s" % \
28 (self.tc.layer['name'], self.td['bbvars']['DISTRO'], distro))
diff --git a/scripts/lib/checklayer/context.py b/scripts/lib/checklayer/context.py
deleted file mode 100644
index 4de8f668fd..0000000000
--- a/scripts/lib/checklayer/context.py
+++ /dev/null
@@ -1,17 +0,0 @@
1# Copyright (C) 2017 Intel Corporation
2#
3# SPDX-License-Identifier: MIT
4#
5
6import os
7import sys
8import glob
9import re
10
11from oeqa.core.context import OETestContext
12
13class CheckLayerTestContext(OETestContext):
14 def __init__(self, td=None, logger=None, layer=None, test_software_layer_signatures=True):
15 super(CheckLayerTestContext, self).__init__(td, logger)
16 self.layer = layer
17 self.test_software_layer_signatures = test_software_layer_signatures