diff options
| author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2025-11-07 13:31:53 +0000 |
|---|---|---|
| committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2025-11-07 13:31:53 +0000 |
| commit | 8c22ff0d8b70d9b12f0487ef696a7e915b9e3173 (patch) | |
| tree | efdc32587159d0050a69009bdf2330a531727d95 /scripts/lib/checklayer | |
| parent | d412d2747595c1cc4a5e3ca975e3adc31b2f7891 (diff) | |
| download | poky-8c22ff0d8b70d9b12f0487ef696a7e915b9e3173.tar.gz | |
The poky repository master branch is no longer being updated.
You can either:
a) switch to individual clones of bitbake, openembedded-core, meta-yocto and yocto-docs
b) use the new bitbake-setup
You can find information about either approach in our documentation:
https://docs.yoctoproject.org/
Note that "poky" the distro setting is still available in meta-yocto as
before and we continue to use and maintain that.
Long live Poky!
Some further information on the background of this change can be found
in: https://lists.openembedded.org/g/openembedded-architecture/message/2179
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/checklayer')
| -rw-r--r-- | scripts/lib/checklayer/__init__.py | 466 | ||||
| -rw-r--r-- | scripts/lib/checklayer/case.py | 9 | ||||
| -rw-r--r-- | scripts/lib/checklayer/cases/__init__.py | 0 | ||||
| -rw-r--r-- | scripts/lib/checklayer/cases/bsp.py | 206 | ||||
| -rw-r--r-- | scripts/lib/checklayer/cases/common.py | 135 | ||||
| -rw-r--r-- | scripts/lib/checklayer/cases/distro.py | 28 | ||||
| -rw-r--r-- | scripts/lib/checklayer/context.py | 17 |
7 files changed, 0 insertions, 861 deletions
diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py deleted file mode 100644 index 86aadf39a6..0000000000 --- a/scripts/lib/checklayer/__init__.py +++ /dev/null | |||
| @@ -1,466 +0,0 @@ | |||
| 1 | # Yocto Project layer check tool | ||
| 2 | # | ||
| 3 | # Copyright (C) 2017 Intel Corporation | ||
| 4 | # | ||
| 5 | # SPDX-License-Identifier: MIT | ||
| 6 | # | ||
| 7 | |||
| 8 | import os | ||
| 9 | import re | ||
| 10 | import subprocess | ||
| 11 | from enum import Enum | ||
| 12 | |||
| 13 | import bb.tinfoil | ||
| 14 | |||
| 15 | class LayerType(Enum): | ||
| 16 | BSP = 0 | ||
| 17 | DISTRO = 1 | ||
| 18 | SOFTWARE = 2 | ||
| 19 | CORE = 3 | ||
| 20 | ERROR_NO_LAYER_CONF = 98 | ||
| 21 | ERROR_BSP_DISTRO = 99 | ||
| 22 | |||
| 23 | def _get_configurations(path): | ||
| 24 | configs = [] | ||
| 25 | |||
| 26 | for f in os.listdir(path): | ||
| 27 | file_path = os.path.join(path, f) | ||
| 28 | if os.path.isfile(file_path) and f.endswith('.conf'): | ||
| 29 | configs.append(f[:-5]) # strip .conf | ||
| 30 | return configs | ||
| 31 | |||
| 32 | def _get_layer_collections(layer_path, lconf=None, data=None): | ||
| 33 | import bb.parse | ||
| 34 | import bb.data | ||
| 35 | |||
| 36 | if lconf is None: | ||
| 37 | lconf = os.path.join(layer_path, 'conf', 'layer.conf') | ||
| 38 | |||
| 39 | if data is None: | ||
| 40 | ldata = bb.data.init() | ||
| 41 | bb.parse.init_parser(ldata) | ||
| 42 | else: | ||
| 43 | ldata = data.createCopy() | ||
| 44 | |||
| 45 | ldata.setVar('LAYERDIR', layer_path) | ||
| 46 | try: | ||
| 47 | ldata = bb.parse.handle(lconf, ldata, include=True, baseconfig=True) | ||
| 48 | except: | ||
| 49 | raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path) | ||
| 50 | ldata.expandVarref('LAYERDIR') | ||
| 51 | |||
| 52 | collections = (ldata.getVar('BBFILE_COLLECTIONS') or '').split() | ||
| 53 | if not collections: | ||
| 54 | name = os.path.basename(layer_path) | ||
| 55 | collections = [name] | ||
| 56 | |||
| 57 | collections = {c: {} for c in collections} | ||
| 58 | for name in collections: | ||
| 59 | priority = ldata.getVar('BBFILE_PRIORITY_%s' % name) | ||
| 60 | pattern = ldata.getVar('BBFILE_PATTERN_%s' % name) | ||
| 61 | depends = ldata.getVar('LAYERDEPENDS_%s' % name) | ||
| 62 | compat = ldata.getVar('LAYERSERIES_COMPAT_%s' % name) | ||
| 63 | try: | ||
| 64 | depDict = bb.utils.explode_dep_versions2(depends or "") | ||
| 65 | except bb.utils.VersionStringException as vse: | ||
| 66 | bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (name, str(vse))) | ||
| 67 | |||
| 68 | collections[name]['priority'] = priority | ||
| 69 | collections[name]['pattern'] = pattern | ||
| 70 | collections[name]['depends'] = ' '.join(depDict.keys()) | ||
| 71 | collections[name]['compat'] = compat | ||
| 72 | |||
| 73 | return collections | ||
| 74 | |||
| 75 | def _detect_layer(layer_path): | ||
| 76 | """ | ||
| 77 | Scans layer directory to detect what type of layer | ||
| 78 | is BSP, Distro or Software. | ||
| 79 | |||
| 80 | Returns a dictionary with layer name, type and path. | ||
| 81 | """ | ||
| 82 | |||
| 83 | layer = {} | ||
| 84 | layer_name = os.path.basename(layer_path) | ||
| 85 | |||
| 86 | layer['name'] = layer_name | ||
| 87 | layer['path'] = layer_path | ||
| 88 | layer['conf'] = {} | ||
| 89 | |||
| 90 | if not os.path.isfile(os.path.join(layer_path, 'conf', 'layer.conf')): | ||
| 91 | layer['type'] = LayerType.ERROR_NO_LAYER_CONF | ||
| 92 | return layer | ||
| 93 | |||
| 94 | machine_conf = os.path.join(layer_path, 'conf', 'machine') | ||
| 95 | distro_conf = os.path.join(layer_path, 'conf', 'distro') | ||
| 96 | |||
| 97 | is_bsp = False | ||
| 98 | is_distro = False | ||
| 99 | |||
| 100 | if os.path.isdir(machine_conf): | ||
| 101 | machines = _get_configurations(machine_conf) | ||
| 102 | if machines: | ||
| 103 | is_bsp = True | ||
| 104 | |||
| 105 | if os.path.isdir(distro_conf): | ||
| 106 | distros = _get_configurations(distro_conf) | ||
| 107 | if distros: | ||
| 108 | is_distro = True | ||
| 109 | |||
| 110 | layer['collections'] = _get_layer_collections(layer['path']) | ||
| 111 | |||
| 112 | if layer_name == "meta" and "core" in layer['collections']: | ||
| 113 | layer['type'] = LayerType.CORE | ||
| 114 | layer['conf']['machines'] = machines | ||
| 115 | layer['conf']['distros'] = distros | ||
| 116 | elif is_bsp and is_distro: | ||
| 117 | layer['type'] = LayerType.ERROR_BSP_DISTRO | ||
| 118 | elif is_bsp: | ||
| 119 | layer['type'] = LayerType.BSP | ||
| 120 | layer['conf']['machines'] = machines | ||
| 121 | elif is_distro: | ||
| 122 | layer['type'] = LayerType.DISTRO | ||
| 123 | layer['conf']['distros'] = distros | ||
| 124 | else: | ||
| 125 | layer['type'] = LayerType.SOFTWARE | ||
| 126 | |||
| 127 | return layer | ||
| 128 | |||
| 129 | def detect_layers(layer_directories, no_auto): | ||
| 130 | layers = [] | ||
| 131 | |||
| 132 | for directory in layer_directories: | ||
| 133 | directory = os.path.realpath(directory) | ||
| 134 | if directory[-1] == '/': | ||
| 135 | directory = directory[0:-1] | ||
| 136 | |||
| 137 | if no_auto: | ||
| 138 | conf_dir = os.path.join(directory, 'conf') | ||
| 139 | if os.path.isdir(conf_dir): | ||
| 140 | layer = _detect_layer(directory) | ||
| 141 | if layer: | ||
| 142 | layers.append(layer) | ||
| 143 | else: | ||
| 144 | for root, dirs, files in os.walk(directory): | ||
| 145 | dir_name = os.path.basename(root) | ||
| 146 | conf_dir = os.path.join(root, 'conf') | ||
| 147 | if os.path.isdir(conf_dir): | ||
| 148 | layer = _detect_layer(root) | ||
| 149 | if layer: | ||
| 150 | layers.append(layer) | ||
| 151 | |||
| 152 | return layers | ||
| 153 | |||
| 154 | def _find_layer(depend, layers): | ||
| 155 | for layer in layers: | ||
| 156 | if 'collections' not in layer: | ||
| 157 | continue | ||
| 158 | |||
| 159 | for collection in layer['collections']: | ||
| 160 | if depend == collection: | ||
| 161 | return layer | ||
| 162 | return None | ||
| 163 | |||
| 164 | def sanity_check_layers(layers, logger): | ||
| 165 | """ | ||
| 166 | Check that we didn't find duplicate collection names, as the layer that will | ||
| 167 | be used is non-deterministic. The precise check is duplicate collections | ||
| 168 | with different patterns, as the same pattern being repeated won't cause | ||
| 169 | problems. | ||
| 170 | """ | ||
| 171 | import collections | ||
| 172 | |||
| 173 | passed = True | ||
| 174 | seen = collections.defaultdict(set) | ||
| 175 | for layer in layers: | ||
| 176 | for name, data in layer.get("collections", {}).items(): | ||
| 177 | seen[name].add(data["pattern"]) | ||
| 178 | |||
| 179 | for name, patterns in seen.items(): | ||
| 180 | if len(patterns) > 1: | ||
| 181 | passed = False | ||
| 182 | logger.error("Collection %s found multiple times: %s" % (name, ", ".join(patterns))) | ||
| 183 | return passed | ||
| 184 | |||
| 185 | def get_layer_dependencies(layer, layers, logger): | ||
| 186 | def recurse_dependencies(depends, layer, layers, logger, ret = []): | ||
| 187 | logger.debug('Processing dependencies %s for layer %s.' % \ | ||
| 188 | (depends, layer['name'])) | ||
| 189 | |||
| 190 | for depend in depends.split(): | ||
| 191 | # core (oe-core) is suppose to be provided | ||
| 192 | if depend == 'core': | ||
| 193 | continue | ||
| 194 | |||
| 195 | layer_depend = _find_layer(depend, layers) | ||
| 196 | if not layer_depend: | ||
| 197 | logger.error('Layer %s depends on %s and isn\'t found.' % \ | ||
| 198 | (layer['name'], depend)) | ||
| 199 | ret = None | ||
| 200 | continue | ||
| 201 | |||
| 202 | # We keep processing, even if ret is None, this allows us to report | ||
| 203 | # multiple errors at once | ||
| 204 | if ret is not None and layer_depend not in ret: | ||
| 205 | ret.append(layer_depend) | ||
| 206 | else: | ||
| 207 | # we might have processed this dependency already, in which case | ||
| 208 | # we should not do it again (avoid recursive loop) | ||
| 209 | continue | ||
| 210 | |||
| 211 | # Recursively process... | ||
| 212 | if 'collections' not in layer_depend: | ||
| 213 | continue | ||
| 214 | |||
| 215 | for collection in layer_depend['collections']: | ||
| 216 | collect_deps = layer_depend['collections'][collection]['depends'] | ||
| 217 | if not collect_deps: | ||
| 218 | continue | ||
| 219 | ret = recurse_dependencies(collect_deps, layer_depend, layers, logger, ret) | ||
| 220 | |||
| 221 | return ret | ||
| 222 | |||
| 223 | layer_depends = [] | ||
| 224 | for collection in layer['collections']: | ||
| 225 | depends = layer['collections'][collection]['depends'] | ||
| 226 | if not depends: | ||
| 227 | continue | ||
| 228 | |||
| 229 | layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends) | ||
| 230 | |||
| 231 | # Note: [] (empty) is allowed, None is not! | ||
| 232 | return layer_depends | ||
| 233 | |||
| 234 | def add_layer_dependencies(bblayersconf, layer, layers, logger): | ||
| 235 | |||
| 236 | layer_depends = get_layer_dependencies(layer, layers, logger) | ||
| 237 | if layer_depends is None: | ||
| 238 | return False | ||
| 239 | else: | ||
| 240 | add_layers(bblayersconf, layer_depends, logger) | ||
| 241 | |||
| 242 | return True | ||
| 243 | |||
| 244 | def add_layers(bblayersconf, layers, logger): | ||
| 245 | # Don't add a layer that is already present. | ||
| 246 | added = set() | ||
| 247 | output = check_command('Getting existing layers failed.', 'bitbake-layers show-layers').decode('utf-8') | ||
| 248 | for layer, path, pri in re.findall(r'^(\S+) +([^\n]*?) +(\d+)$', output, re.MULTILINE): | ||
| 249 | added.add(path) | ||
| 250 | |||
| 251 | with open(bblayersconf, 'a+') as f: | ||
| 252 | for layer in layers: | ||
| 253 | logger.info('Adding layer %s' % layer['name']) | ||
| 254 | name = layer['name'] | ||
| 255 | path = layer['path'] | ||
| 256 | if path in added: | ||
| 257 | logger.info('%s is already in %s' % (name, bblayersconf)) | ||
| 258 | else: | ||
| 259 | added.add(path) | ||
| 260 | f.write("\nBBLAYERS += \"%s\"\n" % path) | ||
| 261 | return True | ||
| 262 | |||
| 263 | def check_bblayers(bblayersconf, layer_path, logger): | ||
| 264 | ''' | ||
| 265 | If layer_path found in BBLAYERS return True | ||
| 266 | ''' | ||
| 267 | import bb.parse | ||
| 268 | import bb.data | ||
| 269 | |||
| 270 | ldata = bb.parse.handle(bblayersconf, bb.data.init(), include=True) | ||
| 271 | for bblayer in (ldata.getVar('BBLAYERS') or '').split(): | ||
| 272 | if os.path.normpath(bblayer) == os.path.normpath(layer_path): | ||
| 273 | return True | ||
| 274 | |||
| 275 | return False | ||
| 276 | |||
| 277 | def check_command(error_msg, cmd, cwd=None): | ||
| 278 | ''' | ||
| 279 | Run a command under a shell, capture stdout and stderr in a single stream, | ||
| 280 | throw an error when command returns non-zero exit code. Returns the output. | ||
| 281 | ''' | ||
| 282 | |||
| 283 | p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=cwd) | ||
| 284 | output, _ = p.communicate() | ||
| 285 | if p.returncode: | ||
| 286 | msg = "%s\nCommand: %s\nOutput:\n%s" % (error_msg, cmd, output.decode('utf-8')) | ||
| 287 | raise RuntimeError(msg) | ||
| 288 | return output | ||
| 289 | |||
| 290 | def get_signatures(builddir, failsafe=False, machine=None, extravars=None): | ||
| 291 | import re | ||
| 292 | |||
| 293 | # some recipes needs to be excluded like meta-world-pkgdata | ||
| 294 | # because a layer can add recipes to a world build so signature | ||
| 295 | # will be change | ||
| 296 | exclude_recipes = ('meta-world-pkgdata',) | ||
| 297 | |||
| 298 | sigs = {} | ||
| 299 | tune2tasks = {} | ||
| 300 | |||
| 301 | cmd = 'BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" ' | ||
| 302 | if extravars: | ||
| 303 | cmd += extravars | ||
| 304 | cmd += ' ' | ||
| 305 | if machine: | ||
| 306 | cmd += 'MACHINE=%s ' % machine | ||
| 307 | cmd += 'bitbake ' | ||
| 308 | if failsafe: | ||
| 309 | cmd += '-k ' | ||
| 310 | cmd += '-S lockedsigs world' | ||
| 311 | sigs_file = os.path.join(builddir, 'locked-sigs.inc') | ||
| 312 | if os.path.exists(sigs_file): | ||
| 313 | os.unlink(sigs_file) | ||
| 314 | try: | ||
| 315 | check_command('Generating signatures failed. This might be due to some parse error and/or general layer incompatibilities.', | ||
| 316 | cmd, builddir) | ||
| 317 | except RuntimeError as ex: | ||
| 318 | if failsafe and os.path.exists(sigs_file): | ||
| 319 | # Ignore the error here. Most likely some recipes active | ||
| 320 | # in a world build lack some dependencies. There is a | ||
| 321 | # separate test_machine_world_build which exposes the | ||
| 322 | # failure. | ||
| 323 | pass | ||
| 324 | else: | ||
| 325 | raise | ||
| 326 | |||
| 327 | sig_regex = re.compile(r"^(?P<task>.*:.*):(?P<hash>.*) .$") | ||
| 328 | tune_regex = re.compile(r"(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*") | ||
| 329 | current_tune = None | ||
| 330 | with open(sigs_file, 'r') as f: | ||
| 331 | for line in f.readlines(): | ||
| 332 | line = line.strip() | ||
| 333 | t = tune_regex.search(line) | ||
| 334 | if t: | ||
| 335 | current_tune = t.group('tune') | ||
| 336 | s = sig_regex.match(line) | ||
| 337 | if s: | ||
| 338 | exclude = False | ||
| 339 | for er in exclude_recipes: | ||
| 340 | (recipe, task) = s.group('task').split(':') | ||
| 341 | if er == recipe: | ||
| 342 | exclude = True | ||
| 343 | break | ||
| 344 | if exclude: | ||
| 345 | continue | ||
| 346 | |||
| 347 | sigs[s.group('task')] = s.group('hash') | ||
| 348 | tune2tasks.setdefault(current_tune, []).append(s.group('task')) | ||
| 349 | |||
| 350 | if not sigs: | ||
| 351 | raise RuntimeError('Can\'t load signatures from %s' % sigs_file) | ||
| 352 | |||
| 353 | return (sigs, tune2tasks) | ||
| 354 | |||
| 355 | def get_depgraph(targets=['world'], failsafe=False): | ||
| 356 | ''' | ||
| 357 | Returns the dependency graph for the given target(s). | ||
| 358 | The dependency graph is taken directly from DepTreeEvent. | ||
| 359 | ''' | ||
| 360 | depgraph = None | ||
| 361 | with bb.tinfoil.Tinfoil() as tinfoil: | ||
| 362 | tinfoil.prepare(config_only=False) | ||
| 363 | tinfoil.set_event_mask(['bb.event.NoProvider', 'bb.event.DepTreeGenerated', 'bb.command.CommandCompleted']) | ||
| 364 | if not tinfoil.run_command('generateDepTreeEvent', targets, 'do_build'): | ||
| 365 | raise RuntimeError('starting generateDepTreeEvent failed') | ||
| 366 | while True: | ||
| 367 | event = tinfoil.wait_event(timeout=1000) | ||
| 368 | if event: | ||
| 369 | if isinstance(event, bb.command.CommandFailed): | ||
| 370 | raise RuntimeError('Generating dependency information failed: %s' % event.error) | ||
| 371 | elif isinstance(event, bb.command.CommandCompleted): | ||
| 372 | break | ||
| 373 | elif isinstance(event, bb.event.NoProvider): | ||
| 374 | if failsafe: | ||
| 375 | # The event is informational, we will get information about the | ||
| 376 | # remaining dependencies eventually and thus can ignore this | ||
| 377 | # here like we do in get_signatures(), if desired. | ||
| 378 | continue | ||
| 379 | if event._reasons: | ||
| 380 | raise RuntimeError('Nothing provides %s: %s' % (event._item, event._reasons)) | ||
| 381 | else: | ||
| 382 | raise RuntimeError('Nothing provides %s.' % (event._item)) | ||
| 383 | elif isinstance(event, bb.event.DepTreeGenerated): | ||
| 384 | depgraph = event._depgraph | ||
| 385 | |||
| 386 | if depgraph is None: | ||
| 387 | raise RuntimeError('Could not retrieve the depgraph.') | ||
| 388 | return depgraph | ||
| 389 | |||
| 390 | def compare_signatures(old_sigs, curr_sigs): | ||
| 391 | ''' | ||
| 392 | Compares the result of two get_signatures() calls. Returns None if no | ||
| 393 | problems found, otherwise a string that can be used as additional | ||
| 394 | explanation in self.fail(). | ||
| 395 | ''' | ||
| 396 | # task -> (old signature, new signature) | ||
| 397 | sig_diff = {} | ||
| 398 | for task in old_sigs: | ||
| 399 | if task in curr_sigs and \ | ||
| 400 | old_sigs[task] != curr_sigs[task]: | ||
| 401 | sig_diff[task] = (old_sigs[task], curr_sigs[task]) | ||
| 402 | |||
| 403 | if not sig_diff: | ||
| 404 | return None | ||
| 405 | |||
| 406 | # Beware, depgraph uses task=<pn>.<taskname> whereas get_signatures() | ||
| 407 | # uses <pn>:<taskname>. Need to convert sometimes. The output follows | ||
| 408 | # the convention from get_signatures() because that seems closer to | ||
| 409 | # normal bitbake output. | ||
| 410 | def sig2graph(task): | ||
| 411 | pn, taskname = task.rsplit(':', 1) | ||
| 412 | return pn + '.' + taskname | ||
| 413 | def graph2sig(task): | ||
| 414 | pn, taskname = task.rsplit('.', 1) | ||
| 415 | return pn + ':' + taskname | ||
| 416 | depgraph = get_depgraph(failsafe=True) | ||
| 417 | depends = depgraph['tdepends'] | ||
| 418 | |||
| 419 | # If a task A has a changed signature, but none of its | ||
| 420 | # dependencies, then we need to report it because it is | ||
| 421 | # the one which introduces a change. Any task depending on | ||
| 422 | # A (directly or indirectly) will also have a changed | ||
| 423 | # signature, but we don't need to report it. It might have | ||
| 424 | # its own changes, which will become apparent once the | ||
| 425 | # issues that we do report are fixed and the test gets run | ||
| 426 | # again. | ||
| 427 | sig_diff_filtered = [] | ||
| 428 | for task, (old_sig, new_sig) in sig_diff.items(): | ||
| 429 | deps_tainted = False | ||
| 430 | for dep in depends.get(sig2graph(task), ()): | ||
| 431 | if graph2sig(dep) in sig_diff: | ||
| 432 | deps_tainted = True | ||
| 433 | break | ||
| 434 | if not deps_tainted: | ||
| 435 | sig_diff_filtered.append((task, old_sig, new_sig)) | ||
| 436 | |||
| 437 | msg = [] | ||
| 438 | msg.append('%d signatures changed, initial differences (first hash before, second after):' % | ||
| 439 | len(sig_diff)) | ||
| 440 | for diff in sorted(sig_diff_filtered): | ||
| 441 | recipe, taskname = diff[0].rsplit(':', 1) | ||
| 442 | cmd = 'bitbake-diffsigs --task %s %s --signature %s %s' % \ | ||
| 443 | (recipe, taskname, diff[1], diff[2]) | ||
| 444 | msg.append(' %s: %s -> %s' % diff) | ||
| 445 | msg.append(' %s' % cmd) | ||
| 446 | try: | ||
| 447 | output = check_command('Determining signature difference failed.', | ||
| 448 | cmd).decode('utf-8') | ||
| 449 | except RuntimeError as error: | ||
| 450 | output = str(error) | ||
| 451 | if output: | ||
| 452 | msg.extend([' ' + line for line in output.splitlines()]) | ||
| 453 | msg.append('') | ||
| 454 | return '\n'.join(msg) | ||
| 455 | |||
| 456 | |||
| 457 | def get_git_toplevel(directory): | ||
| 458 | """ | ||
| 459 | Try and find the top of the git repository that directory might be in. | ||
| 460 | Returns the top-level directory, or None. | ||
| 461 | """ | ||
| 462 | cmd = ["git", "-C", directory, "rev-parse", "--show-toplevel"] | ||
| 463 | try: | ||
| 464 | return subprocess.check_output(cmd, text=True).strip() | ||
| 465 | except: | ||
| 466 | return None | ||
diff --git a/scripts/lib/checklayer/case.py b/scripts/lib/checklayer/case.py deleted file mode 100644 index fa9dee384e..0000000000 --- a/scripts/lib/checklayer/case.py +++ /dev/null | |||
| @@ -1,9 +0,0 @@ | |||
| 1 | # Copyright (C) 2017 Intel Corporation | ||
| 2 | # | ||
| 3 | # SPDX-License-Identifier: MIT | ||
| 4 | # | ||
| 5 | |||
| 6 | from oeqa.core.case import OETestCase | ||
| 7 | |||
| 8 | class OECheckLayerTestCase(OETestCase): | ||
| 9 | pass | ||
diff --git a/scripts/lib/checklayer/cases/__init__.py b/scripts/lib/checklayer/cases/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 --- a/scripts/lib/checklayer/cases/__init__.py +++ /dev/null | |||
diff --git a/scripts/lib/checklayer/cases/bsp.py b/scripts/lib/checklayer/cases/bsp.py deleted file mode 100644 index b76163fb56..0000000000 --- a/scripts/lib/checklayer/cases/bsp.py +++ /dev/null | |||
| @@ -1,206 +0,0 @@ | |||
| 1 | # Copyright (C) 2017 Intel Corporation | ||
| 2 | # | ||
| 3 | # SPDX-License-Identifier: MIT | ||
| 4 | # | ||
| 5 | |||
| 6 | import unittest | ||
| 7 | |||
| 8 | from checklayer import LayerType, get_signatures, check_command, get_depgraph | ||
| 9 | from checklayer.case import OECheckLayerTestCase | ||
| 10 | |||
| 11 | class BSPCheckLayer(OECheckLayerTestCase): | ||
| 12 | @classmethod | ||
| 13 | def setUpClass(self): | ||
| 14 | if self.tc.layer['type'] not in (LayerType.BSP, LayerType.CORE): | ||
| 15 | raise unittest.SkipTest("BSPCheckLayer: Layer %s isn't BSP one." %\ | ||
| 16 | self.tc.layer['name']) | ||
| 17 | |||
| 18 | def test_bsp_defines_machines(self): | ||
| 19 | self.assertTrue(self.tc.layer['conf']['machines'], | ||
| 20 | "Layer is BSP but doesn't defines machines.") | ||
| 21 | |||
| 22 | def test_bsp_no_set_machine(self): | ||
| 23 | from oeqa.utils.commands import get_bb_var | ||
| 24 | |||
| 25 | machine = get_bb_var('MACHINE') | ||
| 26 | self.assertEqual(self.td['bbvars']['MACHINE'], machine, | ||
| 27 | msg="Layer %s modified machine %s -> %s" % \ | ||
| 28 | (self.tc.layer['name'], self.td['bbvars']['MACHINE'], machine)) | ||
| 29 | |||
| 30 | |||
| 31 | def test_machine_world(self): | ||
| 32 | ''' | ||
| 33 | "bitbake world" is expected to work regardless which machine is selected. | ||
| 34 | BSP layers sometimes break that by enabling a recipe for a certain machine | ||
| 35 | without checking whether that recipe actually can be built in the current | ||
| 36 | distro configuration (for example, OpenGL might not enabled). | ||
| 37 | |||
| 38 | This test iterates over all machines. It would be nicer to instantiate | ||
| 39 | it once per machine. It merely checks for errors during parse | ||
| 40 | time. It does not actually attempt to build anything. | ||
| 41 | ''' | ||
| 42 | |||
| 43 | if not self.td['machines']: | ||
| 44 | self.skipTest('No machines set with --machines.') | ||
| 45 | msg = [] | ||
| 46 | for machine in self.td['machines']: | ||
| 47 | # In contrast to test_machine_signatures() below, errors are fatal here. | ||
| 48 | try: | ||
| 49 | get_signatures(self.td['builddir'], failsafe=False, machine=machine) | ||
| 50 | except RuntimeError as ex: | ||
| 51 | msg.append(str(ex)) | ||
| 52 | if msg: | ||
| 53 | msg.insert(0, 'The following machines broke a world build:') | ||
| 54 | self.fail('\n'.join(msg)) | ||
| 55 | |||
| 56 | def test_machine_signatures(self): | ||
| 57 | ''' | ||
| 58 | Selecting a machine may only affect the signature of tasks that are specific | ||
| 59 | to that machine. In other words, when MACHINE=A and MACHINE=B share a recipe | ||
| 60 | foo and the output of foo, then both machine configurations must build foo | ||
| 61 | in exactly the same way. Otherwise it is not possible to use both machines | ||
| 62 | in the same distribution. | ||
| 63 | |||
| 64 | This criteria can only be tested by testing different machines in combination, | ||
| 65 | i.e. one main layer, potentially several additional BSP layers and an explicit | ||
| 66 | choice of machines: | ||
| 67 | yocto-check-layer --additional-layers .../meta-intel --machines intel-corei7-64 imx6slevk -- .../meta-freescale | ||
| 68 | ''' | ||
| 69 | |||
| 70 | if not self.td['machines']: | ||
| 71 | self.skipTest('No machines set with --machines.') | ||
| 72 | |||
| 73 | # Collect signatures for all machines that we are testing | ||
| 74 | # and merge that into a hash: | ||
| 75 | # tune -> task -> signature -> list of machines with that combination | ||
| 76 | # | ||
| 77 | # It is an error if any tune/task pair has more than one signature, | ||
| 78 | # because that implies that the machines that caused those different | ||
| 79 | # signatures do not agree on how to execute the task. | ||
| 80 | tunes = {} | ||
| 81 | # Preserve ordering of machines as chosen by the user. | ||
| 82 | for machine in self.td['machines']: | ||
| 83 | curr_sigs, tune2tasks = get_signatures(self.td['builddir'], failsafe=True, machine=machine) | ||
| 84 | # Invert the tune -> [tasks] mapping. | ||
| 85 | tasks2tune = {} | ||
| 86 | for tune, tasks in tune2tasks.items(): | ||
| 87 | for task in tasks: | ||
| 88 | tasks2tune[task] = tune | ||
| 89 | for task, sighash in curr_sigs.items(): | ||
| 90 | tunes.setdefault(tasks2tune[task], {}).setdefault(task, {}).setdefault(sighash, []).append(machine) | ||
| 91 | |||
| 92 | msg = [] | ||
| 93 | pruned = 0 | ||
| 94 | last_line_key = None | ||
| 95 | # do_fetch, do_unpack, ..., do_build | ||
| 96 | taskname_list = [] | ||
| 97 | if tunes: | ||
| 98 | # The output below is most useful when we start with tasks that are at | ||
| 99 | # the bottom of the dependency chain, i.e. those that run first. If | ||
| 100 | # those tasks differ, the rest also does. | ||
| 101 | # | ||
| 102 | # To get an ordering of tasks, we do a topological sort of the entire | ||
| 103 | # depgraph for the base configuration, then on-the-fly flatten that list by stripping | ||
| 104 | # out the recipe names and removing duplicates. The base configuration | ||
| 105 | # is not necessarily representative, but should be close enough. Tasks | ||
| 106 | # that were not encountered get a default priority. | ||
| 107 | depgraph = get_depgraph() | ||
| 108 | depends = depgraph['tdepends'] | ||
| 109 | WHITE = 1 | ||
| 110 | GRAY = 2 | ||
| 111 | BLACK = 3 | ||
| 112 | color = {} | ||
| 113 | found = set() | ||
| 114 | def visit(task): | ||
| 115 | color[task] = GRAY | ||
| 116 | for dep in depends.get(task, ()): | ||
| 117 | if color.setdefault(dep, WHITE) == WHITE: | ||
| 118 | visit(dep) | ||
| 119 | color[task] = BLACK | ||
| 120 | pn, taskname = task.rsplit('.', 1) | ||
| 121 | if taskname not in found: | ||
| 122 | taskname_list.append(taskname) | ||
| 123 | found.add(taskname) | ||
| 124 | for task in depends.keys(): | ||
| 125 | if color.setdefault(task, WHITE) == WHITE: | ||
| 126 | visit(task) | ||
| 127 | |||
| 128 | taskname_order = dict([(task, index) for index, task in enumerate(taskname_list) ]) | ||
| 129 | def task_key(task): | ||
| 130 | pn, taskname = task.rsplit(':', 1) | ||
| 131 | return (pn, taskname_order.get(taskname, len(taskname_list)), taskname) | ||
| 132 | |||
| 133 | for tune in sorted(tunes.keys()): | ||
| 134 | tasks = tunes[tune] | ||
| 135 | # As for test_signatures it would be nicer to sort tasks | ||
| 136 | # by dependencies here, but that is harder because we have | ||
| 137 | # to report on tasks from different machines, which might | ||
| 138 | # have different dependencies. We resort to pruning the | ||
| 139 | # output by reporting only one task per recipe if the set | ||
| 140 | # of machines matches. | ||
| 141 | # | ||
| 142 | # "bitbake-diffsigs -t -s" is intelligent enough to print | ||
| 143 | # diffs recursively, so often it does not matter that much | ||
| 144 | # if we don't pick the underlying difference | ||
| 145 | # here. However, sometimes recursion fails | ||
| 146 | # (https://bugzilla.yoctoproject.org/show_bug.cgi?id=6428). | ||
| 147 | # | ||
| 148 | # To mitigate that a bit, we use a hard-coded ordering of | ||
| 149 | # tasks that represents how they normally run and prefer | ||
| 150 | # to print the ones that run first. | ||
| 151 | for task in sorted(tasks.keys(), key=task_key): | ||
| 152 | signatures = tasks[task] | ||
| 153 | # do_build can be ignored: it is know to have | ||
| 154 | # different signatures in some cases, for example in | ||
| 155 | # the allarch ca-certificates due to RDEPENDS=openssl. | ||
| 156 | # That particular dependency is marked via | ||
| 157 | # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up | ||
| 158 | # in the sstate signature hash because filtering it | ||
| 159 | # out would be hard and running do_build multiple | ||
| 160 | # times doesn't really matter. | ||
| 161 | if len(signatures.keys()) > 1 and \ | ||
| 162 | not task.endswith(':do_build'): | ||
| 163 | # Error! | ||
| 164 | # | ||
| 165 | # Sort signatures by machines, because the hex values don't mean anything. | ||
| 166 | # => all-arch adwaita-icon-theme:do_build: 1234... (beaglebone, qemux86) != abcdf... (qemux86-64) | ||
| 167 | # | ||
| 168 | # Skip the line if it is covered already by the predecessor (same pn, same sets of machines). | ||
| 169 | pn, taskname = task.rsplit(':', 1) | ||
| 170 | next_line_key = (pn, sorted(signatures.values())) | ||
| 171 | if next_line_key != last_line_key: | ||
| 172 | line = ' %s %s: ' % (tune, task) | ||
| 173 | line += ' != '.join(['%s (%s)' % (signature, ', '.join([m for m in signatures[signature]])) for | ||
| 174 | signature in sorted(signatures.keys(), key=lambda s: signatures[s])]) | ||
| 175 | last_line_key = next_line_key | ||
| 176 | msg.append(line) | ||
| 177 | # Randomly pick two mismatched signatures and remember how to invoke | ||
| 178 | # bitbake-diffsigs for them. | ||
| 179 | iterator = iter(signatures.items()) | ||
| 180 | a = next(iterator) | ||
| 181 | b = next(iterator) | ||
| 182 | diffsig_machines = '(%s) != (%s)' % (', '.join(a[1]), ', '.join(b[1])) | ||
| 183 | diffsig_params = '-t %s %s -s %s %s' % (pn, taskname, a[0], b[0]) | ||
| 184 | else: | ||
| 185 | pruned += 1 | ||
| 186 | |||
| 187 | if msg: | ||
| 188 | msg.insert(0, 'The machines have conflicting signatures for some shared tasks:') | ||
| 189 | if pruned > 0: | ||
| 190 | msg.append('') | ||
| 191 | msg.append('%d tasks where not listed because some other task of the recipe already differed.' % pruned) | ||
| 192 | msg.append('It is likely that differences from different recipes also have the same root cause.') | ||
| 193 | msg.append('') | ||
| 194 | # Explain how to investigate... | ||
| 195 | msg.append('To investigate, run bitbake-diffsigs -t recipename taskname -s fromsig tosig.') | ||
| 196 | cmd = 'bitbake-diffsigs %s' % diffsig_params | ||
| 197 | msg.append('Example: %s in the last line' % diffsig_machines) | ||
| 198 | msg.append('Command: %s' % cmd) | ||
| 199 | # ... and actually do it automatically for that example, but without aborting | ||
| 200 | # when that fails. | ||
| 201 | try: | ||
| 202 | output = check_command('Comparing signatures failed.', cmd).decode('utf-8') | ||
| 203 | except RuntimeError as ex: | ||
| 204 | output = str(ex) | ||
| 205 | msg.extend([' ' + line for line in output.splitlines()]) | ||
| 206 | self.fail('\n'.join(msg)) | ||
diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py deleted file mode 100644 index ddead69a7b..0000000000 --- a/scripts/lib/checklayer/cases/common.py +++ /dev/null | |||
| @@ -1,135 +0,0 @@ | |||
| 1 | # Copyright (C) 2017 Intel Corporation | ||
| 2 | # | ||
| 3 | # SPDX-License-Identifier: MIT | ||
| 4 | # | ||
| 5 | |||
| 6 | import glob | ||
| 7 | import os | ||
| 8 | import unittest | ||
| 9 | import re | ||
| 10 | from checklayer import get_signatures, LayerType, check_command, compare_signatures, get_git_toplevel | ||
| 11 | from checklayer.case import OECheckLayerTestCase | ||
| 12 | |||
| 13 | class CommonCheckLayer(OECheckLayerTestCase): | ||
| 14 | def test_readme(self): | ||
| 15 | if self.tc.layer['type'] == LayerType.CORE: | ||
| 16 | raise unittest.SkipTest("Core layer's README is top level") | ||
| 17 | |||
| 18 | # The top-level README file may have a suffix (like README.rst or README.txt). | ||
| 19 | readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*')) | ||
| 20 | self.assertTrue(len(readme_files) > 0, | ||
| 21 | msg="Layer doesn't contain a README file.") | ||
| 22 | |||
| 23 | # There might be more than one file matching the file pattern above | ||
| 24 | # (for example, README.rst and README-COPYING.rst). The one with the shortest | ||
| 25 | # name is considered the "main" one. | ||
| 26 | readme_file = sorted(readme_files)[0] | ||
| 27 | data = '' | ||
| 28 | with open(readme_file, 'r') as f: | ||
| 29 | data = f.read() | ||
| 30 | self.assertTrue(data, | ||
| 31 | msg="Layer contains a README file but it is empty.") | ||
| 32 | |||
| 33 | # If a layer's README references another README, then the checks below are not valid | ||
| 34 | if re.search('README', data, re.IGNORECASE): | ||
| 35 | return | ||
| 36 | |||
| 37 | self.assertIn('maintainer', data.lower()) | ||
| 38 | self.assertIn('patch', data.lower()) | ||
| 39 | # Check that there is an email address in the README | ||
| 40 | email_regex = re.compile(r"[^@]+@[^@]+") | ||
| 41 | self.assertTrue(email_regex.match(data)) | ||
| 42 | |||
| 43 | def find_file_by_name(self, globs): | ||
| 44 | """ | ||
| 45 | Utility function to find a file that matches the specified list of | ||
| 46 | globs, in either the layer directory itself or the repository top-level | ||
| 47 | directory. | ||
| 48 | """ | ||
| 49 | directories = [self.tc.layer["path"]] | ||
| 50 | toplevel = get_git_toplevel(directories[0]) | ||
| 51 | if toplevel: | ||
| 52 | directories.append(toplevel) | ||
| 53 | |||
| 54 | for path in directories: | ||
| 55 | for name in globs: | ||
| 56 | files = glob.glob(os.path.join(path, name)) | ||
| 57 | if files: | ||
| 58 | return sorted(files)[0] | ||
| 59 | return None | ||
| 60 | |||
| 61 | def test_security(self): | ||
| 62 | """ | ||
| 63 | Test that the layer has a SECURITY.md (or similar) file, either in the | ||
| 64 | layer itself or at the top of the containing git repository. | ||
| 65 | """ | ||
| 66 | if self.tc.layer["type"] == LayerType.CORE: | ||
| 67 | raise unittest.SkipTest("Core layer's SECURITY is top level") | ||
| 68 | |||
| 69 | filename = self.find_file_by_name(("SECURITY", "SECURITY.*")) | ||
| 70 | self.assertTrue(filename, msg="Layer doesn't contain a SECURITY.md file.") | ||
| 71 | |||
| 72 | size = os.path.getsize(filename) | ||
| 73 | self.assertGreater(size, 0, msg=f"{filename} has no content.") | ||
| 74 | |||
| 75 | def test_parse(self): | ||
| 76 | check_command('Layer %s failed to parse.' % self.tc.layer['name'], | ||
| 77 | 'bitbake -p') | ||
| 78 | |||
| 79 | def test_show_environment(self): | ||
| 80 | check_command('Layer %s failed to show environment.' % self.tc.layer['name'], | ||
| 81 | 'bitbake -e') | ||
| 82 | |||
| 83 | def test_world(self): | ||
| 84 | ''' | ||
| 85 | "bitbake world" is expected to work. test_signatures does not cover that | ||
| 86 | because it is more lenient and ignores recipes in a world build that | ||
| 87 | are not actually buildable, so here we fail when "bitbake -S none world" | ||
| 88 | fails. | ||
| 89 | ''' | ||
| 90 | get_signatures(self.td['builddir'], failsafe=False) | ||
| 91 | |||
| 92 | def test_world_inherit_class(self): | ||
| 93 | ''' | ||
| 94 | This also does "bitbake -S none world" along with inheriting "yocto-check-layer" | ||
| 95 | class, which can do additional per-recipe test cases. | ||
| 96 | ''' | ||
| 97 | msg = [] | ||
| 98 | try: | ||
| 99 | get_signatures(self.td['builddir'], failsafe=False, machine=None, extravars='BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS INHERIT" INHERIT="yocto-check-layer"') | ||
| 100 | except RuntimeError as ex: | ||
| 101 | msg.append(str(ex)) | ||
| 102 | if msg: | ||
| 103 | msg.insert(0, 'Layer %s failed additional checks from yocto-check-layer.bbclass\nSee below log for specific recipe parsing errors:\n' % \ | ||
| 104 | self.tc.layer['name']) | ||
| 105 | self.fail('\n'.join(msg)) | ||
| 106 | |||
| 107 | def test_patches_upstream_status(self): | ||
| 108 | import sys | ||
| 109 | sys.path.append(os.path.join(sys.path[0], '../../../../meta/lib/')) | ||
| 110 | import oe.qa | ||
| 111 | patches = [] | ||
| 112 | for dirpath, dirs, files in os.walk(self.tc.layer['path']): | ||
| 113 | for filename in files: | ||
| 114 | if filename.endswith(".patch"): | ||
| 115 | ppath = os.path.join(dirpath, filename) | ||
| 116 | if oe.qa.check_upstream_status(ppath): | ||
| 117 | patches.append(ppath) | ||
| 118 | self.assertEqual(len(patches), 0 , \ | ||
| 119 | msg="Found following patches with malformed or missing upstream status:\n%s" % '\n'.join([str(patch) for patch in patches])) | ||
| 120 | |||
| 121 | def test_signatures(self): | ||
| 122 | if self.tc.layer['type'] == LayerType.SOFTWARE and \ | ||
| 123 | not self.tc.test_software_layer_signatures: | ||
| 124 | raise unittest.SkipTest("Not testing for signature changes in a software layer %s." \ | ||
| 125 | % self.tc.layer['name']) | ||
| 126 | |||
| 127 | curr_sigs, _ = get_signatures(self.td['builddir'], failsafe=True) | ||
| 128 | msg = compare_signatures(self.td['sigs'], curr_sigs) | ||
| 129 | if msg is not None: | ||
| 130 | self.fail('Adding layer %s changed signatures.\n%s' % (self.tc.layer['name'], msg)) | ||
| 131 | |||
| 132 | def test_layerseries_compat(self): | ||
| 133 | for collection_name, collection_data in self.tc.layer['collections'].items(): | ||
| 134 | self.assertTrue(collection_data['compat'], "Collection %s from layer %s does not set compatible oe-core versions via LAYERSERIES_COMPAT_collection." \ | ||
| 135 | % (collection_name, self.tc.layer['name'])) | ||
diff --git a/scripts/lib/checklayer/cases/distro.py b/scripts/lib/checklayer/cases/distro.py deleted file mode 100644 index a35332451c..0000000000 --- a/scripts/lib/checklayer/cases/distro.py +++ /dev/null | |||
| @@ -1,28 +0,0 @@ | |||
| 1 | # Copyright (C) 2017 Intel Corporation | ||
| 2 | # | ||
| 3 | # SPDX-License-Identifier: MIT | ||
| 4 | # | ||
| 5 | |||
| 6 | import unittest | ||
| 7 | |||
| 8 | from checklayer import LayerType | ||
| 9 | from checklayer.case import OECheckLayerTestCase | ||
| 10 | |||
| 11 | class DistroCheckLayer(OECheckLayerTestCase): | ||
| 12 | @classmethod | ||
| 13 | def setUpClass(self): | ||
| 14 | if self.tc.layer['type'] not in (LayerType.DISTRO, LayerType.CORE): | ||
| 15 | raise unittest.SkipTest("DistroCheckLayer: Layer %s isn't Distro one." %\ | ||
| 16 | self.tc.layer['name']) | ||
| 17 | |||
| 18 | def test_distro_defines_distros(self): | ||
| 19 | self.assertTrue(self.tc.layer['conf']['distros'], | ||
| 20 | "Layer is BSP but doesn't defines machines.") | ||
| 21 | |||
| 22 | def test_distro_no_set_distros(self): | ||
| 23 | from oeqa.utils.commands import get_bb_var | ||
| 24 | |||
| 25 | distro = get_bb_var('DISTRO') | ||
| 26 | self.assertEqual(self.td['bbvars']['DISTRO'], distro, | ||
| 27 | msg="Layer %s modified distro %s -> %s" % \ | ||
| 28 | (self.tc.layer['name'], self.td['bbvars']['DISTRO'], distro)) | ||
diff --git a/scripts/lib/checklayer/context.py b/scripts/lib/checklayer/context.py deleted file mode 100644 index 4de8f668fd..0000000000 --- a/scripts/lib/checklayer/context.py +++ /dev/null | |||
| @@ -1,17 +0,0 @@ | |||
| 1 | # Copyright (C) 2017 Intel Corporation | ||
| 2 | # | ||
| 3 | # SPDX-License-Identifier: MIT | ||
| 4 | # | ||
| 5 | |||
| 6 | import os | ||
| 7 | import sys | ||
| 8 | import glob | ||
| 9 | import re | ||
| 10 | |||
| 11 | from oeqa.core.context import OETestContext | ||
| 12 | |||
| 13 | class CheckLayerTestContext(OETestContext): | ||
| 14 | def __init__(self, td=None, logger=None, layer=None, test_software_layer_signatures=True): | ||
| 15 | super(CheckLayerTestContext, self).__init__(td, logger) | ||
| 16 | self.layer = layer | ||
| 17 | self.test_software_layer_signatures = test_software_layer_signatures | ||
