From 07bfa04b7718dd1b2a98c698fb1d1889de885664 Mon Sep 17 00:00:00 2001 From: Bruce Ashfield Date: Fri, 29 May 2026 13:37:32 +0000 Subject: oe-go-mod-fetcher + go-mod-discovery: add --build-target license-scan filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The license-scan filter previously had two tiers — walk GOMODCACHE for unpacked module dirs, falling back to `go list -m all` (the MVS set). For recipes whose do_compile builds multiple binaries spanning a wider import graph than the discovery step's single GO_MOD_DISCOVERY_BUILD_TARGET (incus: 8 cmd/* binaries vs the single ./cmd/incus-migrate used by discovery), the GOMODCACHE walk under-includes and the MVS fallback over-includes — producing go-mod-licenses.inc entries that point to modules bitbake never unpacks, which then trip do_populate_lic's "invalid file" QA gate. Add a tier-0 filter: when a recipe declares its full set of build targets via the new GO_MOD_DISCOVERY_LICENSE_TARGETS variable, the generator runs `go list -deps` on those targets and uses the union of their imported modules as the filter set. This is the most accurate signal for "what bitbake will unpack at build time" — equivalent to what go build would resolve before bitbake's do_unpack runs. Mechanism: - oe-go-mod-fetcher.py: new --build-target flag (repeatable) and _get_imported_modules() helper. Repeats over `action='append'` so the per-target paths survive shell word-splitting in the bbclass invocation without quoting tricks. - go-mod-discovery.bbclass: when GO_MOD_DISCOVERY_LICENSE_TARGETS is set, loop over its values and emit --build-target for each. The list is space-separated in the recipe; the bbclass loop keeps each target as its own shell word. Tier order is now: 1. --build-target (this commit): `go list -deps` on declared targets 2. GOMODCACHE walk (existing): modules unpacked by the discovery build 3. `go list -m all` MVS-selected set (existing): full module graph 4. No filter (existing): all go.sum entries Recipes without GO_MOD_DISCOVERY_LICENSE_TARGETS see no behavior change — verified against cosign via a real bitbake -c discover_and_generate run (zero diff vs HEAD on its sidecars). Incus's go-mod-licenses.inc drops from 418 (MVS fallback) to 176 lines once the new tier is wired up, matching the build's actual ~170 unpacked modules. Signed-off-by: Bruce Ashfield --- scripts/oe-go-mod-fetcher.py | 106 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 8 deletions(-) (limited to 'scripts') diff --git a/scripts/oe-go-mod-fetcher.py b/scripts/oe-go-mod-fetcher.py index 1f2fba19..b098a601 100755 --- a/scripts/oe-go-mod-fetcher.py +++ b/scripts/oe-go-mod-fetcher.py @@ -2125,19 +2125,39 @@ def _execute(args: argparse.Namespace) -> int: if disc_cache and os.path.isdir(disc_cache): license_db = _resolve_license_db(args.common_license_dir) if license_db: - # Preferred filter: modules already unpacked under - # GOMODCACHE by the discovery build (`go build`). This - # matches exactly what bitbake will unpack at build time. - # Fallback: `go list -m all` (MVS-selected set) — includes - # test/tool deps that won't be unpacked, but still - # dramatically smaller than the go.sum entry list. - selected_set = _get_unpacked_modules( + # Filter tier order (most accurate first): + # 1. --build-targets: `go list -deps` on the recipe's + # actual build targets. Needed when do_compile builds + # multiple binaries spanning a wider import graph + # than the discovery's single BUILD_TARGET. + # 2. GOMODCACHE walk: modules unpacked by `go build` of + # the discovery's BUILD_TARGET. Right when the + # discovery target's import set matches the recipe's. + # 3. `go list -m all` (MVS-selected): full module graph + # including test/tool deps. Too wide for some recipes + # but better than no filter. + selected_set = _get_imported_modules( + source_dir=Path.cwd(), + build_targets=args.build_targets, gomodcache=args.gomodcache, ) if selected_set is not None: - print(f" Filtering to GOMODCACHE-unpacked set: " + print(f" Filtering to `go list -deps` set " + f"(from --build-targets): " f"{len(selected_set)} modules") else: + selected_set = _get_unpacked_modules( + gomodcache=args.gomodcache, + ) + if selected_set is None: + # both build-targets and GOMODCACHE walk gave nothing + pass # fall through to MVS + elif args.build_targets is None: + # selected_set is from GOMODCACHE walk + print(f" Filtering to GOMODCACHE-unpacked set: " + f"{len(selected_set)} modules") + + if selected_set is None: selected_set = _get_mvs_selected_modules( source_dir=Path.cwd(), gomodcache=args.gomodcache, @@ -4653,6 +4673,64 @@ def _find_module_zip(discovery_cache: str, module_path: str, version: str) -> Op return None +def _get_imported_modules(source_dir: Optional[Path], + build_targets: Optional[List[str]], + gomodcache: Optional[str] = None + ) -> Optional[Set[str]]: + """ + Run `go list -deps` on the given build targets and return the set of + "module_path@version" strings that those targets actually import. + + This is the most accurate filter when the recipe builds multiple binaries + that span a wider import graph than the discovery step's single + BUILD_TARGET. The discovery's GOMODCACHE walk only sees what + `go build ` populated, which can be a strict subset of + what the recipe's `do_compile` will actually unpack at build time. + + Returns None if `go` is unavailable, source dir has no go.mod, + build_targets is empty, or `go list` fails. + """ + if not build_targets: + return None + src = Path(source_dir or '.').resolve() + if not (src / 'go.mod').exists(): + return None + + env = os.environ.copy() + env['GOFLAGS'] = (env.get('GOFLAGS', '') + ' -mod=mod').strip() + # Same rationale as _get_mvs_selected_modules: do_generate_modules sets + # GOPROXY=off, but we need it on so go-list can resolve metadata and + # download a newer toolchain if go.mod requires it. + env['GOPROXY'] = env.get('GOPROXY_REAL', + 'https://proxy.golang.org,direct') + env.pop('GOTOOLCHAIN', None) + if gomodcache: + env['GOMODCACHE'] = gomodcache + + cmd = ['go', 'list', '-deps', + '-f', '{{if .Module}}{{.Module.Path}}@{{.Module.Version}}{{end}}'] + cmd.extend(build_targets) + try: + result = subprocess.run(cmd, cwd=str(src), env=env, + capture_output=True, text=True, + timeout=300, check=False) + except (FileNotFoundError, subprocess.TimeoutExpired) as exc: + print(f" Note: 'go list -deps' could not be executed: {exc}") + return None + if result.returncode != 0: + print(f" Note: 'go list -deps' exited {result.returncode}:") + for line in (result.stderr or '').strip().splitlines()[:5]: + print(f" {line}") + return None + + selected: Set[str] = set() + for line in result.stdout.strip().splitlines(): + line = line.strip() + if line: + selected.add(line) + return selected if selected else None + + def _get_unpacked_modules(gomodcache: Optional[str] = None ) -> Optional[Set[str]]: """ @@ -4992,6 +5070,18 @@ Examples: help="Clean stale .info files in GOMODCACHE that lack VCS metadata (fixes 'module lookup disabled' errors)" ) + parser.add_argument( + "--build-target", + action='append', + default=None, + dest='build_targets', + help="Go build target used by the recipe's do_compile (e.g. './cmd/foo'). " + "Can be repeated. When set, the license scan filter is derived from " + "`go list -deps` on these targets — the most accurate signal for " + "'what bitbake will unpack at build time'. Takes precedence over the " + "GOMODCACHE walk." + ) + parser.add_argument( "--validate", action="store_true", -- cgit v1.2.3-54-g00ecf