From baa1fddbbed6bdf392413061c0744c66660eedf4 Mon Sep 17 00:00:00 2001 From: Bruce Ashfield Date: Thu, 28 May 2026 02:23:28 +0000 Subject: oe-go-mod-fetcher: filter license scan to actually-unpacked modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --scan-licenses output (go-mod-licenses.inc) was iterating every module in modules.json, which mirrors go.sum and includes every version Go fetched for hash verification — including unselected indirect-dep versions and test-/tool-only deps. bitbake's do_populate_lic validates each LIC_FILES_CHKSUM entry against an unpacked module dir at pkg/mod/@/. Modules that go.sum lists but go build does not import never get unpacked there, so the QA check fails with "invalid file" errors — hundreds at a time on a busy project like cosign. Add a tiered filter to scan_module_licenses(): 1. Walk GOMODCACHE for *@v* directories. This is exactly the set the discovery step's `go build` populated, and matches 1:1 what bitbake will unpack at build time from the SRC_URI entries we generate. 2. Fall back to `go list -m all` (the MVS-selected set). Smaller than go.sum but larger than the unpacked set; useful when GOMODCACHE isn't yet populated. Set GOPROXY explicitly so the helper can download a newer toolchain if go.mod's directive requires one (do_generate_modules's env has GOPROXY=off). 3. No filter — original behavior, kept for safety. For cosign 3.0.6-tip, this trims go-mod-licenses.inc from 1320 to 265 lines (1068 unselected entries pruned) and resolves all do_populate_lic QA errors. Signed-off-by: Bruce Ashfield --- scripts/oe-go-mod-fetcher.py | 152 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 148 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/oe-go-mod-fetcher.py b/scripts/oe-go-mod-fetcher.py index c083b156..1f2fba19 100755 --- a/scripts/oe-go-mod-fetcher.py +++ b/scripts/oe-go-mod-fetcher.py @@ -2125,7 +2125,36 @@ def _execute(args: argparse.Namespace) -> int: if disc_cache and os.path.isdir(disc_cache): license_db = _resolve_license_db(args.common_license_dir) if license_db: - lic_results = scan_module_licenses(modules, disc_cache, license_db) + # Preferred filter: modules already unpacked under + # GOMODCACHE by the discovery build (`go build`). This + # matches exactly what bitbake will unpack at build time. + # Fallback: `go list -m all` (MVS-selected set) — includes + # test/tool deps that won't be unpacked, but still + # dramatically smaller than the go.sum entry list. + selected_set = _get_unpacked_modules( + gomodcache=args.gomodcache, + ) + if selected_set is not None: + print(f" Filtering to GOMODCACHE-unpacked set: " + f"{len(selected_set)} modules") + else: + selected_set = _get_mvs_selected_modules( + source_dir=Path.cwd(), + gomodcache=args.gomodcache, + ) + if selected_set is None: + print(" Warning: could not determine selected " + "module set; scanning all modules (may " + "produce stale entries)") + else: + print(f" Filtering to MVS-selected set " + f"(GOMODCACHE empty, fell back to " + f"`go list -m all`): " + f"{len(selected_set)} modules") + lic_results = scan_module_licenses( + modules, disc_cache, license_db, + selected_set=selected_set, + ) if lic_results: write_license_inc(output_dir, lic_results) else: @@ -4624,11 +4653,115 @@ def _find_module_zip(discovery_cache: str, module_path: str, version: str) -> Op return None +def _get_unpacked_modules(gomodcache: Optional[str] = None + ) -> Optional[Set[str]]: + """ + Walk GOMODCACHE for unpacked module dirs and return them as the set of + "module_path@version" strings (in Go's filesystem encoding, which for + all-lowercase module paths is identical to the canonical form). + + This is the most accurate signal for "what bitbake will unpack at build + time" — it's exactly what `go build` populated for the discovery step, + and the recipe's SRC_URI entries will re-unpack the same set at build + time. Returns None if gomodcache is not set or empty. + + Why this beats `go list -m all`: the MVS-selected set includes test- and + tool-only deps that never reach `go build`'s import graph, so they get + listed in go.sum but never unpacked under pkg/mod/@/. + Walking the already-populated cache skips those entirely. + """ + cache = gomodcache or os.environ.get('GOMODCACHE') + if not cache or not os.path.isdir(cache): + return None + base = Path(cache) + selected: Set[str] = set() + # Walk; prune $GOMODCACHE/cache/ which holds zips and metadata, not + # unpacked module trees. + for root, dirs, _files in os.walk(base): + if Path(root) == base and 'cache' in dirs: + dirs.remove('cache') + for d in list(dirs): + if '@v' in d: + full = Path(root) / d + rel = full.relative_to(base) + selected.add(str(rel)) + dirs.remove(d) # don't recurse into the module dir + return selected if selected else None + + +def _get_mvs_selected_modules(source_dir: Optional[Path] = None, + gomodcache: Optional[str] = None + ) -> Optional[Set[str]]: + """ + Return the set of MVS-selected modules as "module_path@version" strings, + using `go list -m all` from `source_dir` (defaults to CWD). + + Returns None if `go` is unavailable, the source dir has no go.mod, or + `go list` fails for any other reason — in which case the caller should + fall back to scanning all modules. + + Why this matters: the modules list passed to scan_module_licenses() is + derived from go.sum and includes every module version Go fetched for + hash verification, including unselected indirect-dep versions. Only the + MVS-selected subset is actually unpacked into pkg/mod/@/ + at build time. Writing LIC_FILES_CHKSUM entries for unselected versions + causes do_populate_lic to fail with QA errors ("invalid file"). + """ + src = Path(source_dir or '.').resolve() + if not (src / 'go.mod').exists(): + print(f" Note: no go.mod at {src}, cannot determine MVS-selected set") + return None + env = os.environ.copy() + env['GOFLAGS'] = (env.get('GOFLAGS', '') + ' -mod=mod').strip() + # The bitbake do_generate_modules env sets GOPROXY=off, which prevents + # `go list -m all` from resolving transitive metadata and from downloading + # any toolchain version required by go.mod's `toolchain` directive. + # Override here so this helper can do its job — it does not compile or + # install anything, just lists modules. + env['GOPROXY'] = env.get('GOPROXY_REAL', + 'https://proxy.golang.org,direct') + env.pop('GOTOOLCHAIN', None) # let it auto-resolve if go.mod needs newer + if gomodcache: + env['GOMODCACHE'] = gomodcache + try: + result = subprocess.run( + ['go', 'list', '-m', 'all'], + cwd=str(src), + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + except (FileNotFoundError, subprocess.TimeoutExpired) as exc: + print(f" Note: 'go list -m all' could not be executed: {exc}") + return None + if result.returncode != 0: + print(f" Note: 'go list -m all' exited {result.returncode}:") + for line in (result.stderr or '').strip().splitlines()[:5]: + print(f" {line}") + return None + selected: Set[str] = set() + for line in result.stdout.strip().splitlines(): + parts = line.split() + # Main module is "module/path" with no version — skip + if len(parts) >= 2: + module_path, version = parts[0], parts[1] + selected.add(f"{module_path}@{version}") + return selected + + def scan_module_licenses(modules: List[Dict], discovery_cache: str, - license_db: Dict[str, str]) -> Dict[str, Tuple[str, str, str]]: + license_db: Dict[str, str], + selected_set: Optional[Set[str]] = None + ) -> Dict[str, Tuple[str, str, str]]: """ Scan all modules for licenses using their discovery cache zips. + If `selected_set` is provided, modules whose "module_path@version" is + not in that set are skipped (they appear in go.sum but are not MVS- + selected, so they will not be unpacked at build time). + Returns dict mapping "module_path@version" to (spdx_name, license_file, md5). Only the first license file per module (matching OE-core behavior). """ @@ -4636,12 +4769,20 @@ def scan_module_licenses(modules: List[Dict], discovery_cache: str, scanned = 0 no_zip = 0 unknown = 0 + skipped_unselected = 0 for module in modules: module_path = module['module_path'] version = module['version'] key = f"{module_path}@{version}" + # Skip modules not selected by Go's MVS — they're in go.sum for hash + # verification but never unpacked into pkg/mod/@/, + # so a LIC_FILES_CHKSUM entry for them would fail do_populate_lic. + if selected_set is not None and key not in selected_set: + skipped_unselected += 1 + continue + zip_path = _find_module_zip(discovery_cache, module_path, version) if not zip_path: no_zip += 1 @@ -4657,8 +4798,11 @@ def scan_module_licenses(modules: List[Dict], discovery_cache: str, unknown += 1 scanned += 1 - print(f"\n License scan: {len(results)} modules with licenses, " - f"{no_zip} missing zips, {unknown} unknown") + msg = (f"\n License scan: {len(results)} modules with licenses, " + f"{no_zip} missing zips, {unknown} unknown") + if selected_set is not None: + msg += f", {skipped_unselected} skipped (not in selected set)" + print(msg) return results -- cgit v1.2.3-54-g00ecf