diff options
| author | Bruce Ashfield <bruce.ashfield@gmail.com> | 2026-05-30 04:22:21 +0000 |
|---|---|---|
| committer | Bruce Ashfield <bruce.ashfield@gmail.com> | 2026-05-30 04:22:21 +0000 |
| commit | 84fcb4a6dc0009b9140fb6dc042bf8b2dc008314 (patch) | |
| tree | 20ae20187b9dbcd3bbdb77dc1a0fdf2bacf69b7b /scripts | |
| parent | 9557c826718d87f01c0c712f33b7bd5f8e641cd6 (diff) | |
| download | meta-virtualization-84fcb4a6dc0009b9140fb6dc042bf8b2dc008314.tar.gz | |
oe-go-mod-fetcher: fix stale clone cache in pseudo-version expansion
resolve_pseudo_version_commit() expands a 12-character pseudo-version
short hash to its full 40-character form by cloning the upstream repo
into a cache directory and running 'git log --since=...'. The cache is
reused on subsequent invocations and refreshed via 'git fetch --all'.
The initial clone used 'git clone --bare', which does NOT install a
remote.origin.fetch refspec. As a result, the later 'git fetch --all'
exits 0 but actually fetches nothing â the cache stays frozen at the
state of the very first clone, no matter how much time passes.
For any pseudo-version whose timestamp is newer than the first time the
cache was populated, the 'git log --since=<timestamp - 1 day>' search
returns no commits, expansion silently returns None, and the 12-char
hash propagates verbatim into the generated SRC_URI .inc files. bitbake
then refuses to parse the recipe with an unhelpful "Unable to resolve
'<12 chars>' in upstream git repository in git ls-remote output for
<repo>" error, because its git fetcher requires a full 40-char SHA.
Hit on a nerdctl v2.2.1 -> v2.3.1-tip bump: the repo's self-reference
module 'github.com/containerd/nerdctl/v2' at pseudo-version
v2.0.0-20260528082953-89ecd85071c4 (timestamped 2026-05-28) failed to
expand because the cached clone of github.com/containerd/nerdctl was
created on 2026-03-19 and never advanced.
Two changes:
- clone with --mirror instead of --bare. --mirror sets up the
+refs/*:refs/* refspec, so 'git fetch --all' actually pulls new
refs on every subsequent run.
- print a non-VERBOSE warning when expansion leaves any 12-char hash
unresolved, listing the affected modules and pointing at the cache
directory. The previous failure message was gated behind
VERBOSE_MODE, so an exhausted cache produced a silent fall-through
and the next visible error was bitbake's confusing parse failure.
Signed-off-by: Bruce Ashfield <bruce.ashfield@gmail.com>
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/oe-go-mod-fetcher.py | 26 |
1 files changed, 24 insertions, 2 deletions
diff --git a/scripts/oe-go-mod-fetcher.py b/scripts/oe-go-mod-fetcher.py index 74459204..c17582b8 100755 --- a/scripts/oe-go-mod-fetcher.py +++ b/scripts/oe-go-mod-fetcher.py | |||
| @@ -2754,13 +2754,19 @@ def resolve_pseudo_version_commit(vcs_url: str, timestamp_str: str, short_commit | |||
| 2754 | # Fetch failed, try to use existing clone anyway | 2754 | # Fetch failed, try to use existing clone anyway |
| 2755 | pass | 2755 | pass |
| 2756 | else: | 2756 | else: |
| 2757 | # Clone repository (bare clone for efficiency) | 2757 | # Clone repository as a mirror so `git fetch --all` can later |
| 2758 | # refresh refs. A plain `git clone --bare` does NOT install a | ||
| 2759 | # remote.origin.fetch refspec, so subsequent `git fetch --all` | ||
| 2760 | # exits 0 but updates nothing â the cache stays frozen at the | ||
| 2761 | # initial clone, and pseudo-version expansion silently fails | ||
| 2762 | # for any commit newer than the first clone time. --mirror | ||
| 2763 | # sets up the refspec `+refs/*:refs/*` so fetches stay current. | ||
| 2758 | if clone_dir.exists(): | 2764 | if clone_dir.exists(): |
| 2759 | shutil.rmtree(clone_dir) | 2765 | shutil.rmtree(clone_dir) |
| 2760 | clone_dir.mkdir(parents=True, exist_ok=True) | 2766 | clone_dir.mkdir(parents=True, exist_ok=True) |
| 2761 | 2767 | ||
| 2762 | subprocess.run( | 2768 | subprocess.run( |
| 2763 | ['git', 'clone', '--bare', '--quiet', try_url, str(clone_dir)], | 2769 | ['git', 'clone', '--mirror', '--quiet', try_url, str(clone_dir)], |
| 2764 | capture_output=True, | 2770 | capture_output=True, |
| 2765 | check=True, | 2771 | check=True, |
| 2766 | timeout=300, # 5 minute timeout | 2772 | timeout=300, # 5 minute timeout |
| @@ -4419,6 +4425,22 @@ def load_discovered_modules(discovered_modules_path: Path) -> Optional[List[Dict | |||
| 4419 | 4425 | ||
| 4420 | print(f" Expanded {expanded} short hashes, {failed} failed ") | 4426 | print(f" Expanded {expanded} short hashes, {failed} failed ") |
| 4421 | 4427 | ||
| 4428 | # A 12-char vcs_hash left in any module will trip bitbake's git | ||
| 4429 | # fetcher (sha1_re requires 40 chars) and trigger a confusing | ||
| 4430 | # "Unable to resolve" parse error. Surface this even outside | ||
| 4431 | # VERBOSE_MODE so the user catches it before the build fails. | ||
| 4432 | if failed: | ||
| 4433 | still_short = [m for m in short_hash_modules | ||
| 4434 | if len(m.get('vcs_hash', '')) != 40] | ||
| 4435 | if still_short: | ||
| 4436 | print(f"\nâ ď¸ {len(still_short)} module(s) still have 12-char hashes after expansion:") | ||
| 4437 | for m in still_short[:10]: | ||
| 4438 | print(f" - {m['module_path']}@{m.get('version', '')} ({m['vcs_hash']})") | ||
| 4439 | if len(still_short) > 10: | ||
| 4440 | print(f" ... and {len(still_short) - 10} more") | ||
| 4441 | print(f" These will cause bitbake parse errors. Common cause: a stale") | ||
| 4442 | print(f" clone_cache_dir entry. Try: rm -rf {CLONE_CACHE_DIR}") | ||
| 4443 | |||
| 4422 | # Filter out modules with empty vcs_hash - these are typically pre-Go 1.18 | 4444 | # Filter out modules with empty vcs_hash - these are typically pre-Go 1.18 |
| 4423 | # modules lacking Origin metadata (e.g. pre-release pseudo-versions) that | 4445 | # modules lacking Origin metadata (e.g. pre-release pseudo-versions) that |
| 4424 | # cannot be fetched from git. They are usually transitive dependencies that | 4446 | # cannot be fetched from git. They are usually transitive dependencies that |
