go-mod-fetcher: fix shallow clone handling, duplicates, and discovery workflow

  oe-go-mod-fetcher.py:
  - Remove BB_GIT_SHALLOW_EXTRA_REFS generation - refs must be present in
    ALL repositories which isn't the case for module dependencies. Instead,
    use tag= parameter in individual SRC_URI entries.
  - Add tag=<tagname> to SRC_URI when ref is a tag, allowing BitBake's
    shallow clone to include the necessary tag (with BB_GIT_SHALLOW=1)
  - Remove premature _ref_points_to_commit() check that was clearing
    ref_hints before repos were fetched, preventing tag= from being added
  - Fix pseudo-version verification: only use shallow fetch for actual
    tags (refs/tags/...), not branch refs. Pseudo-versions with branch
    refs (refs/heads/...) now correctly use unshallow path to reach
    historical commits that aren't fetchable with depth=1

  oe-go-mod-fetcher-hybrid.py:
  - Fix duplicate SRC_URI entries when multiple modules share the same
    git repo/commit (e.g., errdefs and errdefs/pkg). Track added vcs_hashes
    to skip duplicates.
  - Add --discovery-cache option to calculate module sizes from discovery
    cache .zip files, enabling size recommendations during discover_and_generate

  go-mod-discovery.bbclass:
  - Add automatic hybrid mode recommendations after generate_modules,
    showing module sizes and suggested --git prefixes for conversion
  - Add GO_MOD_DISCOVERY_SKIP_VERIFY variable to skip commit verification
    on retries (useful after fixing verification issues)
  - Pass --discovery-cache to hybrid script for accurate size calculations

Signed-off-by: Bruce Ashfield <bruce.ashfield@gmail.com>

2 files changed, 56 insertions, 22 deletions

diff --git a/scripts/oe-go-mod-fetcher-hybrid.py b/scripts/oe-go-mod-fetcher-hybrid.py
index f5934ed2..9704cca0 100755
--- a/scripts/oe-go-mod-fetcher-hybrid.py
+++ b/scripts/oe-go-mod-fetcher-hybrid.py
@@ -241,6 +241,35 @@ def get_repo_sizes(vcs_info: dict, workdir: Optional[Path] = None) -> dict[str,
     return sizes
 
 
+def get_discovery_sizes(modules: list[dict], discovery_cache: Optional[Path] = None) -> dict[str, int]:
+    """Get sizes of modules from discovery cache .zip files."""
+    sizes = {}
+
+    if discovery_cache is None or not discovery_cache.exists():
+        return sizes
+
+    for mod in modules:
+        module_path = mod.get('module', '')
+        version = mod.get('version', '')
+        vcs_hash = mod.get('vcs_hash', '')
+
+        if not module_path or not version or not vcs_hash:
+            continue
+
+        # Build path to .zip file: discovery_cache/<module>/@v/<version>.zip
+        zip_path = discovery_cache / module_path / '@v' / f'{version}.zip'
+
+        if zip_path.exists():
+            try:
+                size = zip_path.stat().st_size
+                # Accumulate size by vcs_hash (same repo may have multiple modules)
+                sizes[vcs_hash] = sizes.get(vcs_hash, 0) + size
+            except OSError:
+                pass
+
+    return sizes
+
+
 def format_size(size_bytes: int) -> str:
     """Format bytes as human readable."""
     for unit in ['B', 'KB', 'MB', 'GB']:
@@ -604,6 +633,9 @@ def main():
     parser.add_argument('--workdir', type=Path, default=None,
                         help='BitBake workdir containing vcs_cache (for size calculations)')
 
+    parser.add_argument('--discovery-cache', type=Path, default=None,
+                        help='Discovery cache directory containing module .zip files (for size calculations)')
+
     # Actions
     parser.add_argument('--list', action='store_true',
                         help='List all modules with sizes')
@@ -650,12 +682,20 @@ def main():
     vcs_info = parse_go_mod_git_inc(git_inc)
     print(f"  Found {len(vcs_info)} VCS entries")
 
-    # Get sizes if workdir provided
+    # Get sizes from discovery cache and/or workdir
     sizes = {}
+    if args.discovery_cache:
+        print(f"Calculating sizes from discovery cache {args.discovery_cache}...")
+        sizes = get_discovery_sizes(modules, args.discovery_cache)
+        print(f"  Got sizes for {len(sizes)} modules from discovery cache")
+
     if args.workdir:
         print(f"Calculating sizes from {args.workdir}...")
-        sizes = get_repo_sizes(vcs_info, args.workdir)
-        print(f"  Got sizes for {len(sizes)} repos")
+        vcs_sizes = get_repo_sizes(vcs_info, args.workdir)
+        print(f"  Got sizes for {len(vcs_sizes)} repos from vcs_cache")
+        # Merge vcs_sizes into sizes (vcs_cache sizes override discovery if both exist)
+        for k, v in vcs_sizes.items():
+            sizes[k] = v
 
     # Handle actions
     if args.list:
diff --git a/scripts/oe-go-mod-fetcher.py b/scripts/oe-go-mod-fetcher.py
index 20ceee7c..13399a73 100755
--- a/scripts/oe-go-mod-fetcher.py
+++ b/scripts/oe-go-mod-fetcher.py
@@ -815,7 +815,10 @@ def verify_commit_accessible(vcs_url: str, commit: str, ref_hint: str = "", vers
         # Strategy depends on whether this is a tagged version or pseudo-version
         commit_fetched = commit_present  # If already present, no need to fetch
 
-        if ref_hint and not commit_present:
+        # Only use shallow fetch for actual tags - pseudo-versions with branch refs need unshallow
+        is_tag_ref = ref_hint and ref_hint.startswith('refs/tags/')
+
+        if is_tag_ref and not commit_present:
             # Tagged version: try shallow fetch of the specific commit (only if not already present)
             try:
                 fetch_cmd = ["git", "fetch", "--depth=1", "origin", commit]
@@ -3924,9 +3927,9 @@ def generate_recipe(modules: List[Dict], source_dir: Path, output_dir: Optional[
         else:
             commit_sha = repo_info['commits'][commit_hash]['commit_sha']
 
+        # Trust the ref_hint from discovery - it will be validated/corrected during
+        # the verification pass if needed (e.g., force-pushed tags are auto-corrected)
         ref_hint = module.get('vcs_ref', '')
-        if ref_hint and not _ref_points_to_commit(vcs_url, ref_hint, commit_hash):
-            ref_hint = ''
 
         entry = repo_info['commits'][commit_hash]
         entry['modules'].append(module)
@@ -4080,7 +4083,9 @@ def generate_recipe(modules: List[Dict], source_dir: Path, output_dir: Optional[
                 # For branches, use the branch name directly
                 if ref_hint.startswith('refs/tags/'):
                     # Tags: BitBake can fetch tagged commits with nobranch=1
-                    branch_param = ';nobranch=1'
+                    # Add tag= so shallow clones include this tag (with BB_GIT_SHALLOW=1 in recipe)
+                    tag_name = ref_hint[10:]  # Strip "refs/tags/"
+                    branch_param = f';nobranch=1;tag={tag_name}'
                 elif ref_hint.startswith('refs/heads/'):
                     # Branches: use the actual branch name
                     branch_name = ref_hint[11:]  # Strip "refs/heads/"
@@ -4161,21 +4166,10 @@ def generate_recipe(modules: List[Dict], source_dir: Path, output_dir: Optional[
             f.write(f'SRC_URI += "{entry}"\n')
         f.write('\n')
 
-        # Collect all tag references for shallow cloning
-        # BB_GIT_SHALLOW_EXTRA_REFS ensures these refs are included in shallow clones
-        tag_refs = set()
-        for module in modules:
-            vcs_ref = module.get('vcs_ref', '')
-            if vcs_ref and 'refs/tags/' in vcs_ref:
-                tag_refs.add(vcs_ref)
-
-        if tag_refs:
-            f.write("# Tag references for shallow cloning\n")
-            f.write("# Ensures shallow clones include all necessary tags\n")
-            f.write("BB_GIT_SHALLOW_EXTRA_REFS = \"\\\n")
-            for tag_ref in sorted(tag_refs):
-                f.write(f"    {tag_ref} \\\n")
-            f.write('"\n')
+        # Note: BB_GIT_SHALLOW_EXTRA_REFS is NOT used here because those refs must be
+        # present in ALL repositories, which isn't the case for module dependencies.
+        # Instead, we use tag= in individual SRC_URI entries when the ref is a tag.
+        # The recipe should set BB_GIT_SHALLOW = "1" to enable shallow clones globally.
 
         # Note: SRCREV_* variables are not needed since rev= is embedded directly in SRC_URI