go-mod-discovery: split into more tasks

To make it easier to run individual components of the go module
discovery and generation process, we add invidual tasks as well
as a combined task:

 AVAILABLE TASKS:

   bitbake <recipe> -c discover_modules
       Build project and download modules from proxy.golang.org
       This populates the discovery cache but does NOT extract or generate

   bitbake <recipe> -c extract_modules
       Extract module metadata from discovery cache to modules.json
       Requires: discover_modules to have been run first

   bitbake <recipe> -c generate_modules
       Generate go-mod-git.inc and go-mod-cache.inc from modules.json
       Requires: extract_modules to have been run first

   bitbake <recipe> -c discover_and_generate
       Run all three steps: discover -> extract -> generate
       This is the "do everything" convenience task

   bitbake <recipe> -c show_upgrade_commands
       Show copy-pasteable command lines without running anything

   bitbake <recipe> -c clean_discovery
       Remove the persistent discovery cache

Signed-off-by: Bruce Ashfield <bruce.ashfield@gmail.com>

1 files changed, 307 insertions, 214 deletions

diff --git a/classes/go-mod-discovery.bbclass b/classes/go-mod-discovery.bbclass
index 0d703d0a..05265c20 100644
--- a/classes/go-mod-discovery.bbclass
+++ b/classes/go-mod-discovery.bbclass
@@ -6,15 +6,31 @@
 
 # go-mod-discovery.bbclass
 #
-# Provides a do_discover_modules task for Go projects that downloads complete
-# module metadata from proxy.golang.org for use with the bootstrap strategy.
+# Provides tasks for Go module discovery and recipe generation.
 #
-# USAGE:
-#   1. Add to recipe: inherit go-mod-discovery
-#   2. Set required variables (see CONFIGURATION below)
-#   3. Run discovery: bitbake <recipe> -c discover_modules
-#      (This automatically: downloads modules, extracts metadata, regenerates recipe)
-#   4. Build normally: bitbake <recipe>
+# AVAILABLE TASKS:
+#
+#   bitbake <recipe> -c discover_modules
+#       Build project and download modules from proxy.golang.org
+#       This populates the discovery cache but does NOT extract or generate
+#
+#   bitbake <recipe> -c extract_modules
+#       Extract module metadata from discovery cache to modules.json
+#       Requires: discover_modules to have been run first
+#
+#   bitbake <recipe> -c generate_modules
+#       Generate go-mod-git.inc and go-mod-cache.inc from modules.json
+#       Requires: extract_modules to have been run first
+#
+#   bitbake <recipe> -c discover_and_generate
+#       Run all three steps: discover -> extract -> generate
+#       This is the "do everything" convenience task
+#
+#   bitbake <recipe> -c show_upgrade_commands
+#       Show copy-pasteable command lines without running anything
+#
+#   bitbake <recipe> -c clean_discovery
+#       Remove the persistent discovery cache
 #
 # CONFIGURATION:
 #
@@ -48,15 +64,9 @@
 #   GO_MOD_DISCOVERY_MODULES_JSON - Output path for extracted module metadata
 #                                   Default: "${GO_MOD_DISCOVERY_DIR}/modules.json"
 #
-#   GO_MOD_DISCOVERY_SKIP_EXTRACT - Set to "1" to skip automatic extraction
-#                                   Default: "0" (extraction runs automatically)
-#
-#   GO_MOD_DISCOVERY_SKIP_GENERATE - Set to "1" to skip automatic recipe generation
-#                                    Default: "0" (generation runs automatically)
-#
 #   GO_MOD_DISCOVERY_GIT_REPO     - Git repository URL for recipe generation
 #                                   Example: "https://github.com/rancher/k3s.git"
-#                                   Required for automatic generation
+#                                   Required for generate_modules task
 #
 #   GO_MOD_DISCOVERY_GIT_REF      - Git ref (commit/tag) for recipe generation
 #                                   Default: "${SRCREV}" (uses recipe's SRCREV)
@@ -64,32 +74,22 @@
 #   GO_MOD_DISCOVERY_RECIPEDIR    - Output directory for generated .inc files
 #                                   Default: "${FILE_DIRNAME}" (recipe's directory)
 #
-# MINIMAL EXAMPLE (manual generation - no GIT_REPO set):
-#
-#   TAGS = "netcgo osusergo"
-#   GO_MOD_DISCOVERY_BUILD_TARGET = "./cmd/myapp"
-#   inherit go-mod-discovery
-#   # Run: bitbake myapp -c discover_modules
-#   # Then manually: oe-go-mod-fetcher.py --discovered-modules ... --git-repo ...
+# WORKFLOW EXAMPLES:
 #
-# FULL AUTOMATIC EXAMPLE (all-in-one discovery + generation):
+# Full automatic (one command does everything):
+#   bitbake myapp -c discover_and_generate
 #
-#   TAGS = "netcgo osusergo"
-#   GO_MOD_DISCOVERY_BUILD_TARGET = "./cmd/myapp"
-#   GO_MOD_DISCOVERY_GIT_REPO = "https://github.com/example/myapp.git"
-#   inherit go-mod-discovery
-#   # Run: bitbake myapp -c discover_modules
-#   # Recipe files are automatically regenerated!
+# Step by step (useful for debugging or rerunning individual steps):
+#   bitbake myapp -c discover_modules    # Download modules
+#   bitbake myapp -c extract_modules     # Extract metadata
+#   bitbake myapp -c generate_modules    # Generate .inc files
 #
-# See: meta-virtualization/scripts/BOOTSTRAP-STRATEGY.md (Approach B)
-#
-# This task is NOT part of the normal build - it must be explicitly invoked
-# via bitbake <recipe> -c discover_modules
+# Skip BitBake, use scripts directly (see show_upgrade_commands):
+#   bitbake myapp -c show_upgrade_commands
 #
 # PERSISTENT CACHE: The discovery cache is stored in ${TOPDIR}/go-mod-discovery/${PN}/${PV}/
-# instead of ${WORKDIR}. This ensures the cache survives `bitbake <recipe> -c cleanall`
-# since TOPDIR is the build directory root (e.g., /path/to/build/).
-# To clean the discovery cache, run: rm -rf ${TOPDIR}/go-mod-discovery/${PN}/${PV}/
+# This ensures the cache survives `bitbake <recipe> -c cleanall`.
+# To clean: bitbake <recipe> -c clean_discovery
 
 # Required variable (must be set by recipe)
 GO_MOD_DISCOVERY_BUILD_TARGET ?= ""
@@ -107,13 +107,7 @@ GO_MOD_DISCOVERY_DIR ?= "${TOPDIR}/go-mod-discovery/${PN}/${PV}"
 # Output JSON file for discovered modules (used by oe-go-mod-fetcher.py --discovered-modules)
 GO_MOD_DISCOVERY_MODULES_JSON ?= "${GO_MOD_DISCOVERY_DIR}/modules.json"
 
-# Set to "1" to skip automatic extraction (only download modules, don't extract metadata)
-GO_MOD_DISCOVERY_SKIP_EXTRACT ?= "0"
-
-# Set to "1" to skip automatic recipe regeneration (only discover and extract)
-GO_MOD_DISCOVERY_SKIP_GENERATE ?= "0"
-
-# Git repository URL for recipe generation (required if SKIP_GENERATE != "1")
+# Git repository URL for recipe generation (required for generate_modules)
 # Example: "https://github.com/rancher/k3s.git"
 GO_MOD_DISCOVERY_GIT_REPO ?= ""
 
@@ -126,7 +120,14 @@ GO_MOD_DISCOVERY_RECIPEDIR ?= "${FILE_DIRNAME}"
 # Empty default for TAGS if not set by recipe (avoids undefined variable errors)
 TAGS ?= ""
 
-# Shell task that mirrors do_compile but with network access and discovery GOMODCACHE
+# =============================================================================
+# TASK 1: do_discover_modules - Build and download modules
+# =============================================================================
+# This task builds the project with network access to discover and download
+# all required Go modules from proxy.golang.org into a persistent cache.
+#
+# Usage: bitbake <recipe> -c discover_modules
+#
 do_discover_modules() {
     # Validate required variable
     if [ -z "${GO_MOD_DISCOVERY_BUILD_TARGET}" ]; then
@@ -144,10 +145,9 @@ Hint: Set GO_MOD_DISCOVERY_SRCDIR to the directory containing go.mod"
     fi
 
     # Use PERSISTENT cache location outside WORKDIR to survive cleanall
-    # This is stored in ${TOPDIR}/go-mod-discovery/${PN}/${PV}/ so it persists
     DISCOVERY_CACHE="${GO_MOD_DISCOVERY_DIR}/cache"
 
-    # Create required directories first
+    # Create required directories
     mkdir -p "${DISCOVERY_CACHE}"
     mkdir -p "${WORKDIR}/go-tmp"
     mkdir -p "$(dirname "${GO_MOD_DISCOVERY_OUTPUT}")"
@@ -159,21 +159,18 @@ Hint: Set GO_MOD_DISCOVERY_SRCDIR to the directory containing go.mod"
     export GOPROXY="https://proxy.golang.org,direct"
     export GOSUMDB="sum.golang.org"
 
-    # Standard Go environment - use recipe-provided GOPATH or default
+    # Standard Go environment
     export GOPATH="${GO_MOD_DISCOVERY_GOPATH}:${STAGING_DIR_TARGET}/${prefix}/local/go"
     export CGO_ENABLED="1"
     export GOTOOLCHAIN="local"
-
-    # Use system temp directory for Go's work files
     export GOTMPDIR="${WORKDIR}/go-tmp"
 
-    # Disable excessive debug output from BitBake environment
+    # Disable excessive debug output
     unset GODEBUG
 
     # Build tags from recipe configuration
     TAGS="${GO_MOD_DISCOVERY_BUILD_TAGS}"
 
-    # Change to source directory
     cd "${GO_MOD_DISCOVERY_SRCDIR}"
 
     echo "======================================================================"
@@ -187,21 +184,11 @@ Hint: Set GO_MOD_DISCOVERY_SRCDIR to the directory containing go.mod"
     echo "LDFLAGS:       ${GO_MOD_DISCOVERY_LDFLAGS}"
     echo ""
 
-    # Use native go binary (not cross-compiler)
+    # Use native go binary
     GO_NATIVE="${STAGING_DIR_NATIVE}${bindir_native}/go"
 
-    # NOTE: Do NOT run go mod tidy during discovery - it can upgrade versions in go.mod
-    # without adding checksums to go.sum, causing version mismatches.
-    # The source's go.mod/go.sum should already be correct for the commit.
-    # echo "Running: go mod tidy"
-    # ${GO_NATIVE} mod tidy
-    # ${GO_NATIVE} mod download  # If tidy is re-enabled, this ensures go.sum gets all checksums
-
-    echo ""
     echo "Running: go build (to discover all modules)..."
 
-    # Build to discover ALL modules that would be used at compile time
-    # This is better than 'go mod download' because it handles build tags correctly
     BUILD_CMD="${GO_NATIVE} build -v -trimpath"
     if [ -n "${TAGS}" ]; then
         BUILD_CMD="${BUILD_CMD} -tags \"${TAGS}\""
@@ -214,68 +201,39 @@ Hint: Set GO_MOD_DISCOVERY_SRCDIR to the directory containing go.mod"
 
     echo ""
     echo "Fetching ALL modules referenced in go.sum..."
-    # go build downloads .zip files but not always .info files
-    # We need .info files for VCS metadata (Origin.URL, Origin.Hash)
-    # Extract unique module@version pairs from go.sum and download each
-    # go.sum format: "module version/go.mod hash" or "module version hash"
-    #
-    # IMPORTANT: We must download ALL versions, including /go.mod-only entries!
-    # When GOPROXY=off during compile, Go may need these for dependency resolution.
-    # Strip the /go.mod suffix to get the base version, then download it.
     awk '{gsub(/\/go\.mod$/, "", $2); print $1 "@" $2}' go.sum | sort -u | while read modver; do
         ${GO_NATIVE} mod download "$modver" 2>/dev/null || true
     done
 
-    # Download ALL modules in the complete dependency graph.
-    # The go.sum loop above only gets direct dependencies. Replace directives
-    # can introduce transitive deps that aren't in go.sum but are needed at
-    # compile time when GOPROXY=off. `go mod download all` resolves and
-    # downloads the entire module graph, including transitive dependencies.
     echo ""
     echo "Downloading complete module graph (including transitive deps)..."
     ${GO_NATIVE} mod download all 2>&1 || echo "Warning: some modules may have failed to download"
 
-    # Additionally scan for any modules that go build downloaded but don't have .info
-    # This ensures we capture everything that was fetched dynamically
     echo ""
     echo "Ensuring .info files for all cached modules..."
     find "${GOMODCACHE}/cache/download" -name "*.zip" 2>/dev/null | while read zipfile; do
-        # Extract module@version from path like: .../module/@v/version.zip
         version=$(basename "$zipfile" .zip)
         moddir=$(dirname "$zipfile")
         infofile="${moddir}/${version}.info"
         if [ ! -f "$infofile" ]; then
-            # Reconstruct module path from directory structure
-            # cache/download/github.com/foo/bar/@v/v1.0.0.zip -> github.com/foo/bar@v1.0.0
             modpath=$(echo "$moddir" | sed "s|${GOMODCACHE}/cache/download/||" | sed 's|/@v$||')
             echo "  Fetching .info for: ${modpath}@${version}"
             ${GO_NATIVE} mod download "${modpath}@${version}" 2>/dev/null || true
         fi
     done
 
-    # Download transitive deps of REPLACED modules.
-    # Replace directives can point to older versions whose deps aren't in the MVS
-    # graph. At compile time with GOPROXY=off, Go validates the replaced version's
-    # go.mod. We parse replace directives and download each replacement version,
-    # which fetches all its transitive dependencies.
     echo ""
     echo "Downloading dependencies of replaced modules..."
-
-    # Extract replace directives: "old_module => new_module new_version"
     awk '/^replace \($/,/^\)$/ {if ($0 !~ /^replace|^\)/) print}' go.mod | \
     grep "=>" | while read line; do
-        # Parse: github.com/foo/bar => github.com/baz/qux v1.2.3
         new_module=$(echo "$line" | awk '{print $(NF-1)}')
         new_version=$(echo "$line" | awk '{print $NF}')
-
         if [ -n "$new_module" ] && [ -n "$new_version" ] && [ "$new_version" != "=>" ]; then
             echo "  Replace target: ${new_module}@${new_version}"
-            # Download this specific version - Go will fetch all its dependencies
             ${GO_NATIVE} mod download "${new_module}@${new_version}" 2>/dev/null || true
         fi
     done
 
-    # Count modules discovered
     MODULE_COUNT=$(find "${GOMODCACHE}/cache/download" -name "*.info" 2>/dev/null | wc -l)
 
     echo ""
@@ -284,148 +242,190 @@ Hint: Set GO_MOD_DISCOVERY_SRCDIR to the directory containing go.mod"
     echo "======================================================================"
     echo "Modules discovered: ${MODULE_COUNT}"
     echo "Cache location:     ${GOMODCACHE}"
+    echo ""
+    echo "Next steps:"
+    echo "  bitbake ${PN} -c extract_modules   # Extract metadata to JSON"
+    echo "  bitbake ${PN} -c generate_modules  # Generate .inc files"
+    echo ""
+    echo "Or run all at once:"
+    echo "  bitbake ${PN} -c discover_and_generate"
+    echo ""
+}
 
-    # Extract module metadata automatically (unless skipped)
-    if [ "${GO_MOD_DISCOVERY_SKIP_EXTRACT}" != "1" ]; then
-        echo ""
-        echo "Extracting module metadata..."
+addtask discover_modules after do_patch
+do_discover_modules[depends] = "${PN}:do_prepare_recipe_sysroot"
+do_discover_modules[network] = "1"
+do_discover_modules[nostamp] = "1"
+do_discover_modules[vardeps] += "GO_MOD_DISCOVERY_DIR GO_MOD_DISCOVERY_SRCDIR \
+    GO_MOD_DISCOVERY_BUILD_TARGET GO_MOD_DISCOVERY_BUILD_TAGS \
+    GO_MOD_DISCOVERY_LDFLAGS GO_MOD_DISCOVERY_GOPATH GO_MOD_DISCOVERY_OUTPUT"
 
-        # Find the extraction script relative to this class file
-        EXTRACT_SCRIPT="${COREBASE}/../meta-virtualization/scripts/extract-discovered-modules.py"
-        if [ ! -f "${EXTRACT_SCRIPT}" ]; then
-            # Try alternate location
-            EXTRACT_SCRIPT="$(dirname "${COREBASE}")/meta-virtualization/scripts/extract-discovered-modules.py"
-        fi
-        if [ ! -f "${EXTRACT_SCRIPT}" ]; then
-            # Last resort - search in layer path
-            for layer in ${BBLAYERS}; do
-                if [ -f "${layer}/scripts/extract-discovered-modules.py" ]; then
-                    EXTRACT_SCRIPT="${layer}/scripts/extract-discovered-modules.py"
-                    break
-                fi
-            done
-        fi
+# =============================================================================
+# TASK 2: do_extract_modules - Extract metadata from cache
+# =============================================================================
+# This task extracts module metadata from the discovery cache into a JSON file.
+# The JSON file can then be used with oe-go-mod-fetcher.py --discovered-modules.
+#
+# Usage: bitbake <recipe> -c extract_modules
+#
+do_extract_modules() {
+    DISCOVERY_CACHE="${GO_MOD_DISCOVERY_DIR}/cache"
 
-        if [ -f "${EXTRACT_SCRIPT}" ]; then
-            python3 "${EXTRACT_SCRIPT}" \
-                --gomodcache "${GOMODCACHE}" \
-                --output "${GO_MOD_DISCOVERY_MODULES_JSON}"
-            EXTRACT_RC=$?
-            if [ $EXTRACT_RC -eq 0 ]; then
-                echo ""
-                echo "✓ Module metadata extracted to: ${GO_MOD_DISCOVERY_MODULES_JSON}"
-            else
-                bbwarn "Module extraction failed (exit code $EXTRACT_RC)"
-                bbwarn "You can run manually: python3 ${EXTRACT_SCRIPT} --gomodcache ${GOMODCACHE} --output ${GO_MOD_DISCOVERY_MODULES_JSON}"
-                EXTRACT_RC=1  # Mark as failed for generation check
-            fi
-        else
-            bbwarn "Could not find extract-discovered-modules.py script"
-            bbwarn "Run manually: extract-discovered-modules.py --gomodcache ${GOMODCACHE} --output ${GO_MOD_DISCOVERY_MODULES_JSON}"
-            EXTRACT_RC=1  # Mark as failed for generation check
-        fi
-    else
-        echo ""
-        echo "Skipping automatic extraction (GO_MOD_DISCOVERY_SKIP_EXTRACT=1)"
-        EXTRACT_RC=1  # Skip generation too if extraction skipped
+    if [ ! -d "${DISCOVERY_CACHE}/cache/download" ]; then
+        bbfatal "Discovery cache not found: ${DISCOVERY_CACHE}
+Run 'bitbake ${PN} -c discover_modules' first to populate the cache."
     fi
 
-    # Step 3: Generate recipe .inc files (unless skipped or extraction failed)
-    if [ "${GO_MOD_DISCOVERY_SKIP_GENERATE}" != "1" ] && [ "${EXTRACT_RC:-0}" = "0" ]; then
-        # Validate required git repo
-        if [ -z "${GO_MOD_DISCOVERY_GIT_REPO}" ]; then
-            bbwarn "GO_MOD_DISCOVERY_GIT_REPO not set - skipping recipe generation"
-            bbwarn "Set GO_MOD_DISCOVERY_GIT_REPO in your recipe to enable automatic generation"
-            echo ""
-            echo "NEXT STEP: Regenerate recipe manually:"
-            echo ""
-            echo "   ./meta-virtualization/scripts/oe-go-mod-fetcher.py \\"
-            echo "     --discovered-modules ${GO_MOD_DISCOVERY_MODULES_JSON} \\"
-            echo "     --git-repo <your-git-repo-url> \\"
-            echo "     --git-ref ${GO_MOD_DISCOVERY_GIT_REF} \\"
-            echo "     --recipedir ${GO_MOD_DISCOVERY_RECIPEDIR}"
-        else
-            echo ""
-            echo "Generating recipe .inc files..."
-
-            # Find the fetcher script (same search as extraction script)
-            FETCHER_SCRIPT="${COREBASE}/../meta-virtualization/scripts/oe-go-mod-fetcher.py"
-            if [ ! -f "${FETCHER_SCRIPT}" ]; then
-                FETCHER_SCRIPT="$(dirname "${COREBASE}")/meta-virtualization/scripts/oe-go-mod-fetcher.py"
-            fi
-            if [ ! -f "${FETCHER_SCRIPT}" ]; then
-                for layer in ${BBLAYERS}; do
-                    if [ -f "${layer}/scripts/oe-go-mod-fetcher.py" ]; then
-                        FETCHER_SCRIPT="${layer}/scripts/oe-go-mod-fetcher.py"
-                        break
-                    fi
-                done
-            fi
-
-            if [ -f "${FETCHER_SCRIPT}" ]; then
-                python3 "${FETCHER_SCRIPT}" \
-                    --discovered-modules "${GO_MOD_DISCOVERY_MODULES_JSON}" \
-                    --git-repo "${GO_MOD_DISCOVERY_GIT_REPO}" \
-                    --git-ref "${GO_MOD_DISCOVERY_GIT_REF}" \
-                    --recipedir "${GO_MOD_DISCOVERY_RECIPEDIR}"
-                GENERATE_RC=$?
-                if [ $GENERATE_RC -eq 0 ]; then
-                    echo ""
-                    echo "✓ Recipe files regenerated in: ${GO_MOD_DISCOVERY_RECIPEDIR}"
-                else
-                    bbwarn "Recipe generation failed (exit code $GENERATE_RC)"
-                    bbwarn "Check the output above for errors"
-                fi
-            else
-                bbwarn "Could not find oe-go-mod-fetcher.py script"
-                bbwarn "Run manually: oe-go-mod-fetcher.py --discovered-modules ${GO_MOD_DISCOVERY_MODULES_JSON} --git-repo ${GO_MOD_DISCOVERY_GIT_REPO} --git-ref ${GO_MOD_DISCOVERY_GIT_REF} --recipedir ${GO_MOD_DISCOVERY_RECIPEDIR}"
-            fi
+    echo "======================================================================"
+    echo "EXTRACTING MODULE METADATA: ${PN} ${PV}"
+    echo "======================================================================"
+    echo "Cache:  ${DISCOVERY_CACHE}"
+    echo "Output: ${GO_MOD_DISCOVERY_MODULES_JSON}"
+    echo ""
+
+    # Find the extraction script
+    EXTRACT_SCRIPT=""
+    for layer in ${BBLAYERS}; do
+        if [ -f "${layer}/scripts/extract-discovered-modules.py" ]; then
+            EXTRACT_SCRIPT="${layer}/scripts/extract-discovered-modules.py"
+            break
         fi
-    elif [ "${GO_MOD_DISCOVERY_SKIP_GENERATE}" = "1" ]; then
+    done
+
+    if [ -z "${EXTRACT_SCRIPT}" ]; then
+        bbfatal "Could not find extract-discovered-modules.py in any layer"
+    fi
+
+    python3 "${EXTRACT_SCRIPT}" \
+        --gomodcache "${DISCOVERY_CACHE}" \
+        --output "${GO_MOD_DISCOVERY_MODULES_JSON}"
+
+    if [ $? -eq 0 ]; then
+        MODULE_COUNT=$(python3 -c "import json; print(len(json.load(open('${GO_MOD_DISCOVERY_MODULES_JSON}'))['modules']))" 2>/dev/null || echo "?")
         echo ""
-        echo "Skipping automatic generation (GO_MOD_DISCOVERY_SKIP_GENERATE=1)"
+        echo "======================================================================"
+        echo "EXTRACTION COMPLETE"
+        echo "======================================================================"
+        echo "Modules extracted: ${MODULE_COUNT}"
+        echo "Output file:       ${GO_MOD_DISCOVERY_MODULES_JSON}"
         echo ""
-        echo "NEXT STEP: Regenerate recipe manually:"
+        echo "Next step:"
+        echo "  bitbake ${PN} -c generate_modules"
         echo ""
-        echo "   ./meta-virtualization/scripts/oe-go-mod-fetcher.py \\"
-        echo "     --discovered-modules ${GO_MOD_DISCOVERY_MODULES_JSON} \\"
-        echo "     --git-repo <your-git-repo-url> \\"
-        echo "     --git-ref <your-git-ref> \\"
-        echo "     --recipedir ${GO_MOD_DISCOVERY_RECIPEDIR}"
+    else
+        bbfatal "Module extraction failed"
+    fi
+}
+
+addtask extract_modules
+do_extract_modules[nostamp] = "1"
+do_extract_modules[vardeps] += "GO_MOD_DISCOVERY_DIR GO_MOD_DISCOVERY_MODULES_JSON"
+
+# =============================================================================
+# TASK 3: do_generate_modules - Generate .inc files
+# =============================================================================
+# This task generates go-mod-git.inc and go-mod-cache.inc from the extracted
+# modules.json file.
+#
+# Usage: bitbake <recipe> -c generate_modules
+#
+do_generate_modules() {
+    if [ ! -f "${GO_MOD_DISCOVERY_MODULES_JSON}" ]; then
+        bbfatal "Modules JSON not found: ${GO_MOD_DISCOVERY_MODULES_JSON}
+Run 'bitbake ${PN} -c extract_modules' first to create the modules file."
+    fi
+
+    if [ -z "${GO_MOD_DISCOVERY_GIT_REPO}" ]; then
+        bbfatal "GO_MOD_DISCOVERY_GIT_REPO must be set for recipe generation.
+Add to your recipe: GO_MOD_DISCOVERY_GIT_REPO = \"https://github.com/...\"
+Or run 'bitbake ${PN} -c show_upgrade_commands' to see manual options."
     fi
 
-    echo ""
-    echo "NOTE: Cache is stored OUTSIDE WORKDIR in a persistent location."
-    echo "      This cache survives 'bitbake ${PN} -c cleanall'!"
-    echo "      To clean: rm -rf ${GO_MOD_DISCOVERY_DIR}"
-    echo ""
     echo "======================================================================"
-}
+    echo "GENERATING RECIPE FILES: ${PN} ${PV}"
+    echo "======================================================================"
+    echo "Modules JSON: ${GO_MOD_DISCOVERY_MODULES_JSON}"
+    echo "Git repo:     ${GO_MOD_DISCOVERY_GIT_REPO}"
+    echo "Git ref:      ${GO_MOD_DISCOVERY_GIT_REF}"
+    echo "Recipe dir:   ${GO_MOD_DISCOVERY_RECIPEDIR}"
+    echo ""
 
-# Make this task manually runnable (not part of default build)
-# Run after unpack and patch so source is available
-addtask discover_modules after do_patch
+    # Find the fetcher script
+    FETCHER_SCRIPT=""
+    for layer in ${BBLAYERS}; do
+        if [ -f "${layer}/scripts/oe-go-mod-fetcher.py" ]; then
+            FETCHER_SCRIPT="${layer}/scripts/oe-go-mod-fetcher.py"
+            break
+        fi
+    done
 
-# Task dependencies - need source unpacked and full toolchain available
-# Depend on do_prepare_recipe_sysroot to get cross-compiler for CGO
-do_discover_modules[depends] = "${PN}:do_prepare_recipe_sysroot"
+    if [ -z "${FETCHER_SCRIPT}" ]; then
+        bbfatal "Could not find oe-go-mod-fetcher.py in any layer"
+    fi
 
-# Enable network access for this task ONLY
-do_discover_modules[network] = "1"
+    python3 "${FETCHER_SCRIPT}" \
+        --discovered-modules "${GO_MOD_DISCOVERY_MODULES_JSON}" \
+        --git-repo "${GO_MOD_DISCOVERY_GIT_REPO}" \
+        --git-ref "${GO_MOD_DISCOVERY_GIT_REF}" \
+        --recipedir "${GO_MOD_DISCOVERY_RECIPEDIR}"
 
-# Don't create stamp file - allow running multiple times
-do_discover_modules[nostamp] = "1"
+    if [ $? -eq 0 ]; then
+        echo ""
+        echo "======================================================================"
+        echo "GENERATION COMPLETE"
+        echo "======================================================================"
+        echo "Files generated in: ${GO_MOD_DISCOVERY_RECIPEDIR}"
+        echo "  - go-mod-git.inc"
+        echo "  - go-mod-cache.inc"
+        echo ""
+        echo "You can now build the recipe:"
+        echo "  bitbake ${PN}"
+        echo ""
+    else
+        bbfatal "Recipe generation failed"
+    fi
+}
 
-# Track all configuration variables for proper task hashing
-do_discover_modules[vardeps] += "GO_MOD_DISCOVERY_DIR GO_MOD_DISCOVERY_SRCDIR \
-    GO_MOD_DISCOVERY_BUILD_TARGET GO_MOD_DISCOVERY_BUILD_TAGS \
-    GO_MOD_DISCOVERY_LDFLAGS GO_MOD_DISCOVERY_GOPATH GO_MOD_DISCOVERY_OUTPUT \
-    GO_MOD_DISCOVERY_MODULES_JSON GO_MOD_DISCOVERY_SKIP_EXTRACT \
-    GO_MOD_DISCOVERY_SKIP_GENERATE GO_MOD_DISCOVERY_GIT_REPO \
+addtask generate_modules
+do_generate_modules[nostamp] = "1"
+do_generate_modules[vardeps] += "GO_MOD_DISCOVERY_MODULES_JSON GO_MOD_DISCOVERY_GIT_REPO \
     GO_MOD_DISCOVERY_GIT_REF GO_MOD_DISCOVERY_RECIPEDIR"
 
-# Task to clean the persistent discovery cache
-# Usage: bitbake <recipe> -c clean_discovery
+# =============================================================================
+# TASK 4: do_discover_and_generate - All-in-one convenience task
+# =============================================================================
+# This task runs discover_modules, extract_modules, and generate_modules
+# in sequence. It's the "do everything" option.
+#
+# Usage: bitbake <recipe> -c discover_and_generate
+#
+do_discover_and_generate() {
+    echo "======================================================================"
+    echo "FULL DISCOVERY AND GENERATION: ${PN} ${PV}"
+    echo "======================================================================"
+    echo ""
+    echo "This task will run:"
+    echo "  1. discover_modules  - Build and download modules"
+    echo "  2. extract_modules   - Extract metadata to JSON"
+    echo "  3. generate_modules  - Generate .inc files"
+    echo ""
+}
+
+# Chain the tasks together using task dependencies
+python do_discover_and_generate_setdeps() {
+    # This runs discover -> extract -> generate in sequence
+    pass
+}
+
+addtask discover_and_generate after do_patch
+do_discover_and_generate[depends] = "${PN}:do_prepare_recipe_sysroot"
+do_discover_and_generate[network] = "1"
+do_discover_and_generate[nostamp] = "1"
+do_discover_and_generate[postfuncs] = "do_discover_modules do_extract_modules do_generate_modules"
+
+# =============================================================================
+# TASK: do_clean_discovery - Clean the persistent cache
+# =============================================================================
 do_clean_discovery() {
     if [ -d "${GO_MOD_DISCOVERY_DIR}" ]; then
         echo "Removing discovery cache: ${GO_MOD_DISCOVERY_DIR}"
@@ -439,3 +439,96 @@ do_clean_discovery() {
 addtask clean_discovery
 do_clean_discovery[nostamp] = "1"
 do_clean_discovery[vardeps] += "GO_MOD_DISCOVERY_DIR"
+
+# =============================================================================
+# TASK: do_show_upgrade_commands - Show command lines without running
+# =============================================================================
+python do_show_upgrade_commands() {
+    import os
+
+    pn = d.getVar('PN')
+    pv = d.getVar('PV')
+    git_repo = d.getVar('GO_MOD_DISCOVERY_GIT_REPO') or '<GIT_REPO_URL>'
+    git_ref = d.getVar('GO_MOD_DISCOVERY_GIT_REF') or d.getVar('SRCREV') or '<GIT_REF>'
+    recipedir = d.getVar('GO_MOD_DISCOVERY_RECIPEDIR') or d.getVar('FILE_DIRNAME')
+    discovery_dir = d.getVar('GO_MOD_DISCOVERY_DIR')
+    modules_json = d.getVar('GO_MOD_DISCOVERY_MODULES_JSON')
+
+    # Find script locations
+    fetcher_script = None
+    extract_script = None
+    for layer in d.getVar('BBLAYERS').split():
+        candidate = os.path.join(layer, 'scripts', 'oe-go-mod-fetcher.py')
+        if os.path.exists(candidate):
+            fetcher_script = candidate
+        candidate = os.path.join(layer, 'scripts', 'extract-discovered-modules.py')
+        if os.path.exists(candidate):
+            extract_script = candidate
+
+    fetcher_script = fetcher_script or './meta-virtualization/scripts/oe-go-mod-fetcher.py'
+    extract_script = extract_script or './meta-virtualization/scripts/extract-discovered-modules.py'
+
+    bb.plain("")
+    bb.plain("=" * 70)
+    bb.plain(f"UPGRADE COMMANDS FOR: {pn} {pv}")
+    bb.plain("=" * 70)
+    bb.plain("")
+    bb.plain("Option 1: Generate from git repository (no BitBake required)")
+    bb.plain("-" * 70)
+    bb.plain("")
+    bb.plain("Run from your build directory:")
+    bb.plain("")
+    bb.plain(f"  {fetcher_script} \\")
+    bb.plain(f"    --git-repo {git_repo} \\")
+    bb.plain(f"    --git-ref {git_ref} \\")
+    bb.plain(f"    --recipedir {recipedir}")
+    bb.plain("")
+    bb.plain("")
+    bb.plain("Option 2: BitBake discovery (step by step)")
+    bb.plain("-" * 70)
+    bb.plain("")
+    bb.plain(f"  bitbake {pn} -c discover_modules    # Download modules (needs network)")
+    bb.plain(f"  bitbake {pn} -c extract_modules     # Extract metadata to JSON")
+    bb.plain(f"  bitbake {pn} -c generate_modules    # Generate .inc files")
+    bb.plain("")
+    bb.plain("")
+    bb.plain("Option 3: BitBake discovery (all-in-one)")
+    bb.plain("-" * 70)
+    bb.plain("")
+    bb.plain(f"  bitbake {pn} -c discover_and_generate")
+    bb.plain("")
+    bb.plain("")
+    bb.plain("Option 4: Use existing discovery cache")
+    bb.plain("-" * 70)
+    bb.plain("")
+    bb.plain(f"Discovery cache: {discovery_dir}")
+    bb.plain("")
+    bb.plain("Extract modules from cache:")
+    bb.plain("")
+    bb.plain(f"  {extract_script} \\")
+    bb.plain(f"    --gomodcache {discovery_dir}/cache \\")
+    bb.plain(f"    --output {modules_json}")
+    bb.plain("")
+    bb.plain("Then generate .inc files:")
+    bb.plain("")
+    bb.plain(f"  {fetcher_script} \\")
+    bb.plain(f"    --discovered-modules {modules_json} \\")
+    bb.plain(f"    --git-repo {git_repo} \\")
+    bb.plain(f"    --git-ref {git_ref} \\")
+    bb.plain(f"    --recipedir {recipedir}")
+    bb.plain("")
+    bb.plain("")
+    bb.plain("Generated files:")
+    bb.plain("-" * 70)
+    bb.plain("")
+    bb.plain("  go-mod-git.inc   - SRC_URI entries for fetching module git repos")
+    bb.plain("  go-mod-cache.inc - Module path mappings for cache creation")
+    bb.plain("")
+    bb.plain("=" * 70)
+    bb.plain("")
+}
+
+addtask show_upgrade_commands
+do_show_upgrade_commands[nostamp] = "1"
+do_show_upgrade_commands[vardeps] += "GO_MOD_DISCOVERY_GIT_REPO GO_MOD_DISCOVERY_GIT_REF \
+    GO_MOD_DISCOVERY_RECIPEDIR GO_MOD_DISCOVERY_DIR GO_MOD_DISCOVERY_MODULES_JSON"