From 2385a741401386d129a0cd4dc4c332d3d54af4b3 Mon Sep 17 00:00:00 2001 From: Bruce Ashfield Date: Mon, 8 Dec 2025 14:23:46 +0000 Subject: docs: add QUICKSTART for go-mod-vcs Signed-off-by: Bruce Ashfield --- scripts/QUICKSTART-oe-go-mod-vcs.md | 408 ++++++++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 scripts/QUICKSTART-oe-go-mod-vcs.md (limited to 'scripts') diff --git a/scripts/QUICKSTART-oe-go-mod-vcs.md b/scripts/QUICKSTART-oe-go-mod-vcs.md new file mode 100644 index 00000000..40db7885 --- /dev/null +++ b/scripts/QUICKSTART-oe-go-mod-vcs.md @@ -0,0 +1,408 @@ +# Quickstart: Go Module VCS Build System for Yocto/BitBake + +This guide covers how to create and maintain Go recipes using the `go-mod-vcs` system, which provides reproducible, offline Go builds by fetching dependencies directly from their git repositories. + +## Overview + +The `go-mod-vcs` system replaces vendor directories with a build-time module cache constructed from git repositories. This provides: + +- **Reproducible builds** - Every module comes from a verified git commit +- **Offline builds** - After `do_fetch`, no network access is required +- **Auditable dependencies** - Every dependency is traceable to a specific git commit +- **Smaller recipes** - No need to maintain large vendor directories in source trees + +### How It Works + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Your Recipe │ +│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────────┐ │ +│ │ myapp_git.bb │ │ go-mod-git.inc │ │ go-mod-cache.inc │ │ +│ │ (your code) │ │ (git fetches) │ │ (module metadata) │ │ +│ └─────────────────┘ └─────────────────┘ └─────────────────────┘ │ +└───────────────────────────────┬─────────────────────────────────────┘ + │ + ┌───────────────────────┼───────────────────────┐ + │ │ │ + ▼ ▼ ▼ + ┌─────────┐ ┌─────────────┐ ┌───────────────┐ + │do_fetch │ │do_create_ │ │ do_compile │ + │ │──────────▶│module_cache │────────▶│ │ + │(git) │ │(build cache)│ │(go build) │ + └─────────┘ └─────────────┘ └───────────────┘ +``` + +1. **do_fetch** - BitBake fetches all git repositories listed in `go-mod-git.inc` +2. **do_create_module_cache** - The `go-mod-vcs` class builds a Go module cache from git checkouts +3. **do_compile** - Go builds offline using the pre-populated module cache + +--- + +## Converting an Existing Recipe + +### Step 1: Add Discovery Configuration + +Add these lines to your recipe (before `inherit go`): + +```bitbake +# go-mod-discovery configuration +GO_MOD_DISCOVERY_BUILD_TARGET = "./cmd/..." # Your build target +GO_MOD_DISCOVERY_GIT_REPO = "https://github.com/org/repo.git" +GO_MOD_DISCOVERY_GIT_REF = "${SRCREV}" + +inherit go-mod-discovery +``` + +### Step 2: Include the Generated Files + +Add includes after your `SRC_URI`: + +```bitbake +SRC_URI = "git://github.com/org/repo;branch=main;protocol=https;destsuffix=${GO_SRCURI_DESTSUFFIX}" + +include go-mod-git.inc +include go-mod-cache.inc +``` + +### Step 3: Create Placeholder Files + +Create empty placeholder files (they'll be generated): + +```bash +cd recipes-containers/myapp/ +touch go-mod-git.inc go-mod-cache.inc +``` + +### Step 4: Run Discovery + +```bash +bitbake myapp -c discover_and_generate +``` + +This will: +1. Build your project with network access to discover dependencies +2. Extract module metadata from the Go module cache +3. Generate `go-mod-git.inc` and `go-mod-cache.inc` files + +### Step 5: Update do_compile() + +Ensure your `do_compile()` doesn't set conflicting environment variables. The `go-mod-vcs.bbclass` automatically sets: + +- `GOMODCACHE` - Points to the built module cache +- `GOPROXY=off` - Enforces offline build +- `GOSUMDB=off` - Disables checksum database +- `GOTOOLCHAIN=local` - Uses the native Go toolchain + +A minimal `do_compile()`: + +```bitbake +do_compile() { + cd ${S}/src/import + + export GOPATH="${S}/src/import/.gopath:${STAGING_DIR_TARGET}/${prefix}/local/go" + export CGO_ENABLED="1" + + ${GO} build -trimpath ./cmd/... +} +``` + +### Step 6: Build and Test + +```bash +bitbake myapp +``` + +--- + +## Creating a New Recipe + +### Minimal Recipe Template + +```bitbake +SUMMARY = "My Go Application" +HOMEPAGE = "https://github.com/org/myapp" + +SRCREV = "abc123def456..." +SRC_URI = "git://github.com/org/myapp;branch=main;protocol=https;destsuffix=${GO_SRCURI_DESTSUFFIX}" + +include go-mod-git.inc +include go-mod-cache.inc + +LICENSE = "Apache-2.0" +LIC_FILES_CHKSUM = "file://src/import/LICENSE;md5=..." + +GO_IMPORT = "import" +PV = "v1.0.0+git" + +# go-mod-discovery configuration +GO_MOD_DISCOVERY_BUILD_TARGET = "./cmd/..." +GO_MOD_DISCOVERY_GIT_REPO = "https://github.com/org/myapp.git" +GO_MOD_DISCOVERY_GIT_REF = "${SRCREV}" + +inherit go goarch +inherit go-mod-discovery + +do_compile() { + cd ${S}/src/import + + export GOPATH="${S}/src/import/.gopath:${STAGING_DIR_TARGET}/${prefix}/local/go" + export CGO_ENABLED="1" + + ${GO} build -trimpath -o ${B}/myapp ./cmd/myapp +} + +do_install() { + install -d ${D}${bindir} + install -m 755 ${B}/myapp ${D}${bindir}/ +} +``` + +### Generate Dependencies + +```bash +# Create placeholder files +touch go-mod-git.inc go-mod-cache.inc + +# Run discovery +bitbake myapp -c discover_and_generate + +# Build +bitbake myapp +``` + +--- + +## Updating a Recipe to a New Version + +### Quick Update (Same Repository) + +1. **Update the SRCREV** in your recipe: + ```bitbake + SRCREV = "new_commit_hash_here" + ``` + +2. **Re-run discovery**: + ```bash + bitbake myapp -c discover_and_generate + ``` + +3. **Build**: + ```bash + bitbake myapp + ``` + +### Full Workflow Example + +```bash +# 1. Find the new commit/tag +git ls-remote https://github.com/org/myapp refs/tags/v2.0.0 + +# 2. Update SRCREV in recipe +# SRCREV = "abc123..." + +# 3. Clean old discovery cache (optional, recommended for major updates) +bitbake myapp -c clean_discovery + +# 4. Run discovery and generation +bitbake myapp -c discover_and_generate + +# 5. Review generated files +git diff recipes-containers/myapp/go-mod-*.inc + +# 6. Build and test +bitbake myapp + +# 7. Commit changes +git add recipes-containers/myapp/ +git commit -m "myapp: update to v2.0.0" +``` + +--- + +## Discovery Tasks Reference + +| Task | Purpose | Network? | +|------|---------|----------| +| `discover_modules` | Build project and download modules from proxy | Yes | +| `extract_modules` | Extract metadata from cache to JSON | No | +| `generate_modules` | Generate .inc files from metadata | No | +| `discover_and_generate` | All three steps in sequence | Yes | +| `show_upgrade_commands` | Print command lines without running | No | +| `clean_discovery` | Remove the discovery cache | No | + +### Step-by-Step vs All-in-One + +**All-in-one** (recommended for most cases): +```bash +bitbake myapp -c discover_and_generate +``` + +**Step-by-step** (useful for debugging): +```bash +bitbake myapp -c discover_modules # Download modules +bitbake myapp -c extract_modules # Extract to JSON +bitbake myapp -c generate_modules # Generate .inc files +``` + +--- + +## Configuration Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `GO_MOD_DISCOVERY_BUILD_TARGET` | *(required)* | Go build target (e.g., `./cmd/...`) | +| `GO_MOD_DISCOVERY_GIT_REPO` | *(required for generate)* | Git repository URL | +| `GO_MOD_DISCOVERY_GIT_REF` | `${SRCREV}` | Git commit/tag | +| `GO_MOD_DISCOVERY_SRCDIR` | `${S}/src/import` | Directory containing go.mod | +| `GO_MOD_DISCOVERY_BUILD_TAGS` | `${TAGS}` | Go build tags | +| `GO_MOD_DISCOVERY_RECIPEDIR` | `${FILE_DIRNAME}` | Output directory for .inc files | + +--- + +## Troubleshooting + +### "module lookup disabled by GOPROXY=off" + +This error during `do_compile` means a module is missing from the cache. + +**Fix:** +```bash +# Re-run discovery to find missing modules +bitbake myapp -c discover_and_generate +bitbake myapp +``` + +### Discovery Cache Location + +The discovery cache persists in `${TOPDIR}/go-mod-discovery/${PN}/${PV}/` and survives `bitbake -c cleanall`. To fully reset: + +```bash +bitbake myapp -c clean_discovery +bitbake myapp -c discover_and_generate +``` + +### Viewing Generated Commands + +To see what commands would be run without executing them: + +```bash +bitbake myapp -c show_upgrade_commands +``` + +### Build Fails After SRCREV Update + +If changing SRCREV causes sstate errors: + +```bash +# Clean sstate for the recipe +bitbake myapp -c cleansstate + +# Re-run discovery +bitbake myapp -c discover_and_generate + +# Build +bitbake myapp +``` + +### Multiple Source Repositories + +For recipes with multiple git sources, use named SRCREVs: + +```bitbake +SRCREV_myapp = "abc123..." +SRCREV_plugins = "def456..." +SRCREV_FORMAT = "myapp_plugins" + +SRC_URI = "\ + git://github.com/org/myapp;name=myapp;branch=main;protocol=https;destsuffix=${GO_SRCURI_DESTSUFFIX} \ + git://github.com/org/plugins;name=plugins;branch=main;protocol=https;destsuffix=${GO_SRCURI_DESTSUFFIX}/plugins \ +" + +GO_MOD_DISCOVERY_GIT_REF = "${SRCREV_myapp}" +``` + +--- + +## Example Recipes + +### Simple Recipe (rootlesskit) + +```bitbake +SRCREV_rootless = "8059d35092db167ec53cae95fb6aa37fc577060c" +SRCREV_FORMAT = "rootless" + +SRC_URI = "git://github.com/rootless-containers/rootlesskit;name=rootless;branch=master;protocol=https;destsuffix=${GO_SRCURI_DESTSUFFIX}" + +include go-mod-git.inc +include go-mod-cache.inc + +GO_MOD_DISCOVERY_BUILD_TARGET = "./cmd/..." +GO_MOD_DISCOVERY_GIT_REPO = "https://github.com/rootless-containers/rootlesskit.git" +GO_MOD_DISCOVERY_GIT_REF = "${SRCREV_rootless}" + +inherit go goarch go-mod-discovery +``` + +### Complex Recipe (k3s with build tags) + +```bitbake +TAGS = "static_build netcgo osusergo providerless" + +GO_MOD_DISCOVERY_BUILD_TARGET = "./cmd/server/main.go" +GO_MOD_DISCOVERY_GIT_REPO = "https://github.com/rancher/k3s.git" +GO_MOD_DISCOVERY_GIT_REF = "${SRCREV_k3s}" + +inherit go goarch go-mod-discovery +``` + +--- + +## Files Generated + +### go-mod-git.inc + +Contains `SRC_URI` entries for each module's git repository: + +```bitbake +SRC_URI += "git://github.com/spf13/cobra;protocol=https;nobranch=1;rev=e94f6d0...;name=git_41456771_1;destsuffix=vcs_cache/2d91d6bc..." +``` + +### go-mod-cache.inc + +Contains module metadata and inherits the build class: + +```bitbake +inherit go-mod-vcs + +GO_MODULE_CACHE_DATA = '[{"module":"github.com/spf13/cobra","version":"v1.8.1","vcs_hash":"2d91d6bc...","timestamp":"2024-06-01T10:31:11Z","subdir":""},...]' +``` + +--- + +## Advanced: Manual Script Invocation + +For cases where BitBake isn't available or you need more control: + +```bash +# Direct generation from git (no BitBake needed) +python3 ./meta-virtualization/scripts/oe-go-mod-fetcher.py \ + --git-repo https://github.com/org/myapp.git \ + --git-ref abc123... \ + --recipedir ./meta-virtualization/recipes-containers/myapp/ + +# Using existing discovery cache +python3 ./meta-virtualization/scripts/oe-go-mod-fetcher.py \ + --discovered-modules ${TOPDIR}/go-mod-discovery/myapp/v1.0.0/modules.json \ + --git-repo https://github.com/org/myapp.git \ + --git-ref abc123... \ + --recipedir ./meta-virtualization/recipes-containers/myapp/ +``` + +--- + +## Getting Help + +- **Show available commands**: `bitbake myapp -c show_upgrade_commands` +- **Architecture details**: See `scripts/ARCHITECTURE.md` +- **Report issues**: Check the generated files and error messages from discovery + -- cgit v1.2.3-54-g00ecf