From 545ecc18af39a48a0543c844aa1652c2ed29c5d0 Mon Sep 17 00:00:00 2001 From: Bruce Ashfield Date: Mon, 6 Apr 2026 20:47:03 +0000 Subject: k3s: embed manifests and traefik charts, add server config Embed system manifests and Helm charts into the k3s binary so that k3s's deploy.Stage() can extract and process them at runtime with template variable substitution (e.g., %{SYSTEM_DEFAULT_REGISTRY}%). Previously, raw manifest files were installed directly to the server manifests directory, but template variables were not substituted, causing InvalidImageName errors for system pods. Now manifests are copied to pkg/deploy/embed/ before Go compilation, matching upstream's scripts/build workflow. Traefik ingress controller support is added as a PACKAGECONFIG option (enabled by default). When enabled, the traefik and traefik-crd Helm chart tarballs are downloaded during do_fetch and embedded into the binary at pkg/static/embed/charts/. When disabled, traefik is added to the disable list in config.yaml. A default /etc/rancher/k3s/config.yaml is installed that disables the cloud-controller-manager (not needed for standalone/QEMU environments) and conditionally disables traefik. The --disable-cloud-controller flag is removed from k3s.service since it is now in the config file. To disable traefik: PACKAGECONFIG:remove:pn-k3s = "traefik" Signed-off-by: Bruce Ashfield --- recipes-containers/k3s/k3s/k3s.service | 2 +- recipes-containers/k3s/k3s_git.bb | 46 +++++++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/recipes-containers/k3s/k3s/k3s.service b/recipes-containers/k3s/k3s/k3s.service index 42dc1832..f56110a4 100644 --- a/recipes-containers/k3s/k3s/k3s.service +++ b/recipes-containers/k3s/k3s/k3s.service @@ -34,7 +34,7 @@ ExecStartPre=-/sbin/modprobe overlay # Create flannel subnet.env if not present — k3s's embedded flannel # controller expects this file for CNI plugin configuration ExecStartPre=/bin/sh -c 'mkdir -p /run/flannel && test -f /run/flannel/subnet.env || echo "FLANNEL_NETWORK=10.42.0.0/16\nFLANNEL_SUBNET=10.42.0.1/24\nFLANNEL_MTU=1450\nFLANNEL_IPMASQ=true" > /run/flannel/subnet.env' -ExecStart=/usr/local/bin/k3s server --disable-cloud-controller +ExecStart=/usr/local/bin/k3s server # Avoid any delay due to this service when the system is rebooting or shutting # down by using the k3s-killall.sh script to kill all of the running k3s # services and containers diff --git a/recipes-containers/k3s/k3s_git.bb b/recipes-containers/k3s/k3s_git.bb index f478592f..e2d0a33e 100644 --- a/recipes-containers/k3s/k3s_git.bb +++ b/recipes-containers/k3s/k3s_git.bb @@ -13,6 +13,18 @@ SRC_URI = "git://github.com/rancher/k3s.git;branch=release-1.35;name=k3s;protoco file://k3s-killall.sh \ " +# Traefik Helm charts — downloaded and embedded into the k3s binary +# so they can be served via the k3s static chart endpoint at runtime. +# Opt-in via PACKAGECONFIG since traefik is not needed for basic k3s +# operation and the chart images must be pullable at runtime. +TRAEFIK_CHART_VERSION = "39.0.501+up39.0.5" +SRC_URI += "${@bb.utils.contains('PACKAGECONFIG', 'traefik', \ + 'https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-${TRAEFIK_CHART_VERSION}.tgz;name=traefik-crd;unpack=0;subdir=charts \ + https://k3s.io/k3s-charts/assets/traefik/traefik-${TRAEFIK_CHART_VERSION}.tgz;name=traefik;unpack=0;subdir=charts', \ + '', d)}" +SRC_URI[traefik-crd.sha256sum] = "c6245bdcfd193d10ec956d90c50e0d1a3fa1bde541df80744e72eee91b054640" +SRC_URI[traefik.sha256sum] = "888de9d098769b9199238076bee225d1b10dba1e889b0e28127eddc38cdb435b" + SRC_URI[k3s.md5sum] = "363d3a08dc0b72ba6e6577964f6e94a5" SRCREV_k3s = "4841276da0cf9f6f3e323b6cc8b10da381331f98" @@ -22,6 +34,9 @@ PV = "v1.35.2+k3s1+git" # K3s uses flannel for CNI networking, not the containerd bridge config CNI_NETWORKING_FILES ?= "${UNPACKDIR}/cni-flannel.conflist" +PACKAGECONFIG ??= "traefik" +PACKAGECONFIG[traefik] = ",,," + # Build tags - used by both do_compile and do_discover_modules TAGS = "static_build netcgo osusergo providerless" @@ -89,6 +104,20 @@ do_compile() { # go.mod and go.sum are synchronized by do_sync_go_files task # No manual fixes needed - discovery and recipe generation handle version matching + # Populate embed directories before build — upstream scripts/build + # does this to embed manifests and static content into the binary + # via go:embed. Without this, k3s's deploy.Stage() has no manifests + # to process and system components (coredns, metrics-server, etc.) + # are not deployed on first server start. + cp -a manifests/* pkg/deploy/embed/ + + # Embed traefik Helm charts if enabled via PACKAGECONFIG + if ${@bb.utils.contains('PACKAGECONFIG', 'traefik', 'true', 'false', d)}; then + mkdir -p pkg/static/embed/charts + cp -a ${WORKDIR}/sources/charts/traefik-crd-${TRAEFIK_CHART_VERSION}.tgz pkg/static/embed/charts/ + cp -a ${WORKDIR}/sources/charts/traefik-${TRAEFIK_CHART_VERSION}.tgz pkg/static/embed/charts/ + fi + VERSION_GOLANG="$(go version | cut -d" " -f3)" ${GO} build -trimpath -tags "${TAGS}" -ldflags "-X github.com/k3s-io/k3s/pkg/version.UpstreamGolang=$VERSION_GOLANG ${GO_BUILD_LDFLAGS} -w -s" -o ./dist/artifacts/k3s ./cmd/server/main.go @@ -118,9 +147,20 @@ do_install() { mkdir -p ${D}${datadir}/k3s/ install -m 0755 ${S}/src/import/contrib/util/check-config.sh ${D}${datadir}/k3s/ - # Create server manifests directory — k3s expects this at startup for - # auto-deploying system components (coredns, traefik, etc.) + # Create server manifests directory — k3s's deploy.Stage() writes + # processed manifests here at runtime (template variables substituted) install -d "${D}/var/lib/rancher/k3s/server/manifests" + + # Install default k3s config — disable traefik if not in PACKAGECONFIG + install -d "${D}${sysconfdir}/rancher/k3s" + echo "# k3s server configuration (generated by recipe)" \ + > "${D}${sysconfdir}/rancher/k3s/config.yaml" + echo "disable-cloud-controller: true" \ + >> "${D}${sysconfdir}/rancher/k3s/config.yaml" + if ! ${@bb.utils.contains('PACKAGECONFIG', 'traefik', 'true', 'false', d)}; then + echo "disable:" >> "${D}${sysconfdir}/rancher/k3s/config.yaml" + echo " - traefik" >> "${D}${sysconfdir}/rancher/k3s/config.yaml" + fi } PACKAGES =+ "${PN}-server ${PN}-agent" @@ -131,7 +171,7 @@ SYSTEMD_SERVICE:${PN}-agent = "${@bb.utils.contains('DISTRO_FEATURES','systemd', SYSTEMD_AUTO_ENABLE:${PN}-agent = "disable" FILES:${PN}-agent = "${BIN_PREFIX}/bin/k3s-agent" -FILES:${PN} += "${BIN_PREFIX}/bin/*" +FILES:${PN} += "${BIN_PREFIX}/bin/* /var/lib/rancher/k3s/server/manifests ${sysconfdir}/rancher/k3s" RDEPENDS:${PN} = "k3s-cni conntrack-tools coreutils findutils iptables iproute2 ipset virtual-containerd" RDEPENDS:${PN}-server = "${PN}" -- cgit v1.2.3-54-g00ecf