k3s: add multi-node cluster support with role-based boot

Add infrastructure for booting the same container-image-host image
as either a k3s server or agent, controlled via kernel cmdline
parameters (k3s.role=server|agent).

k3s-role-setup.service / k3s-role-setup.sh:
- Reads k3s.role, k3s.server, k3s.token, k3s.node-name, k3s.node-ip,
  k3s.iface from kernel cmdline
- Configures cluster network interface IP via networkd drop-in
- For agent role: masks k3s.service, writes agent environment file,
  starts k3s-agent.service
- For server role: masks k3s-agent.service (default)

10-k3s-cluster.network:
- Claims the cluster interface (eth1) via virt_networking bbclass
- Disables DHCP to prevent systemd-networkd from interfering
- Static IP added at boot by role-setup via drop-in

k3s-get-token.sh:
- Helper script to display the server join token
- Waits for token file if k3s is still starting

k3s-agent.service:
- Add EnvironmentFile for /etc/default/k3s-agent (K3S_URL, K3S_TOKEN)
- Add After=k3s-role-setup.service and network-online.target

k3s.service:
- Add After=k3s-role-setup.service

packagegroup-kubernetes.bb:
- k3s-host packagegroup now includes k3s-agent (both roles available)
- Both k3s-host and k3s-node include k3s-net-conf

Signed-off-by: Bruce Ashfield <bruce.ashfield@gmail.com>

7 files changed, 203 insertions, 3 deletions

diff --git a/recipes-containers/k3s/k3s/10-k3s-cluster.network b/recipes-containers/k3s/k3s/10-k3s-cluster.network
new file mode 100644
index 00000000..c821a8fe
--- /dev/null
+++ b/recipes-containers/k3s/k3s/10-k3s-cluster.network
@@ -0,0 +1,20 @@
+# K3s cluster network interface configuration
+#
+# Claims the secondary interface (eth1) for k3s cluster networking,
+# preventing systemd-networkd's default catch-all from configuring
+# it with DHCP. The static IP address is configured at boot by
+# k3s-role-setup.service via a runtime drop-in based on the
+# k3s.node-ip= kernel cmdline parameter.
+#
+# If no k3s.node-ip= is set, the interface stays up with no address
+# (link-local only), which is harmless.
+
+[Match]
+Name=eth1
+
+[Network]
+DHCP=no
+LinkLocalAddressing=ipv6
+
+[Link]
+RequiredForOnline=no
diff --git a/recipes-containers/k3s/k3s/k3s-agent.service b/recipes-containers/k3s/k3s/k3s-agent.service
index 40d0564b..0304d640 100644
--- a/recipes-containers/k3s/k3s/k3s-agent.service
+++ b/recipes-containers/k3s/k3s/k3s-agent.service
@@ -3,13 +3,17 @@
 Description=Lightweight Kubernetes Agent
 Documentation=https://k3s.io
 Requires=containerd.service
-After=containerd.service
+After=containerd.service k3s-role-setup.service
+After=network-online.target
+Wants=network-online.target
 
 [Install]
 WantedBy=multi-user.target
 
 [Service]
 Type=notify
+EnvironmentFile=-/etc/default/%N
+EnvironmentFile=-/etc/sysconfig/%N
 # Ensure CNI plugin binaries are discoverable
 Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/cni/bin:/usr/libexec/cni
 KillMode=control-group
diff --git a/recipes-containers/k3s/k3s/k3s-get-token.sh b/recipes-containers/k3s/k3s/k3s-get-token.sh
new file mode 100644
index 00000000..359ea2ff
--- /dev/null
+++ b/recipes-containers/k3s/k3s/k3s-get-token.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+# SPDX-FileCopyrightText: Copyright (C) 2025 Bruce Ashfield
+# SPDX-License-Identifier: MIT
+#
+# k3s-get-token — Display the k3s server join token
+#
+# Waits for the token file to be created (k3s server generates it
+# on first start) and prints it. Useful for setting up agent nodes.
+
+TOKEN_FILE="/var/lib/rancher/k3s/server/node-token"
+TIMEOUT=60
+
+if [ ! -f "$TOKEN_FILE" ]; then
+    echo "Waiting for k3s server to generate token..."
+    i=0
+    while [ ! -f "$TOKEN_FILE" ] && [ $i -lt $TIMEOUT ]; do
+        sleep 2
+        i=$((i + 2))
+    done
+fi
+
+if [ -f "$TOKEN_FILE" ]; then
+    echo ""
+    echo "=== K3s Join Token ==="
+    cat "$TOKEN_FILE"
+    echo ""
+    echo "To join an agent node:"
+    echo "  run-k3s-multinode.sh agent --token \$(k3s-get-token)"
+    echo ""
+else
+    echo "Token not found. Is k3s server running?"
+    echo "  systemctl status k3s"
+fi
diff --git a/recipes-containers/k3s/k3s/k3s-role-setup.service b/recipes-containers/k3s/k3s/k3s-role-setup.service
new file mode 100644
index 00000000..26d51ba3
--- /dev/null
+++ b/recipes-containers/k3s/k3s/k3s-role-setup.service
@@ -0,0 +1,16 @@
+[Unit]
+Description=K3s Role Setup from Kernel Cmdline
+Documentation=https://k3s.io
+After=local-fs.target
+Before=systemd-networkd.service network-online.target
+
+# Only run if k3s.role= is on the kernel cmdline
+ConditionKernelCommandLine=k3s.role
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/bin/k3s-role-setup.sh
+
+[Install]
+WantedBy=multi-user.target
diff --git a/recipes-containers/k3s/k3s/k3s-role-setup.sh b/recipes-containers/k3s/k3s/k3s-role-setup.sh
new file mode 100644
index 00000000..fa4ffb05
--- /dev/null
+++ b/recipes-containers/k3s/k3s/k3s-role-setup.sh
@@ -0,0 +1,113 @@
+#!/bin/sh
+# SPDX-FileCopyrightText: Copyright (C) 2025 Bruce Ashfield
+# SPDX-License-Identifier: MIT
+#
+# k3s-role-setup.sh — Configure k3s role from kernel cmdline
+#
+# Reads k3s.role= from /proc/cmdline and configures the system
+# to run as either a k3s server or agent.
+#
+# Kernel cmdline parameters:
+#   k3s.role=server  — run k3s server (default if not specified)
+#   k3s.role=agent   — disable k3s server, enable k3s agent
+#   k3s.server=IP    — server address for agent mode
+#   k3s.token=TOKEN  — join token for agent mode
+#   k3s.node-name=N  — override node name (useful when multiple VMs
+#                       boot from the same image)
+#   k3s.node-ip=IP   — set node IP address
+#   k3s.iface=IFACE  — flannel interface (default: eth1)
+
+CMDLINE=$(cat /proc/cmdline)
+
+get_param() {
+    echo "$CMDLINE" | tr ' ' '\n' | grep "^$1=" | cut -d= -f2-
+}
+
+ROLE=$(get_param k3s.role)
+SERVER=$(get_param k3s.server)
+TOKEN=$(get_param k3s.token)
+NODE_NAME=$(get_param k3s.node-name)
+NODE_IP=$(get_param k3s.node-ip)
+IFACE=$(get_param k3s.iface)
+
+# Default role is server
+ROLE="${ROLE:-server}"
+IFACE="${IFACE:-eth1}"
+
+# Configure cluster network interface IP via networkd drop-in.
+# The base 10-k3s-cluster.network (installed by virt_networking bbclass)
+# claims eth1 and disables DHCP. This drop-in adds the static address.
+if [ -n "$NODE_IP" ] && [ -n "$IFACE" ]; then
+    echo "Configuring $IFACE with $NODE_IP/24 via networkd drop-in"
+    DROPIN_DIR="/etc/systemd/network/10-k3s-cluster.network.d"
+    mkdir -p "$DROPIN_DIR"
+    cat > "$DROPIN_DIR/address.conf" << EOF
+[Network]
+Address=${NODE_IP}/24
+EOF
+    networkctl reload 2>/dev/null || true
+fi
+
+case "$ROLE" in
+    server)
+        # Server is the default — k3s.service is already enabled.
+        # Mask the agent service to prevent it from starting.
+        systemctl mask k3s-agent.service 2>/dev/null || true
+        echo "k3s role: server"
+        ;;
+
+    agent)
+        # Mask the server service to prevent it from starting
+        systemctl mask k3s.service 2>/dev/null || true
+
+        # Wipe any server state from previous boot
+        rm -rf /var/lib/rancher/k3s/server/tls \
+               /var/lib/rancher/k3s/server/cred \
+               /var/lib/rancher/k3s/server/token \
+               /var/lib/rancher/k3s/server/agent-token \
+               /var/lib/rancher/k3s/server/node-token \
+               /var/lib/rancher/k3s/server/db \
+               /etc/rancher/k3s/k3s.yaml
+
+        # Remove server-only config (disable-cloud-controller is
+        # server-only and crashes the agent)
+        rm -f /etc/rancher/k3s/config.yaml
+
+        # Build agent environment file with cmdline parameters
+        mkdir -p /etc/default
+        {
+            echo "# Generated by k3s-role-setup from kernel cmdline"
+            [ -n "$SERVER" ] && echo "K3S_URL=https://${SERVER}:6443"
+            [ -n "$TOKEN" ] && echo "K3S_TOKEN=${TOKEN}"
+        } > /etc/default/k3s-agent
+
+        # Build extra args
+        AGENT_ARGS=""
+        [ -n "$NODE_NAME" ] && AGENT_ARGS="$AGENT_ARGS --node-name $NODE_NAME"
+        [ -n "$NODE_IP" ] && AGENT_ARGS="$AGENT_ARGS --node-ip $NODE_IP"
+        [ -n "$IFACE" ] && AGENT_ARGS="$AGENT_ARGS --flannel-iface $IFACE"
+
+        if [ -n "$AGENT_ARGS" ]; then
+            # Create systemd override for extra args
+            mkdir -p /run/systemd/system/k3s-agent.service.d
+            cat > /run/systemd/system/k3s-agent.service.d/cmdline.conf << EOF
+[Service]
+ExecStart=
+ExecStart=/usr/local/bin/k3s agent $AGENT_ARGS
+EOF
+        fi
+
+        # Reload systemd to pick up overrides, then unmask and start
+        # the agent. --no-block queues the start without waiting,
+        # avoiding deadlock with the After= ordering.
+        systemctl daemon-reload
+        systemctl unmask k3s-agent.service 2>/dev/null || true
+        systemctl start k3s-agent.service --no-block
+
+        echo "k3s role: agent (server: ${SERVER:-not set})"
+        ;;
+
+    *)
+        echo "Unknown k3s.role=$ROLE, defaulting to server"
+        ;;
+esac
diff --git a/recipes-containers/k3s/k3s/k3s.service b/recipes-containers/k3s/k3s/k3s.service
index f56110a4..e7402867 100644
--- a/recipes-containers/k3s/k3s/k3s.service
+++ b/recipes-containers/k3s/k3s/k3s.service
@@ -4,7 +4,7 @@ Description=Lightweight Kubernetes
 Documentation=https://k3s.io
 Requires=containerd.service
 After=containerd.service
-After=network-online.target
+After=network-online.target k3s-role-setup.service
 Wants=network-online.target
 
 [Install]
diff --git a/recipes-containers/k3s/k3s_git.bb b/recipes-containers/k3s/k3s_git.bb
index e2d0a33e..1b5f3430 100644
--- a/recipes-containers/k3s/k3s_git.bb
+++ b/recipes-containers/k3s/k3s_git.bb
@@ -11,6 +11,10 @@ SRC_URI = "git://github.com/rancher/k3s.git;branch=release-1.35;name=k3s;protoco
            file://k3s-clean \
            file://cni-flannel.conflist \
            file://k3s-killall.sh \
+           file://k3s-role-setup.sh \
+           file://k3s-role-setup.service \
+           file://k3s-get-token.sh \
+           file://10-k3s-cluster.network \
           "
 
 # Traefik Helm charts — downloaded and embedded into the k3s binary
@@ -34,6 +38,11 @@ PV = "v1.35.2+k3s1+git"
 # K3s uses flannel for CNI networking, not the containerd bridge config
 CNI_NETWORKING_FILES ?= "${UNPACKDIR}/cni-flannel.conflist"
 
+# Claim the cluster network interface (eth1) so systemd-networkd's
+# default catch-all doesn't configure it with DHCP. The static IP
+# is set at boot by k3s-role-setup.service via a networkd drop-in.
+VIRT_NETWORKING_FILES ?= "${UNPACKDIR}/10-k3s-cluster.network"
+
 PACKAGECONFIG ??= "traefik"
 PACKAGECONFIG[traefik] = ",,,"
 
@@ -63,6 +72,7 @@ inherit go
 inherit goarch
 inherit systemd
 inherit cni_networking
+inherit virt_networking
 inherit go-mod-discovery
 
 BB_GIT_SHALLOW = "1"
@@ -140,8 +150,11 @@ do_install() {
         if ${@bb.utils.contains('DISTRO_FEATURES','systemd','true','false',d)}; then
                 install -D -m 0644 "${UNPACKDIR}/k3s.service" "${D}${systemd_system_unitdir}/k3s.service"
                 install -D -m 0644 "${UNPACKDIR}/k3s-agent.service" "${D}${systemd_system_unitdir}/k3s-agent.service"
+                install -D -m 0644 "${UNPACKDIR}/k3s-role-setup.service" "${D}${systemd_system_unitdir}/k3s-role-setup.service"
                 sed -i "s#\(Exec\)\(.*\)=\(.*\)\(k3s\)#\1\2=${BIN_PREFIX}/bin/\4#g" "${D}${systemd_system_unitdir}/k3s.service" "${D}${systemd_system_unitdir}/k3s-agent.service"
                 install -m 755 "${UNPACKDIR}/k3s-agent" "${D}${BIN_PREFIX}/bin"
+                install -m 755 "${UNPACKDIR}/k3s-role-setup.sh" "${D}${BIN_PREFIX}/bin/k3s-role-setup.sh"
+                install -m 755 "${UNPACKDIR}/k3s-get-token.sh" "${D}${BIN_PREFIX}/bin/k3s-get-token"
         fi
 
         mkdir -p ${D}${datadir}/k3s/
@@ -165,9 +178,10 @@ do_install() {
 
 PACKAGES =+ "${PN}-server ${PN}-agent"
 
-SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}-server ${PN}-agent','',d)}"
+SYSTEMD_PACKAGES = "${@bb.utils.contains('DISTRO_FEATURES','systemd','${PN}-server ${PN}-agent ${PN}','',d)}"
 SYSTEMD_SERVICE:${PN}-server = "${@bb.utils.contains('DISTRO_FEATURES','systemd','k3s.service','',d)}"
 SYSTEMD_SERVICE:${PN}-agent = "${@bb.utils.contains('DISTRO_FEATURES','systemd','k3s-agent.service','',d)}"
+SYSTEMD_SERVICE:${PN} = "${@bb.utils.contains('DISTRO_FEATURES','systemd','k3s-role-setup.service','',d)}"
 SYSTEMD_AUTO_ENABLE:${PN}-agent = "disable"
 
 FILES:${PN}-agent = "${BIN_PREFIX}/bin/k3s-agent"