From 52a307475a13a92cd3025d94b26ee43b1b59fcbd Mon Sep 17 00:00:00 2001 From: Bruce Ashfield Date: Thu, 5 Feb 2026 21:37:24 +0000 Subject: image-oci: add host layer type and delta-only copying Add two enhancements to multi-layer OCI image support: 1. Delta-only copying for directories/files layers: - directories and files layers now only copy content that doesn't already exist in the bundle rootfs from earlier layers - Prevents duplication when a directories layer references paths that were already populated by a packages layer - Logs show "delta: N copied, M skipped" for visibility 2. New 'host' layer type for build machine content: - Copies files from the build machine filesystem (outside Yocto) - Format: name:host:source_path:dest_path - Multiple pairs: name:host:src1:dst1+src2:dst2 - Emits warning at parse time about reproducibility impact - Fatal error if source path doesn't exist - Use case: deployment-specific config, certificates, keys that cannot be packaged in recipes Example: OCI_LAYERS = "\ base:packages:busybox \ app:directories:/opt/myapp \ certs:host:/etc/ssl/certs/ca.crt:/etc/ssl/certs/ca.crt \ " Signed-off-by: Bruce Ashfield --- classes/image-oci-umoci.inc | 61 +++++++++++++++++++++++++++++++++++++++------ classes/image-oci.bbclass | 43 +++++++++++++++++++++++++++----- 2 files changed, 90 insertions(+), 14 deletions(-) (limited to 'classes') diff --git a/classes/image-oci-umoci.inc b/classes/image-oci-umoci.inc index fbb77cd0..616c20ae 100644 --- a/classes/image-oci-umoci.inc +++ b/classes/image-oci-umoci.inc @@ -589,22 +589,67 @@ IMAGE_CMD:oci() { fi elif [ "$oci_layer_type" = "directories" ]; then - # Copy directories from IMAGE_ROOTFS + # Copy directories from IMAGE_ROOTFS (delta-only: skip files already in bundle) for oci_dir in $oci_layer_content; do if [ -d "${IMAGE_ROOTFS}$oci_dir" ]; then - mkdir -p "$image_bundle_name/rootfs$(dirname $oci_dir)" - cp -a "${IMAGE_ROOTFS}$oci_dir" "$image_bundle_name/rootfs$oci_dir" - bbnote "OCI: Added directory $oci_dir" + oci_delta_copied=0 + oci_delta_skipped=0 + # Walk the directory and copy only files not in bundle + while IFS= read -r oci_src_file; do + oci_rel_path="${oci_src_file#${IMAGE_ROOTFS}}" + oci_dst_file="$image_bundle_name/rootfs$oci_rel_path" + if [ ! -e "$oci_dst_file" ]; then + mkdir -p "$(dirname "$oci_dst_file")" + cp -a "$oci_src_file" "$oci_dst_file" + oci_delta_copied=$((oci_delta_copied + 1)) + else + oci_delta_skipped=$((oci_delta_skipped + 1)) + fi + done < <(find "${IMAGE_ROOTFS}$oci_dir" -type f -o -type l) + # Also copy empty directories + while IFS= read -r oci_src_dir; do + oci_rel_path="${oci_src_dir#${IMAGE_ROOTFS}}" + oci_dst_dir="$image_bundle_name/rootfs$oci_rel_path" + if [ ! -e "$oci_dst_dir" ]; then + mkdir -p "$oci_dst_dir" + fi + done < <(find "${IMAGE_ROOTFS}$oci_dir" -type d) + bbnote "OCI: Added directory $oci_dir (delta: $oci_delta_copied copied, $oci_delta_skipped skipped)" + else + bbwarn "OCI: Directory not found in IMAGE_ROOTFS: $oci_dir" fi done elif [ "$oci_layer_type" = "files" ]; then - # Copy specific files from IMAGE_ROOTFS + # Copy specific files from IMAGE_ROOTFS (delta-only: skip files already in bundle) for oci_file in $oci_layer_content; do if [ -e "${IMAGE_ROOTFS}$oci_file" ]; then - mkdir -p "$image_bundle_name/rootfs$(dirname $oci_file)" - cp -a "${IMAGE_ROOTFS}$oci_file" "$image_bundle_name/rootfs$oci_file" - bbnote "OCI: Added file $oci_file" + oci_dst_file="$image_bundle_name/rootfs$oci_file" + if [ ! -e "$oci_dst_file" ]; then + mkdir -p "$(dirname "$oci_dst_file")" + cp -a "${IMAGE_ROOTFS}$oci_file" "$oci_dst_file" + bbnote "OCI: Added file $oci_file" + else + bbnote "OCI: Skipped file $oci_file (already in bundle)" + fi + else + bbwarn "OCI: File not found in IMAGE_ROOTFS: $oci_file" + fi + done + + elif [ "$oci_layer_type" = "host" ]; then + # Copy files from build machine filesystem (outside Yocto) + # Format: source_path:dest_path pairs separated by + (already converted to space) + for oci_host_pair in $oci_layer_content; do + # Split on last : to handle paths that might contain : + oci_host_src="${oci_host_pair%:*}" + oci_host_dst="${oci_host_pair##*:}" + if [ -e "$oci_host_src" ]; then + mkdir -p "$image_bundle_name/rootfs$(dirname $oci_host_dst)" + cp -a "$oci_host_src" "$image_bundle_name/rootfs$oci_host_dst" + bbnote "OCI: Added from host: $oci_host_src -> $oci_host_dst" + else + bbfatal "OCI: Host path not found: $oci_host_src" fi done fi diff --git a/classes/image-oci.bbclass b/classes/image-oci.bbclass index 41f7b3d1..ea2b63df 100644 --- a/classes/image-oci.bbclass +++ b/classes/image-oci.bbclass @@ -167,9 +167,10 @@ OCI_LAYER_MODE ?= "single" # Each layer is defined as: "name:type:content" # # Layer Types: -# packages - Copy files installed by specified packages -# directories - Copy specific directories from IMAGE_ROOTFS -# files - Copy specific files from IMAGE_ROOTFS +# packages - Install packages using Yocto's package manager +# directories - Copy specific directories from IMAGE_ROOTFS (delta-only) +# files - Copy specific files from IMAGE_ROOTFS (delta-only) +# host - Copy files from build machine filesystem (outside Yocto) # # Format: Space-separated list of layer definitions # OCI_LAYERS = "layer1:type:content layer2:type:content ..." @@ -181,6 +182,18 @@ OCI_LAYER_MODE ?= "single" # "app:directories:/opt/myapp+/etc/myapp" # "config:files:/etc/myapp.conf+/etc/default/myapp" # +# NOTE: directories/files only copy content NOT already present in +# earlier layers (delta-only), avoiding duplication with packages layers. +# +# For host type, content is source:dest pairs (use + as delimiter): +# "certs:host:/etc/ssl/certs/my-ca.crt:/etc/ssl/certs/my-ca.crt" +# "config:host:/home/builder/config:/etc/myapp/config+/home/builder/keys:/etc/myapp/keys" +# +# WARNING: host layers copy content from the build machine that is NOT +# part of the Yocto build. This affects reproducibility - the build output +# depends on the state of the build machine. Use sparingly for deployment- +# specific config, keys, or certificates that cannot be packaged. +# # Note: Use + as delimiter because ; is interpreted as shell command separator # # Example: @@ -189,9 +202,10 @@ OCI_LAYER_MODE ?= "single" # base:packages:base-files+base-passwd+netbase+busybox \ # python:packages:python3+python3-pip \ # app:directories:/opt/myapp \ +# certs:host:/etc/ssl/certs/my-ca.crt:/etc/ssl/certs/ \ # " # -# Result: 3 layers (base, python, app) plus any base image layers +# Result: 4 layers (base, python, app, certs) plus any base image layers # OCI_LAYERS ?= "" @@ -288,6 +302,7 @@ python __anonymous() { bb.fatal("OCI_LAYER_MODE = 'multi' requires OCI_LAYERS to be defined") has_packages_layer = False + host_layer_warnings = [] # Parse and validate layer definitions for layer_def in oci_layers.split(): @@ -296,11 +311,27 @@ python __anonymous() { bb.fatal(f"Invalid OCI_LAYERS entry '{layer_def}': " "format is 'name:type:content'") layer_name, layer_type, layer_content = parts[0], parts[1], ':'.join(parts[2:]) - if layer_type not in ('packages', 'directories', 'files'): + if layer_type not in ('packages', 'directories', 'files', 'host'): bb.fatal(f"Invalid layer type '{layer_type}' in '{layer_def}': " - "must be 'packages', 'directories', or 'files'") + "must be 'packages', 'directories', 'files', or 'host'") if layer_type == 'packages': has_packages_layer = True + elif layer_type == 'host': + # Validate host layer format and collect warnings + # Format: source:dest pairs separated by + + for pair in layer_content.replace('+', ' ').split(): + if ':' not in pair: + bb.fatal(f"Invalid host layer content '{pair}' in '{layer_def}': " + "format is 'source_path:dest_path'") + src_path = pair.rsplit(':', 1)[0] + host_layer_warnings.append(f" Layer '{layer_name}': {src_path}") + + # Emit warning for host layers (content from build machine, not Yocto) + if host_layer_warnings: + bb.warn("OCI image includes content from build machine filesystem (host layers).\n" + "This content is NOT part of the Yocto build and affects reproducibility.\n" + "The build output will depend on the state of the build machine.\n" + "Host paths used:\n" + "\n".join(host_layer_warnings)) # Add package manager native dependency if using 'packages' layer type if has_packages_layer: -- cgit v1.2.3-54-g00ecf