From 08ffbeef62a426193528c539bcb02fa4b8b1c191 Mon Sep 17 00:00:00 2001 From: Bruce Ashfield Date: Wed, 14 Jan 2026 04:45:53 +0000 Subject: image-oci: add multi-layer OCI support and CMD default Add support for multi-layer OCI images, enabling base + app layer builds: Multi-layer support: - Add OCI_BASE_IMAGE variable to specify base layer (recipe name or path) - Add OCI_BASE_IMAGE_TAG for selecting base image tag (default: latest) - Resolve base image type (recipe/path/remote) at parse time - Copy base OCI layout before adding new layer via umoci repack - Fix merged-usr whiteout ordering issue for non-merged-usr base images (replaces problematic whiteouts with filtered entries to avoid Docker pull failures when layering merged-usr on traditional layout) CMD/ENTRYPOINT behavior change: - Add OCI_IMAGE_CMD variable (default: "/bin/sh") - Change OCI_IMAGE_ENTRYPOINT default to empty string - This makes `docker run image /bin/sh` work as expected (like Docker Hub images) - OCI_IMAGE_ENTRYPOINT_ARGS still works for legacy compatibility - Fix shlex.split() for proper shell quoting in CMD/ENTRYPOINT values The multi-layer feature requires umoci backend (default). The sloci backend only supports single-layer images and will error if OCI_BASE_IMAGE is set. Example usage: OCI_BASE_IMAGE = "container-base" IMAGE_INSTALL = "myapp" OCI_IMAGE_CMD = "/usr/bin/myapp" Signed-off-by: Bruce Ashfield --- classes/image-oci-umoci.inc | 305 ++++++++++++++++++++++++++++++++++++++++++-- classes/image-oci.bbclass | 105 ++++++++++++++- 2 files changed, 395 insertions(+), 15 deletions(-) (limited to 'classes') diff --git a/classes/image-oci-umoci.inc b/classes/image-oci-umoci.inc index 0dba347c..340f298b 100644 --- a/classes/image-oci-umoci.inc +++ b/classes/image-oci-umoci.inc @@ -1,3 +1,187 @@ +# Fix merged-usr whiteout issues in OCI layer +# When a directory becomes a symlink, umoci creates whiteouts inside it, but +# puts them after the symlink in the tar. Docker fails because it can't create +# files inside a symlink. This function replaces individual whiteouts with +# opaque whiteouts and fixes tar ordering. +oci_fix_merged_usr_whiteouts() { + local image_dir="$1" + local tag="$2" + local needs_fix=false + + # Find the manifest for this tag + local manifest_digest=$(jq -r '.manifests[] | select(.annotations["org.opencontainers.image.ref.name"] == "'"$tag"'") | .digest' "$image_dir/index.json" | sed 's/sha256://') + if [ -z "$manifest_digest" ]; then + bbdebug 1 "OCI fix: Could not find manifest for tag $tag" + return 0 + fi + + # Get the last layer (newest, the one we just added) + local layer_digest=$(jq -r '.layers[-1].digest' "$image_dir/blobs/sha256/$manifest_digest" | sed 's/sha256://') + if [ -z "$layer_digest" ]; then + bbdebug 1 "OCI fix: Could not find layer digest" + return 0 + fi + + local layer_blob="$image_dir/blobs/sha256/$layer_digest" + if [ ! -f "$layer_blob" ]; then + bbdebug 1 "OCI fix: Layer blob not found: $layer_blob" + return 0 + fi + + # Convert to absolute path before we cd elsewhere + layer_blob=$(readlink -f "$layer_blob") + image_dir=$(readlink -f "$image_dir") + + # Get tar listing with details to identify symlinks and whiteouts + local layer_listing=$(tar -tvzf "$layer_blob" 2>/dev/null || true) + local layer_files=$(tar -tzf "$layer_blob" 2>/dev/null || true) + + # Find directories that are symlinks but have whiteouts listed inside + # Include merged-usr dirs (bin, sbin, lib) and var/* symlinks + local dirs_to_fix="" + for dir in bin sbin lib lib64 var/lock var/log var/run var/tmp; do + # Check if $dir is a symlink in the tar (line starts with 'l') + if echo "$layer_listing" | grep -q "^l.* ${dir} -> "; then + # Check if there are whiteouts "inside" it + if echo "$layer_files" | grep -q "^${dir}/\.wh\."; then + bbnote "OCI fix: Found problematic whiteout pattern in $dir" + dirs_to_fix="$dirs_to_fix $dir" + needs_fix=true + fi + fi + done + + if [ "$needs_fix" != "true" ]; then + bbdebug 1 "OCI fix: No merged-usr whiteout issues detected" + return 0 + fi + + bbnote "OCI fix: Fixing merged-usr whiteout ordering in layer" + bbnote "OCI fix: Directories to fix:$dirs_to_fix" + + # Save current directory + local orig_dir=$(pwd) + + # Create temp directory for fix + local fix_dir=$(mktemp -d) + local fixed_tar="$fix_dir/fixed-layer.tar" + + cd "$fix_dir" + + # Strategy: Simply remove the problematic whiteouts + # The symlink itself will hide the base directory contents. + # We don't need opaque whiteouts - they would hide ALL base content. + + # Build exclude pattern - whiteouts in symlinked dirs + local exclude_pattern="" + for dir in $dirs_to_fix; do + exclude_pattern="${exclude_pattern}|^${dir}/\\.wh\\." + done + exclude_pattern="${exclude_pattern#|}" # Remove leading | + + # Use Python to filter the tar - just remove problematic whiteouts + python3 << PYEOF +import tarfile +import gzip +import re + +src_blob = "$layer_blob" +dst_tar = "$fixed_tar" +exclude_re = re.compile(r'$exclude_pattern') + +removed_count = 0 + +# Read source tar and filter out problematic whiteouts +with gzip.open(src_blob, 'rb') as gz: + with tarfile.open(fileobj=gz, mode='r:') as src: + with tarfile.open(dst_tar, 'w') as dst: + for member in src.getmembers(): + # Skip whiteouts in dirs that became symlinks + if exclude_re.match(member.name): + removed_count += 1 + continue + + # Copy the member + if member.isfile(): + dst.addfile(member, src.extractfile(member)) + else: + dst.addfile(member) + +print(f"Removed {removed_count} problematic whiteouts from layer") +PYEOF + + # Calculate diff_id (uncompressed digest) before compressing + local new_diff_id=$(sha256sum "$fixed_tar" | cut -d' ' -f1) + local old_diff_id=$(gunzip -c "$layer_blob" | sha256sum | cut -d' ' -f1) + + # Compress the fixed tar + gzip -n -f "$fixed_tar" + local fixed_blob="$fixed_tar.gz" + + # Calculate new digest (compressed) + local new_digest=$(sha256sum "$fixed_blob" | cut -d' ' -f1) + local new_size=$(stat -c%s "$fixed_blob") + + bbnote "OCI fix: New layer digest: sha256:$new_digest (was sha256:$layer_digest)" + bbnote "OCI fix: New diff_id: sha256:$new_diff_id (was sha256:$old_diff_id)" + + # Replace the blob + cp "$fixed_blob" "$image_dir/blobs/sha256/$new_digest" + rm -f "$image_dir/blobs/sha256/$layer_digest" + + # Update manifest with new layer digest and size + local manifest_file="$image_dir/blobs/sha256/$manifest_digest" + jq --arg old "sha256:$layer_digest" --arg new "sha256:$new_digest" --argjson size "$new_size" \ + '(.layers[] | select(.digest == $old)) |= (.digest = $new | .size = $size)' \ + "$manifest_file" > "$manifest_file.new" + mv "$manifest_file.new" "$manifest_file" + + # Get config digest from manifest and update diff_ids in config + local config_digest=$(jq -r '.config.digest' "$manifest_file" | sed 's/sha256://') + local config_file="$image_dir/blobs/sha256/$config_digest" + + bbnote "OCI fix: Updating config $config_digest" + + # Update the last diff_id in the config (our layer) + # Use direct index replacement since we know which layer we fixed + jq --arg new "sha256:$new_diff_id" \ + '.rootfs.diff_ids[-1] = $new' \ + "$config_file" > "$config_file.new" + mv "$config_file.new" "$config_file" + + # Recalculate config digest + local new_config_digest=$(sha256sum "$config_file" | cut -d' ' -f1) + local new_config_size=$(stat -c%s "$config_file") + + if [ "$new_config_digest" != "$config_digest" ]; then + mv "$config_file" "$image_dir/blobs/sha256/$new_config_digest" + # Update manifest with new config digest + jq --arg old "sha256:$config_digest" --arg new "sha256:$new_config_digest" --argjson size "$new_config_size" \ + '.config |= (if .digest == $old then .digest = $new | .size = $size else . end)' \ + "$manifest_file" > "$manifest_file.new" + mv "$manifest_file.new" "$manifest_file" + fi + + # Recalculate manifest digest + local new_manifest_digest=$(sha256sum "$manifest_file" | cut -d' ' -f1) + local new_manifest_size=$(stat -c%s "$manifest_file") + + if [ "$new_manifest_digest" != "$manifest_digest" ]; then + mv "$manifest_file" "$image_dir/blobs/sha256/$new_manifest_digest" + # Update index.json + jq --arg old "sha256:$manifest_digest" --arg new "sha256:$new_manifest_digest" --argjson size "$new_manifest_size" \ + '(.manifests[] | select(.digest == $old)) |= (.digest = $new | .size = $size)' \ + "$image_dir/index.json" > "$image_dir/index.json.new" + mv "$image_dir/index.json.new" "$image_dir/index.json" + fi + + # Restore original directory and cleanup + cd "$orig_dir" + rm -rf "$fix_dir" + + bbnote "OCI fix: Layer whiteout fix complete" +} + IMAGE_CMD:oci() { umoci_options="" @@ -69,22 +253,97 @@ IMAGE_CMD:oci() { OCI_IMAGE_TAG="initial-tag" fi - if [ -n "$new_image" ]; then - bbdebug 1 "OCI: umoci init --layout $image_name" - umoci init --layout $image_name - umoci new --image $image_name:${OCI_IMAGE_TAG} - umoci unpack --rootless --image $image_name:${OCI_IMAGE_TAG} $image_bundle_name + # ======================================================================== + # PHASE 1: Initialize OCI layout (from scratch or from base image) + # ======================================================================== + if [ -n "${_OCI_BASE_RECIPE}" ] || [ -n "${_OCI_BASE_PATH}" ]; then + # Using a base image + base_oci_dir="" + base_tag="${OCI_BASE_IMAGE_TAG}" + + if [ -n "${_OCI_BASE_RECIPE}" ]; then + # Use exact symlink naming: ${recipe}-${tag}-oci + base_oci_dir="${DEPLOY_DIR_IMAGE}/${_OCI_BASE_RECIPE}-${base_tag}-oci" + + if [ ! -d "$base_oci_dir" ] || [ ! -f "$base_oci_dir/index.json" ]; then + bbfatal "OCI: Base image '${_OCI_BASE_RECIPE}' not found at expected path: $base_oci_dir" + fi + elif [ -n "${_OCI_BASE_PATH}" ]; then + base_oci_dir="${_OCI_BASE_PATH}" + if [ ! -d "$base_oci_dir" ] || [ ! -f "$base_oci_dir/index.json" ]; then + bbfatal "OCI: Base image path not valid: $base_oci_dir" + fi + fi + + # Resolve symlinks to get actual directory + base_oci_dir=$(readlink -f "$base_oci_dir") + bbnote "OCI: Using base image from: $base_oci_dir" + + # Copy base image layout to our image directory (-L to follow symlinks) + cp -rL "$base_oci_dir" "$image_name" + + # Count existing layers for logging (simplified) + base_layers=$(ls "$image_name/blobs/sha256/" 2>/dev/null | wc -l) + bbnote "OCI: Base image has approximately $base_layers blob(s)" + + # Unpack base image for modification + umoci unpack --rootless --image "$image_name:$base_tag" "$image_bundle_name" + elif [ -n "$new_image" ]; then + # No base image - create empty OCI layout + bbdebug 1 "OCI: umoci init --layout $image_name" + umoci init --layout $image_name + umoci new --image $image_name:${OCI_IMAGE_TAG} + umoci unpack --rootless --image $image_name:${OCI_IMAGE_TAG} $image_bundle_name else - # todo: create a different tag, after checking if the passed one exists - true + # todo: create a different tag, after checking if the passed one exists + true fi + # ======================================================================== + # PHASE 2: Add content layer(s) + # ======================================================================== bbdebug 1 "OCI: populating rootfs" - bbdebug 1 "OCI: cp -r ${IMAGE_ROOTFS}/* $image_bundle_name/rootfs/" - cp -r -a --no-preserve=ownership ${IMAGE_ROOTFS}/* $image_bundle_name/rootfs - bbdebug 1 "OCI: umoci repack --image $image_name:${OCI_IMAGE_TAG} $image_bundle_name" - umoci repack --image $image_name:${OCI_IMAGE_TAG} $image_bundle_name + # Use rsync for robust merging when base image exists (handles symlink vs dir conflicts) + # For no-base builds, cp is sufficient and faster + # Note: When source has symlinks replacing dest directories, we first remove conflicting dirs + if [ -n "${_OCI_BASE_RECIPE}" ] || [ -n "${_OCI_BASE_PATH}" ]; then + # Handle Yocto's merged-usr symlinks (/bin -> /usr/bin) and /var symlinks + # replacing Alpine's or other base image directories + for p in bin lib lib64 sbin var/lock var/log var/tmp; do + src="${IMAGE_ROOTFS}/$p" + dst="$image_bundle_name/rootfs/$p" + if [ -L "$src" ] && [ -d "$dst" ] && [ ! -L "$dst" ]; then + bbdebug 1 "OCI: removing directory $dst to replace with symlink" + rm -rf "$dst" + fi + done + bbdebug 1 "OCI: rsync -a --no-owner --no-group ${IMAGE_ROOTFS}/ $image_bundle_name/rootfs/" + rsync -a --no-owner --no-group ${IMAGE_ROOTFS}/ $image_bundle_name/rootfs/ + else + bbdebug 1 "OCI: cp -r ${IMAGE_ROOTFS}/* $image_bundle_name/rootfs/" + cp -r -a --no-preserve=ownership ${IMAGE_ROOTFS}/* $image_bundle_name/rootfs + fi + + # Determine which tag to use for repack + repack_tag="${OCI_IMAGE_TAG}" + if [ -n "${_OCI_BASE_RECIPE}" ] || [ -n "${_OCI_BASE_PATH}" ]; then + repack_tag="${OCI_BASE_IMAGE_TAG}" + fi + + bbdebug 1 "OCI: umoci repack --image $image_name:$repack_tag $image_bundle_name" + umoci repack --image $image_name:$repack_tag $image_bundle_name + + # If we used a base image with different tag, re-tag to our target tag + if [ -n "${_OCI_BASE_RECIPE}" ] || [ -n "${_OCI_BASE_PATH}" ]; then + if [ "$repack_tag" != "${OCI_IMAGE_TAG}" ]; then + umoci tag --image "$image_name:$repack_tag" "${OCI_IMAGE_TAG}" + fi + + # Log final layer count (simplified - count blobs minus config/manifest) + final_blobs=$(ls "$image_name/blobs/sha256/" 2>/dev/null | wc -l) + bbnote "OCI: Final image has approximately $final_blobs blob(s)" + fi bbdebug 1 "OCI: configuring image" if [ -n "${OCI_IMAGE_LABELS}" ]; then @@ -137,13 +396,31 @@ IMAGE_CMD:oci() { bbnote "OCI: image subarch is set to: ${OCI_IMAGE_SUBARCH}, but umoci does not" bbnote " expose variants. use sloci instead if this is important" fi - umoci config --image $image_name:${OCI_IMAGE_TAG} \ - ${@" ".join("--config.entrypoint %s" % s for s in d.getVar("OCI_IMAGE_ENTRYPOINT").split())} + # Set entrypoint if specified (for wrapper script patterns) + if [ -n "${OCI_IMAGE_ENTRYPOINT}" ]; then + umoci config --image $image_name:${OCI_IMAGE_TAG} \ + ${@" ".join("--config.entrypoint '%s'" % s for s in __import__('shlex').split(d.getVar("OCI_IMAGE_ENTRYPOINT")))} + fi + # Set CMD: use OCI_IMAGE_ENTRYPOINT_ARGS if set (legacy), otherwise OCI_IMAGE_CMD if [ -n "${OCI_IMAGE_ENTRYPOINT_ARGS}" ]; then - umoci config --image $image_name:${OCI_IMAGE_TAG} ${@" ".join("--config.cmd %s" % s for s in d.getVar("OCI_IMAGE_ENTRYPOINT_ARGS").split())} + umoci config --image $image_name:${OCI_IMAGE_TAG} ${@" ".join("--config.cmd '%s'" % s for s in __import__('shlex').split(d.getVar("OCI_IMAGE_ENTRYPOINT_ARGS")))} + elif [ -n "${OCI_IMAGE_CMD}" ]; then + umoci config --image $image_name:${OCI_IMAGE_TAG} ${@" ".join("--config.cmd '%s'" % s for s in __import__('shlex').split(d.getVar("OCI_IMAGE_CMD")))} fi umoci config --image $image_name:${OCI_IMAGE_TAG} --author ${OCI_IMAGE_AUTHOR_EMAIL} + # ======================================================================== + # PHASE 3: Fix merged-usr whiteout issues for non-merged-usr base images + # ======================================================================== + # When layering merged-usr (symlinks) on traditional layout (directories), + # umoci creates whiteouts like bin/.wh.file but puts them AFTER the bin symlink + # in the tar. Docker can't create files inside a symlink, causing pull failures. + # Fix: Replace individual whiteouts with opaque whiteouts, reorder tar entries. + # NOTE: Must run AFTER all umoci config commands since they create new config blobs. + if [ -n "${_OCI_BASE_RECIPE}" ] || [ -n "${_OCI_BASE_PATH}" ]; then + oci_fix_merged_usr_whiteouts "$image_name" "${OCI_IMAGE_TAG}" + fi + # OCI_IMAGE_TAG may contain ":", but these are not allowed in OCI file # names so replace them image_tag="${@d.getVar("OCI_IMAGE_TAG").replace(":", "_")}" diff --git a/classes/image-oci.bbclass b/classes/image-oci.bbclass index 70f32bf1..6f8011ca 100644 --- a/classes/image-oci.bbclass +++ b/classes/image-oci.bbclass @@ -42,6 +42,8 @@ IMAGE_TYPEDEP:oci = "container tar.bz2" # OCI_IMAGE_BACKEND ?= "sloci-image" OCI_IMAGE_BACKEND ?= "umoci" do_image_oci[depends] += "${OCI_IMAGE_BACKEND}-native:do_populate_sysroot" +# jq-native is needed for the merged-usr whiteout fix +do_image_oci[depends] += "jq-native:do_populate_sysroot" # # image type configuration block @@ -55,8 +57,12 @@ OCI_IMAGE_RUNTIME_UID ?= "" OCI_IMAGE_ARCH ?= "${@oe.go.map_arch(d.getVar('TARGET_ARCH'))}" OCI_IMAGE_SUBARCH ?= "${@oci_map_subarch(d.getVar('TARGET_ARCH'), d.getVar('TUNE_FEATURES'), d)}" -OCI_IMAGE_ENTRYPOINT ?= "sh" +# OCI_IMAGE_ENTRYPOINT: If set, this command always runs (args appended). +# OCI_IMAGE_CMD: Default command (replaced when user passes arguments). +# Most base images use CMD only for flexibility. Use ENTRYPOINT for wrapper scripts. +OCI_IMAGE_ENTRYPOINT ?= "" OCI_IMAGE_ENTRYPOINT_ARGS ?= "" +OCI_IMAGE_CMD ?= "/bin/sh" OCI_IMAGE_WORKINGDIR ?= "" OCI_IMAGE_STOPSIGNAL ?= "" @@ -112,6 +118,27 @@ OCI_IMAGE_BUILD_DATE ?= "" # Enable/disable auto-detection of git metadata (set to "0" to disable) OCI_IMAGE_AUTO_LABELS ?= "1" +# ============================================================================= +# Multi-Layer OCI Support +# ============================================================================= +# +# OCI_BASE_IMAGE: Base image to build on top of +# - Recipe name: "container-base" (uses local recipe's OCI output) +# - Path: "/path/to/oci-dir" (uses existing OCI layout) +# - Registry URL: "docker.io/library/alpine:3.19" (fetches external image) +# +# OCI_LAYER_MODE: How to create layers +# - "single" (default): Single layer with complete rootfs (backward compatible) +# - "multi": Multiple layers from OCI_LAYERS definitions +# +# When OCI_BASE_IMAGE is set: +# - Base image layers are preserved +# - New content from IMAGE_ROOTFS is added as additional layer(s) +# +OCI_BASE_IMAGE ?= "" +OCI_BASE_IMAGE_TAG ?= "latest" +OCI_LAYER_MODE ?= "single" + # whether the oci image dir should be left as a directory, or # bundled into a tarball. OCI_IMAGE_TAR_OUTPUT ?= "true" @@ -131,6 +158,82 @@ def oci_map_subarch(a, f, d): return '' return '' +# ============================================================================= +# Base Image Resolution and Dependency Setup +# ============================================================================= + +def oci_resolve_base_image(d): + """Resolve OCI_BASE_IMAGE to determine its type. + + Returns dict with 'type' key: + - {'type': 'recipe', 'name': 'container-base'} + - {'type': 'path', 'path': '/path/to/oci-dir'} + - {'type': 'remote', 'url': 'docker.io/library/alpine:3.19'} + - None if no base image + """ + base = d.getVar('OCI_BASE_IMAGE') or '' + if not base: + return None + + # Check if it's a path (starts with /) + if base.startswith('/'): + return {'type': 'path', 'path': base} + + # Check if it looks like a registry URL (contains / or has registry prefix) + if '/' in base or '.' in base.split(':')[0]: + return {'type': 'remote', 'url': base} + + # Assume it's a recipe name + return {'type': 'recipe', 'name': base} + +python __anonymous() { + import os + + backend = d.getVar('OCI_IMAGE_BACKEND') or 'umoci' + base_image = d.getVar('OCI_BASE_IMAGE') or '' + layer_mode = d.getVar('OCI_LAYER_MODE') or 'single' + + # sloci doesn't support multi-layer + if backend == 'sloci-image': + if layer_mode != 'single' or base_image: + bb.fatal("Multi-layer OCI requires umoci backend. " + "Set OCI_IMAGE_BACKEND = 'umoci' or remove OCI_BASE_IMAGE") + + # Resolve base image and set up dependencies + if base_image: + resolved = oci_resolve_base_image(d) + if resolved: + if resolved['type'] == 'recipe': + # Add dependency on base recipe's OCI output + # Use do_build as it works for both image recipes and oci-fetch recipes + base_recipe = resolved['name'] + d.setVar('_OCI_BASE_RECIPE', base_recipe) + d.appendVarFlag('do_image_oci', 'depends', + f" {base_recipe}:do_build rsync-native:do_populate_sysroot") + bb.debug(1, f"OCI: Using base image from recipe: {base_recipe}") + + elif resolved['type'] == 'path': + d.setVar('_OCI_BASE_PATH', resolved['path']) + d.appendVarFlag('do_image_oci', 'depends', + " rsync-native:do_populate_sysroot") + bb.debug(1, f"OCI: Using base image from path: {resolved['path']}") + + elif resolved['type'] == 'remote': + # Remote URLs are not supported directly - use a container-bundle recipe + remote_url = resolved['url'] + # Create sanitized key for CONTAINER_DIGESTS varflag + sanitized_key = remote_url.replace('/', '_').replace(':', '_') + bb.fatal(f"Remote base images cannot be used directly: {remote_url}\n\n" + f"Create a container-bundle recipe to fetch the external image:\n\n" + f" # recipes-containers/oci-base-images/my-base.bb\n" + f" inherit container-bundle\n" + f" CONTAINER_BUNDLES = \"{remote_url}\"\n" + f" CONTAINER_DIGESTS[{sanitized_key}] = \"sha256:...\"\n" + f" CONTAINER_BUNDLE_DEPLOY = \"1\"\n\n" + f"Get digest with: skopeo inspect docker://{remote_url} | jq -r '.Digest'\n\n" + f"Then use: OCI_BASE_IMAGE = \"my-base\"") +} + # the IMAGE_CMD:oci comes from the .inc OCI_IMAGE_BACKEND_INC ?= "${@"image-oci-" + "${OCI_IMAGE_BACKEND}" + ".inc"}" include ${OCI_IMAGE_BACKEND_INC} -- cgit v1.2.3-54-g00ecf