summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBruce Ashfield <bruce.ashfield@gmail.com>2026-01-14 20:59:14 +0000
committerBruce Ashfield <bruce.ashfield@gmail.com>2026-01-21 18:00:26 -0500
commit4d8fc28985dbd69ca5b0b2cfe3e977d74fe5b3dd (patch)
tree6da2330220ef822bb59231ee01e5d87763cf85c6
parent24c604854c6ffe79ac7973e333b2df7f7f82ddd9 (diff)
downloadmeta-virtualization-4d8fc28985dbd69ca5b0b2cfe3e977d74fe5b3dd.tar.gz
image-oci: add layer caching for multi-layer OCI builds
Add layer caching to speed up multi-layer OCI image rebuilds. When enabled, pre-installed package layers are cached to disk and restored on subsequent builds, avoiding repeated package installation. New variables: - OCI_LAYER_CACHE: Enable/disable caching (default "1") - OCI_LAYER_CACHE_DIR: Cache location (default ${TOPDIR}/oci-layer-cache/${MACHINE}) Cache key is computed from: - Layer name and type - Sorted package list - Package versions from PKGDATA_DIR - MACHINE and TUNE_PKGARCH Cache automatically invalidates when: - Package versions change - Layer definition changes - Architecture changes Benefits: - First build: ~10-30s per layer (cache miss, packages installed) - Subsequent builds: ~1s per layer (cache hit, files copied) - Shared across recipes with identical layer definitions Build log shows cache status: NOTE: OCI Cache HIT: Layer 'base' (be88c180f651416b) NOTE: OCI: Pre-installed packages for 3 layers (cache: 3 hits, 0 misses) Also adds comprehensive pytest suite for multi-layer OCI functionality including tests for 1/2/3 layer modes and cache behavior. Signed-off-by: Bruce Ashfield <bruce.ashfield@gmail.com>
-rw-r--r--classes/image-oci-umoci.inc177
-rw-r--r--classes/image-oci.bbclass26
-rw-r--r--docs/container-bundling.md64
-rw-r--r--tests/test_multilayer_oci.py466
4 files changed, 731 insertions, 2 deletions
diff --git a/classes/image-oci-umoci.inc b/classes/image-oci-umoci.inc
index 1ff36718..fbb77cd0 100644
--- a/classes/image-oci-umoci.inc
+++ b/classes/image-oci-umoci.inc
@@ -5,12 +5,152 @@
5# directories using Yocto's package manager classes. The shell code then copies 5# directories using Yocto's package manager classes. The shell code then copies
6# from these pre-installed directories. 6# from these pre-installed directories.
7 7
8def oci_compute_layer_cache_key(d, layer_name, layer_type, layer_packages):
9 """
10 Compute a cache key for a layer based on its definition and package versions.
11
12 The cache key is a SHA256 hash of:
13 - Layer name and type
14 - Sorted package list
15 - Package versions from PKGDATA_DIR
16 - Machine and architecture info
17
18 Returns: (cache_key, cache_info) tuple where cache_info is human-readable
19 """
20 import hashlib
21 import os
22 import json
23
24 packages = sorted(layer_packages.split())
25 pkgdata_dir = d.getVar('PKGDATA_DIR')
26 machine = d.getVar('MACHINE')
27 tune_pkgarch = d.getVar('TUNE_PKGARCH')
28
29 # Build cache key components
30 cache_components = {
31 'layer_name': layer_name,
32 'layer_type': layer_type,
33 'packages': packages,
34 'machine': machine,
35 'tune_pkgarch': tune_pkgarch,
36 'pkg_versions': {}
37 }
38
39 # Get package versions from pkgdata
40 for pkg in packages:
41 pkg_info_file = os.path.join(pkgdata_dir, 'runtime', pkg)
42 if os.path.exists(pkg_info_file):
43 try:
44 with open(pkg_info_file, 'r') as f:
45 for line in f:
46 if line.startswith('PKGV:'):
47 cache_components['pkg_versions'][pkg] = line.split(':', 1)[1].strip()
48 break
49 elif line.startswith(f'PKGV_{pkg}:'):
50 cache_components['pkg_versions'][pkg] = line.split(':', 1)[1].strip()
51 break
52 except Exception:
53 pass
54
55 # Create deterministic JSON for hashing
56 cache_json = json.dumps(cache_components, sort_keys=True)
57 cache_key = hashlib.sha256(cache_json.encode()).hexdigest()[:16]
58
59 # Human-readable info for logging
60 pkg_vers = [f"{p}={cache_components['pkg_versions'].get(p, '?')}" for p in packages[:3]]
61 if len(packages) > 3:
62 pkg_vers.append(f"...+{len(packages)-3} more")
63 cache_info = f"{layer_name}:{' '.join(pkg_vers)}"
64
65 return cache_key, cache_info
66
67
68def oci_check_layer_cache(d, cache_key, layer_name):
69 """
70 Check if a cached layer exists.
71
72 Returns: path to cached layer rootfs if found, None otherwise
73 """
74 import os
75
76 cache_dir = d.getVar('OCI_LAYER_CACHE_DIR')
77 if not cache_dir:
78 return None
79
80 cached_path = os.path.join(cache_dir, f'{cache_key}-{layer_name}')
81 marker_file = os.path.join(cached_path, '.oci-layer-cache')
82
83 if os.path.isdir(cached_path) and os.path.exists(marker_file):
84 return cached_path
85
86 return None
87
88
89def oci_cache_layer(d, cache_key, layer_name, layer_rootfs):
90 """
91 Save a layer rootfs to the cache.
92 """
93 import os
94 import shutil
95 import time
96
97 cache_dir = d.getVar('OCI_LAYER_CACHE_DIR')
98 if not cache_dir:
99 return
100
101 bb.utils.mkdirhier(cache_dir)
102
103 cached_path = os.path.join(cache_dir, f'{cache_key}-{layer_name}')
104
105 # Remove any existing cache for this key
106 if os.path.exists(cached_path):
107 shutil.rmtree(cached_path)
108
109 # Copy layer rootfs to cache
110 bb.note(f"OCI Cache: Saving layer '{layer_name}' to cache ({cache_key})")
111 shutil.copytree(layer_rootfs, cached_path, symlinks=True)
112
113 # Write cache marker with metadata
114 marker_file = os.path.join(cached_path, '.oci-layer-cache')
115 with open(marker_file, 'w') as f:
116 f.write(f"cache_key={cache_key}\n")
117 f.write(f"layer_name={layer_name}\n")
118 f.write(f"created={time.strftime('%Y-%m-%dT%H:%M:%SZ')}\n")
119 f.write(f"machine={d.getVar('MACHINE')}\n")
120
121
122def oci_restore_layer_from_cache(d, cached_path, layer_rootfs):
123 """
124 Restore a layer rootfs from the cache.
125 """
126 import os
127 import shutil
128
129 # Ensure target directory exists and is empty
130 if os.path.exists(layer_rootfs):
131 shutil.rmtree(layer_rootfs)
132 bb.utils.mkdirhier(layer_rootfs)
133
134 # Copy cached content (excluding the cache marker)
135 for item in os.listdir(cached_path):
136 if item == '.oci-layer-cache':
137 continue
138 src = os.path.join(cached_path, item)
139 dst = os.path.join(layer_rootfs, item)
140 if os.path.isdir(src):
141 shutil.copytree(src, dst, symlinks=True)
142 else:
143 shutil.copy2(src, dst)
144
145
8python oci_multilayer_install_packages() { 146python oci_multilayer_install_packages() {
9 """ 147 """
10 Pre-install packages for each packages layer in OCI_LAYERS. 148 Pre-install packages for each packages layer in OCI_LAYERS.
11 149
12 Creates temp rootfs directories with packages installed using Yocto's PM. 150 Creates temp rootfs directories with packages installed using Yocto's PM.
13 The shell IMAGE_CMD:oci then copies from these directories. 151 The shell IMAGE_CMD:oci then copies from these directories.
152
153 Supports layer caching when OCI_LAYER_CACHE = "1" to speed up rebuilds.
14 """ 154 """
15 import os 155 import os
16 import shutil 156 import shutil
@@ -32,7 +172,15 @@ python oci_multilayer_install_packages() {
32 shutil.rmtree(layer_rootfs_base) 172 shutil.rmtree(layer_rootfs_base)
33 bb.utils.mkdirhier(layer_rootfs_base) 173 bb.utils.mkdirhier(layer_rootfs_base)
34 174
35 bb.note("OCI: Pre-installing packages for multi-layer mode") 175 # Check if caching is enabled
176 cache_enabled = d.getVar('OCI_LAYER_CACHE') == '1'
177 if cache_enabled:
178 bb.note("OCI: Pre-installing packages for multi-layer mode (caching enabled)")
179 else:
180 bb.note("OCI: Pre-installing packages for multi-layer mode (caching disabled)")
181
182 cache_hits = 0
183 cache_misses = 0
36 184
37 # Parse OCI_LAYERS and install packages for each packages layer 185 # Parse OCI_LAYERS and install packages for each packages layer
38 layer_num = 0 186 layer_num = 0
@@ -48,17 +196,42 @@ python oci_multilayer_install_packages() {
48 layer_num += 1 196 layer_num += 1
49 layer_rootfs = os.path.join(layer_rootfs_base, f'layer-{layer_num}-{layer_name}') 197 layer_rootfs = os.path.join(layer_rootfs_base, f'layer-{layer_num}-{layer_name}')
50 198
199 # Check cache if enabled
200 if cache_enabled:
201 cache_key, cache_info = oci_compute_layer_cache_key(d, layer_name, layer_type, layer_content)
202 cached_path = oci_check_layer_cache(d, cache_key, layer_name)
203
204 if cached_path:
205 bb.note(f"OCI Cache HIT: Layer '{layer_name}' ({cache_key})")
206 oci_restore_layer_from_cache(d, cached_path, layer_rootfs)
207 cache_hits += 1
208 # Store the path for the shell code to use
209 d.setVar(f'OCI_LAYER_{layer_num}_ROOTFS', layer_rootfs)
210 d.setVar(f'OCI_LAYER_{layer_num}_NAME', layer_name)
211 continue
212 else:
213 bb.note(f"OCI Cache MISS: Layer '{layer_name}' ({cache_info})")
214 cache_misses += 1
215
51 bb.note(f"OCI: Pre-installing layer {layer_num} '{layer_name}' to {layer_rootfs}") 216 bb.note(f"OCI: Pre-installing layer {layer_num} '{layer_name}' to {layer_rootfs}")
52 217
53 # Call the package installation function 218 # Call the package installation function
54 oci_install_layer_packages(d, layer_rootfs, layer_content, layer_name) 219 oci_install_layer_packages(d, layer_rootfs, layer_content, layer_name)
55 220
221 # Cache the installed layer if caching is enabled
222 if cache_enabled:
223 oci_cache_layer(d, cache_key, layer_name, layer_rootfs)
224
56 # Store the path for the shell code to use 225 # Store the path for the shell code to use
57 d.setVar(f'OCI_LAYER_{layer_num}_ROOTFS', layer_rootfs) 226 d.setVar(f'OCI_LAYER_{layer_num}_ROOTFS', layer_rootfs)
58 d.setVar(f'OCI_LAYER_{layer_num}_NAME', layer_name) 227 d.setVar(f'OCI_LAYER_{layer_num}_NAME', layer_name)
59 228
60 d.setVar('OCI_LAYER_COUNT', str(layer_num)) 229 d.setVar('OCI_LAYER_COUNT', str(layer_num))
61 bb.note(f"OCI: Pre-installed packages for {layer_num} layers") 230
231 if cache_enabled:
232 bb.note(f"OCI: Pre-installed packages for {layer_num} layers (cache: {cache_hits} hits, {cache_misses} misses)")
233 else:
234 bb.note(f"OCI: Pre-installed packages for {layer_num} layers")
62} 235}
63 236
64# Run the Python function before IMAGE_CMD:oci 237# Run the Python function before IMAGE_CMD:oci
diff --git a/classes/image-oci.bbclass b/classes/image-oci.bbclass
index 64b17d97..41f7b3d1 100644
--- a/classes/image-oci.bbclass
+++ b/classes/image-oci.bbclass
@@ -195,6 +195,32 @@ OCI_LAYER_MODE ?= "single"
195# 195#
196OCI_LAYERS ?= "" 196OCI_LAYERS ?= ""
197 197
198# =============================================================================
199# Layer Caching (for multi-layer mode)
200# =============================================================================
201#
202# OCI_LAYER_CACHE: Enable/disable layer caching ("1" or "0")
203# When enabled, pre-installed package layers are cached to avoid
204# reinstalling packages on subsequent builds.
205#
206# OCI_LAYER_CACHE_DIR: Directory for storing cached layers
207# Default: ${TOPDIR}/oci-layer-cache/${MACHINE}
208# Cache is keyed by: layer definition + package versions + architecture
209#
210# Cache key components:
211# - Layer name and type
212# - Sorted package list
213# - Package versions (from PKGDATA_DIR)
214# - MACHINE and TUNE_PKGARCH
215#
216# Cache invalidation:
217# - Any package version change invalidates layers containing that package
218# - Layer definition changes invalidate that specific layer
219# - MACHINE/arch changes use separate cache directories
220#
221OCI_LAYER_CACHE ?= "1"
222OCI_LAYER_CACHE_DIR ?= "${TOPDIR}/oci-layer-cache/${MACHINE}"
223
198# whether the oci image dir should be left as a directory, or 224# whether the oci image dir should be left as a directory, or
199# bundled into a tarball. 225# bundled into a tarball.
200OCI_IMAGE_TAR_OUTPUT ?= "true" 226OCI_IMAGE_TAR_OUTPUT ?= "true"
diff --git a/docs/container-bundling.md b/docs/container-bundling.md
index a695a5b8..745622b5 100644
--- a/docs/container-bundling.md
+++ b/docs/container-bundling.md
@@ -193,6 +193,70 @@ OCI_LAYERS = "\
193" 193"
194``` 194```
195 195
196### Layer Caching
197
198Multi-layer builds cache pre-installed package layers for faster rebuilds.
199Installing packages requires configuring the package manager, resolving
200dependencies, and running post-install scripts - this is slow. Caching
201saves the fully-installed layer rootfs after the first build so subsequent
202builds can skip package installation entirely.
203
204#### Configuration
205
206 # Enabled by default
207 OCI_LAYER_CACHE ?= "1"
208 OCI_LAYER_CACHE_DIR ?= "${TOPDIR}/oci-layer-cache/${MACHINE}"
209
210#### Cache Key Components
211
212The cache key is a SHA256 hash of:
213
214| Component | Why It Matters |
215|-----------|----------------|
216| Layer name | Different layers cached separately |
217| Layer type | `packages` vs `directories` vs `files` |
218| Package list (sorted) | Adding/removing packages invalidates cache |
219| Package versions | Upgrading a package invalidates cache |
220| MACHINE, TUNE_PKGARCH | Architecture-specific packages |
221
222#### Advantages
223
224**Faster rebuilds**: Subsequent builds restore cached layers in ~1 second
225instead of ~10-30 seconds per layer for package installation.
226
227**Efficient development**: When only your app layer changes, base and
228dependency layers are restored from cache:
229
230 OCI_LAYERS = "\
231 base:packages:base-files+busybox \ # Cached - stable
232 deps:packages:python3+python3-pip \ # Cached - stable
233 app:packages:myapp \ # Rebuilt - changes often
234 "
235
236**Automatic invalidation**: Cache invalidates when packages change version,
237layers are modified, or architecture changes. No manual clearing needed.
238
239**Shared across recipes**: Cache stored in `${TOPDIR}/oci-layer-cache/` so
240recipes with identical layers share the same cached content.
241
242#### Build Log Example
243
244 # First build - cache misses
245 NOTE: OCI Cache MISS: Layer 'base' (base:base-files=3.0.14 ...)
246 NOTE: OCI Cache: Saving layer 'base' to cache (be88c180f651416b)
247 NOTE: OCI: Pre-installed packages for 3 layers (cache: 0 hits, 3 misses)
248
249 # Second build - cache hits
250 NOTE: OCI Cache HIT: Layer 'base' (be88c180f651416b)
251 NOTE: OCI: Pre-installed packages for 3 layers (cache: 3 hits, 0 misses)
252
253#### When to Disable
254
255Disable caching with `OCI_LAYER_CACHE = "0"` if you:
256- Suspect cache corruption
257- Need fully reproducible builds with no local state
258- Are debugging package installation issues
259
196### OCI_IMAGE_CMD vs OCI_IMAGE_ENTRYPOINT 260### OCI_IMAGE_CMD vs OCI_IMAGE_ENTRYPOINT
197 261
198 # CMD (default) - replaced when user passes arguments 262 # CMD (default) - replaced when user passes arguments
diff --git a/tests/test_multilayer_oci.py b/tests/test_multilayer_oci.py
new file mode 100644
index 00000000..eac3d23e
--- /dev/null
+++ b/tests/test_multilayer_oci.py
@@ -0,0 +1,466 @@
1# SPDX-FileCopyrightText: Copyright (C) 2025 Bruce Ashfield
2#
3# SPDX-License-Identifier: MIT
4"""
5Tests for multi-layer OCI container image support.
6
7These tests verify that OCI_LAYER_MODE = "multi" creates proper multi-layer
8OCI images and that layer caching works correctly.
9
10Run with:
11 pytest tests/test_multilayer_oci.py -v --poky-dir /opt/bruce/poky
12
13Environment variables:
14 POKY_DIR: Path to poky directory (default: /opt/bruce/poky)
15 BUILD_DIR: Path to build directory (default: $POKY_DIR/build)
16 MACHINE: Target machine (default: qemux86-64)
17
18Note: These tests require a configured Yocto build environment.
19"""
20
21import os
22import json
23import subprocess
24import shutil
25import pytest
26from pathlib import Path
27
28
29# Note: Command line options are defined in conftest.py
30
31
32@pytest.fixture(scope="module")
33def poky_dir(request):
34 """Path to poky directory."""
35 path = Path(request.config.getoption("--poky-dir"))
36 if not path.exists():
37 pytest.skip(f"Poky directory not found: {path}")
38 return path
39
40
41@pytest.fixture(scope="module")
42def build_dir(request, poky_dir):
43 """Path to build directory."""
44 path = request.config.getoption("--build-dir")
45 if path:
46 path = Path(path)
47 else:
48 path = poky_dir / "build"
49
50 if not path.exists():
51 pytest.skip(f"Build directory not found: {path}")
52 return path
53
54
55@pytest.fixture(scope="module")
56def machine(request):
57 """Target machine."""
58 return request.config.getoption("--machine")
59
60
61@pytest.fixture(scope="module")
62def deploy_dir(build_dir, machine):
63 """Path to deploy directory for the machine."""
64 path = build_dir / "tmp" / "deploy" / "images" / machine
65 if not path.exists():
66 pytest.skip(f"Deploy directory not found: {path}")
67 return path
68
69
70@pytest.fixture(scope="module")
71def meta_virt_dir(poky_dir):
72 """Path to meta-virtualization layer."""
73 path = poky_dir / "meta-virtualization"
74 if not path.exists():
75 pytest.skip(f"meta-virtualization not found: {path}")
76 return path
77
78
79@pytest.fixture(scope="module")
80def layer_cache_dir(build_dir, machine):
81 """Path to OCI layer cache directory."""
82 return build_dir / "oci-layer-cache" / machine
83
84
85def run_bitbake(build_dir, recipe, task=None, extra_args=None, timeout=1800):
86 """Run a bitbake command within the Yocto environment."""
87 # Build the bitbake command
88 bb_cmd = "bitbake"
89 if task:
90 bb_cmd += f" -c {task}"
91 bb_cmd += f" {recipe}"
92 if extra_args:
93 bb_cmd += " " + " ".join(extra_args)
94
95 # Source oe-init-build-env and run bitbake
96 poky_dir = build_dir.parent
97 full_cmd = f"bash -c 'cd {poky_dir} && source oe-init-build-env {build_dir} >/dev/null 2>&1 && {bb_cmd}'"
98
99 result = subprocess.run(
100 full_cmd,
101 shell=True,
102 cwd=build_dir,
103 timeout=timeout,
104 capture_output=True,
105 text=True,
106 )
107 return result
108
109
110def get_oci_layer_count(oci_dir):
111 """Get the number of layers in an OCI image using skopeo."""
112 result = subprocess.run(
113 ["skopeo", "inspect", f"oci:{oci_dir}"],
114 capture_output=True,
115 text=True,
116 timeout=30,
117 )
118 if result.returncode != 0:
119 return None
120
121 try:
122 data = json.loads(result.stdout)
123 return len(data.get("Layers", []))
124 except json.JSONDecodeError:
125 return None
126
127
128def get_task_log(build_dir, machine, recipe, task):
129 """Get the path to a bitbake task log."""
130 work_dir = build_dir / "tmp" / "work"
131
132 # Find the work directory for the recipe
133 for arch_dir in work_dir.glob(f"*{machine}*"):
134 recipe_dir = arch_dir / recipe
135 if recipe_dir.exists():
136 # Find the latest version directory
137 for version_dir in sorted(recipe_dir.iterdir(), reverse=True):
138 log_dir = version_dir / "temp"
139 logs = list(log_dir.glob(f"log.{task}.*"))
140 if logs:
141 return max(logs, key=lambda p: p.stat().st_mtime)
142 return None
143
144
145class TestMultiLayerOCIClass:
146 """Test OCI multi-layer bbclass functionality."""
147
148 def test_bbclass_exists(self, meta_virt_dir):
149 """Test that the image-oci.bbclass file exists."""
150 class_file = meta_virt_dir / "classes" / "image-oci.bbclass"
151 assert class_file.exists(), f"Class file not found: {class_file}"
152
153 def test_umoci_inc_exists(self, meta_virt_dir):
154 """Test that the image-oci-umoci.inc file exists."""
155 inc_file = meta_virt_dir / "classes" / "image-oci-umoci.inc"
156 assert inc_file.exists(), f"Include file not found: {inc_file}"
157
158 def test_multilayer_recipe_exists(self, meta_virt_dir):
159 """Test that the multi-layer demo recipe exists."""
160 recipe = meta_virt_dir / "recipes-demo" / "images" / "app-container-multilayer.bb"
161 assert recipe.exists(), f"Recipe not found: {recipe}"
162
163 def test_cache_variables_defined(self, meta_virt_dir):
164 """Test that layer caching variables are defined in bbclass."""
165 class_file = meta_virt_dir / "classes" / "image-oci.bbclass"
166 content = class_file.read_text()
167
168 assert "OCI_LAYER_CACHE" in content, "OCI_LAYER_CACHE not defined"
169 assert "OCI_LAYER_CACHE_DIR" in content, "OCI_LAYER_CACHE_DIR not defined"
170
171 def test_layer_mode_variables_defined(self, meta_virt_dir):
172 """Test that layer mode variables are defined in bbclass."""
173 class_file = meta_virt_dir / "classes" / "image-oci.bbclass"
174 content = class_file.read_text()
175
176 assert "OCI_LAYER_MODE" in content, "OCI_LAYER_MODE not defined"
177 assert "OCI_LAYERS" in content, "OCI_LAYERS not defined"
178
179
180class TestMultiLayerOCIBuild:
181 """Test building multi-layer OCI images."""
182
183 @pytest.mark.slow
184 def test_multilayer_recipe_builds(self, build_dir):
185 """Test that app-container-multilayer recipe builds successfully."""
186 result = run_bitbake(build_dir, "app-container-multilayer", timeout=3600)
187
188 if result.returncode != 0:
189 if "Nothing PROVIDES" in result.stderr:
190 pytest.skip("app-container-multilayer recipe not available")
191 pytest.fail(f"Build failed:\nstdout: {result.stdout}\nstderr: {result.stderr}")
192
193 @pytest.mark.slow
194 def test_multilayer_produces_correct_layers(self, build_dir, deploy_dir):
195 """Test that multi-layer build produces 3 layers."""
196 # Ensure the recipe is built
197 result = run_bitbake(build_dir, "app-container-multilayer", timeout=3600)
198 if result.returncode != 0:
199 pytest.skip("Build failed, skipping layer count check")
200
201 # Find the OCI directory
202 oci_dirs = list(deploy_dir.glob("app-container-multilayer*-oci"))
203 assert len(oci_dirs) > 0, "No OCI directory found for app-container-multilayer"
204
205 # Get the actual OCI directory (resolve symlink if needed)
206 oci_dir = oci_dirs[0]
207 if oci_dir.is_symlink():
208 oci_dir = oci_dir.resolve()
209
210 # Check layer count
211 layer_count = get_oci_layer_count(oci_dir)
212 assert layer_count is not None, f"Failed to inspect OCI image: {oci_dir}"
213 assert layer_count == 3, f"Expected 3 layers, got {layer_count}"
214
215
216class TestLayerCaching:
217 """Test OCI layer caching functionality."""
218
219 @pytest.mark.slow
220 def test_cache_directory_created(self, build_dir, layer_cache_dir):
221 """Test that the layer cache directory is created after build."""
222 # Run the build
223 result = run_bitbake(build_dir, "app-container-multilayer", timeout=3600)
224 if result.returncode != 0:
225 pytest.skip("Build failed, skipping cache test")
226
227 # Check cache directory exists
228 assert layer_cache_dir.exists(), f"Cache directory not created: {layer_cache_dir}"
229
230 @pytest.mark.slow
231 def test_cache_entries_exist(self, build_dir, layer_cache_dir):
232 """Test that cache entries are created for each layer."""
233 # Run the build
234 result = run_bitbake(build_dir, "app-container-multilayer", timeout=3600)
235 if result.returncode != 0:
236 pytest.skip("Build failed, skipping cache test")
237
238 # Skip if cache dir doesn't exist
239 if not layer_cache_dir.exists():
240 pytest.skip("Cache directory not found")
241
242 # Check for cache entries (format: {hash}-{layer_name})
243 cache_entries = list(layer_cache_dir.iterdir())
244 assert len(cache_entries) >= 3, f"Expected at least 3 cache entries, found {len(cache_entries)}"
245
246 # Check for expected layer names
247 entry_names = [e.name for e in cache_entries]
248 has_base = any("base" in name for name in entry_names)
249 has_shell = any("shell" in name for name in entry_names)
250 has_app = any("app" in name for name in entry_names)
251
252 assert has_base, f"No cache entry for 'base' layer. Found: {entry_names}"
253 assert has_shell, f"No cache entry for 'shell' layer. Found: {entry_names}"
254 assert has_app, f"No cache entry for 'app' layer. Found: {entry_names}"
255
256 @pytest.mark.slow
257 def test_cache_marker_file(self, build_dir, layer_cache_dir):
258 """Test that cache entries have marker files."""
259 # Run the build
260 result = run_bitbake(build_dir, "app-container-multilayer", timeout=3600)
261 if result.returncode != 0:
262 pytest.skip("Build failed, skipping cache test")
263
264 if not layer_cache_dir.exists():
265 pytest.skip("Cache directory not found")
266
267 # Check each cache entry has a marker file
268 cache_entries = [e for e in layer_cache_dir.iterdir() if e.is_dir()]
269 for entry in cache_entries:
270 marker = entry / ".oci-layer-cache"
271 assert marker.exists(), f"No marker file in cache entry: {entry}"
272
273 # Check marker content
274 content = marker.read_text()
275 assert "cache_key=" in content
276 assert "layer_name=" in content
277 assert "created=" in content
278
279 @pytest.mark.slow
280 def test_cache_hit_on_rebuild(self, build_dir, machine):
281 """Test that cache hits occur on rebuild."""
282 # First build - should have cache misses
283 result = run_bitbake(build_dir, "app-container-multilayer", timeout=3600)
284 if result.returncode != 0:
285 pytest.skip("First build failed")
286
287 # Clean the work directory to force re-run of do_image_oci
288 work_pattern = f"tmp/work/*{machine}*/app-container-multilayer/*/oci-layer-rootfs"
289 for work_dir in build_dir.glob(work_pattern):
290 if work_dir.exists():
291 shutil.rmtree(work_dir)
292
293 # Remove stamp file to force task re-run
294 stamp_pattern = f"tmp/stamps/*{machine}*/app-container-multilayer/*.do_image_oci*"
295 for stamp in build_dir.glob(stamp_pattern):
296 stamp.unlink()
297
298 # Second build - should have cache hits
299 result = run_bitbake(build_dir, "app-container-multilayer", timeout=3600)
300 if result.returncode != 0:
301 pytest.fail(f"Second build failed:\n{result.stderr}")
302
303 # Check the log for cache hit messages
304 log_file = get_task_log(build_dir, machine, "app-container-multilayer", "do_image_oci")
305 if log_file and log_file.exists():
306 log_content = log_file.read_text()
307 assert "OCI Cache HIT" in log_content, \
308 "No cache hits found in log. Expected 'OCI Cache HIT' messages."
309 # Count hits vs misses
310 hits = log_content.count("OCI Cache HIT")
311 misses = log_content.count("OCI Cache MISS")
312 assert hits >= 3, f"Expected at least 3 cache hits, got {hits} hits and {misses} misses"
313
314
315class TestSingleLayerBackwardCompat:
316 """Test that single-layer mode (default) still works."""
317
318 @pytest.mark.slow
319 def test_single_layer_recipe_builds(self, build_dir, meta_virt_dir):
320 """Test that a single-layer OCI recipe still builds."""
321 # Check if app-container (single-layer) recipe exists
322 recipe = meta_virt_dir / "recipes-demo" / "images" / "app-container.bb"
323 if not recipe.exists():
324 pytest.skip("app-container recipe not found")
325
326 result = run_bitbake(build_dir, "app-container", timeout=3600)
327 if result.returncode != 0:
328 if "Nothing PROVIDES" in result.stderr:
329 pytest.skip("app-container recipe not available")
330 pytest.fail(f"Build failed: {result.stderr}")
331
332 @pytest.mark.slow
333 def test_single_layer_produces_one_layer(self, build_dir, deploy_dir, meta_virt_dir):
334 """Test that single-layer build produces 1 layer."""
335 # Check if recipe exists
336 recipe = meta_virt_dir / "recipes-demo" / "images" / "app-container.bb"
337 if not recipe.exists():
338 pytest.skip("app-container recipe not found")
339
340 result = run_bitbake(build_dir, "app-container", timeout=3600)
341 if result.returncode != 0:
342 pytest.skip("Build failed")
343
344 # Find the OCI directory
345 oci_dirs = list(deploy_dir.glob("app-container-*-oci"))
346 # Filter out multilayer
347 oci_dirs = [d for d in oci_dirs if "multilayer" not in d.name]
348
349 if not oci_dirs:
350 pytest.skip("No OCI directory found for app-container")
351
352 oci_dir = oci_dirs[0]
353 if oci_dir.is_symlink():
354 oci_dir = oci_dir.resolve()
355
356 layer_count = get_oci_layer_count(oci_dir)
357 assert layer_count is not None, f"Failed to inspect OCI image: {oci_dir}"
358 assert layer_count == 1, f"Expected 1 layer for single-layer mode, got {layer_count}"
359
360
361class TestTwoLayerBaseImage:
362 """Test two-layer OCI images using OCI_BASE_IMAGE."""
363
364 def test_layered_recipe_exists(self, meta_virt_dir):
365 """Test that the two-layer demo recipe exists."""
366 recipe = meta_virt_dir / "recipes-demo" / "images" / "app-container-layered.bb"
367 assert recipe.exists(), f"Recipe not found: {recipe}"
368
369 def test_layered_recipe_uses_base_image(self, meta_virt_dir):
370 """Test that the layered recipe uses OCI_BASE_IMAGE."""
371 recipe = meta_virt_dir / "recipes-demo" / "images" / "app-container-layered.bb"
372 if not recipe.exists():
373 pytest.skip("Recipe not found")
374
375 content = recipe.read_text()
376 assert "OCI_BASE_IMAGE" in content, "Recipe should use OCI_BASE_IMAGE"
377 assert "container-base" in content, "Recipe should use container-base as base"
378
379 @pytest.mark.slow
380 def test_layered_recipe_builds(self, build_dir):
381 """Test that app-container-layered recipe builds successfully."""
382 # First ensure the base image is built
383 result = run_bitbake(build_dir, "container-base", timeout=3600)
384 if result.returncode != 0:
385 if "Nothing PROVIDES" in result.stderr:
386 pytest.skip("container-base recipe not available")
387 pytest.fail(f"Base image build failed: {result.stderr}")
388
389 # Now build the layered image
390 result = run_bitbake(build_dir, "app-container-layered", timeout=3600)
391 if result.returncode != 0:
392 if "Nothing PROVIDES" in result.stderr:
393 pytest.skip("app-container-layered recipe not available")
394 pytest.fail(f"Build failed:\nstdout: {result.stdout}\nstderr: {result.stderr}")
395
396 @pytest.mark.slow
397 def test_layered_produces_two_layers(self, build_dir, deploy_dir):
398 """Test that two-layer build produces 2 layers (base + app)."""
399 # Ensure the base is built first
400 result = run_bitbake(build_dir, "container-base", timeout=3600)
401 if result.returncode != 0:
402 pytest.skip("Base image build failed")
403
404 # Build the layered image
405 result = run_bitbake(build_dir, "app-container-layered", timeout=3600)
406 if result.returncode != 0:
407 pytest.skip("Build failed, skipping layer count check")
408
409 # Find the OCI directory
410 oci_dirs = list(deploy_dir.glob("app-container-layered*-oci"))
411 assert len(oci_dirs) > 0, "No OCI directory found for app-container-layered"
412
413 # Get the actual OCI directory (resolve symlink if needed)
414 oci_dir = oci_dirs[0]
415 if oci_dir.is_symlink():
416 oci_dir = oci_dir.resolve()
417
418 # Check layer count - should be 2 (base + app)
419 layer_count = get_oci_layer_count(oci_dir)
420 assert layer_count is not None, f"Failed to inspect OCI image: {oci_dir}"
421 assert layer_count == 2, f"Expected 2 layers (base + app), got {layer_count}"
422
423 @pytest.mark.slow
424 def test_base_image_produces_one_layer(self, build_dir, deploy_dir):
425 """Test that container-base (the base image) produces 1 layer."""
426 result = run_bitbake(build_dir, "container-base", timeout=3600)
427 if result.returncode != 0:
428 pytest.skip("Build failed")
429
430 # Find the OCI directory
431 oci_dirs = list(deploy_dir.glob("container-base*-oci"))
432 if not oci_dirs:
433 pytest.skip("No OCI directory found for container-base")
434
435 oci_dir = oci_dirs[0]
436 if oci_dir.is_symlink():
437 oci_dir = oci_dir.resolve()
438
439 layer_count = get_oci_layer_count(oci_dir)
440 assert layer_count is not None, f"Failed to inspect OCI image: {oci_dir}"
441 assert layer_count == 1, f"Expected 1 layer for base image, got {layer_count}"
442
443
444class TestLayerTypes:
445 """Test different OCI_LAYERS types."""
446
447 def test_packages_layer_type(self, meta_virt_dir):
448 """Test that 'packages' layer type is supported."""
449 recipe = meta_virt_dir / "recipes-demo" / "images" / "app-container-multilayer.bb"
450 if not recipe.exists():
451 pytest.skip("Recipe not found")
452
453 content = recipe.read_text()
454 assert "packages" in content, "Recipe should use 'packages' layer type"
455
456 def test_directories_layer_type_documented(self, meta_virt_dir):
457 """Test that 'directories' layer type is documented."""
458 class_file = meta_virt_dir / "classes" / "image-oci.bbclass"
459 content = class_file.read_text()
460 assert "directories" in content, "directories layer type should be documented"
461
462 def test_files_layer_type_documented(self, meta_virt_dir):
463 """Test that 'files' layer type is documented."""
464 class_file = meta_virt_dir / "classes" / "image-oci.bbclass"
465 content = class_file.read_text()
466 assert "files" in content, "files layer type should be documented"