summaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa/selftest/cases/sstatetests.py
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/oeqa/selftest/cases/sstatetests.py')
-rw-r--r--meta/lib/oeqa/selftest/cases/sstatetests.py711
1 files changed, 572 insertions, 139 deletions
diff --git a/meta/lib/oeqa/selftest/cases/sstatetests.py b/meta/lib/oeqa/selftest/cases/sstatetests.py
index c46e8ba489..86d6cd7464 100644
--- a/meta/lib/oeqa/selftest/cases/sstatetests.py
+++ b/meta/lib/oeqa/selftest/cases/sstatetests.py
@@ -1,4 +1,6 @@
1# 1#
2# Copyright OpenEmbedded Contributors
3#
2# SPDX-License-Identifier: MIT 4# SPDX-License-Identifier: MIT
3# 5#
4 6
@@ -7,54 +9,77 @@ import shutil
7import glob 9import glob
8import subprocess 10import subprocess
9import tempfile 11import tempfile
12import datetime
13import re
10 14
15from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, get_bb_vars
11from oeqa.selftest.case import OESelftestTestCase 16from oeqa.selftest.case import OESelftestTestCase
12from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer, create_temp_layer 17from oeqa.core.decorator import OETestTag
13from oeqa.selftest.cases.sstate import SStateBase
14 18
19import oe
15import bb.siggen 20import bb.siggen
16 21
17class SStateTests(SStateBase): 22# Set to True to preserve stamp files after test execution for debugging failures
18 def test_autorev_sstate_works(self): 23keep_temp_files = False
19 # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} 24
20 # when PV does not contain SRCPV 25class SStateBase(OESelftestTestCase):
21 26
22 tempdir = tempfile.mkdtemp(prefix='sstate_autorev') 27 def setUpLocal(self):
23 tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') 28 super(SStateBase, self).setUpLocal()
24 self.track_for_cleanup(tempdir) 29 self.temp_sstate_location = None
25 self.track_for_cleanup(tempdldir) 30 needed_vars = ['SSTATE_DIR', 'NATIVELSBSTRING', 'TCLIBC', 'TUNE_ARCH',
26 create_temp_layer(tempdir, 'selftestrecipetool') 31 'TOPDIR', 'TARGET_VENDOR', 'TARGET_OS']
27 self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) 32 bb_vars = get_bb_vars(needed_vars)
28 self.append_config("DL_DIR = \"%s\"" % tempdldir) 33 self.sstate_path = bb_vars['SSTATE_DIR']
29 runCmd('bitbake-layers add-layer %s' % tempdir) 34 self.hostdistro = bb_vars['NATIVELSBSTRING']
30 35 self.tclibc = bb_vars['TCLIBC']
31 # Use dbus-wait as a local git repo we can add a commit between two builds in 36 self.tune_arch = bb_vars['TUNE_ARCH']
32 pn = 'dbus-wait' 37 self.topdir = bb_vars['TOPDIR']
33 srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' 38 self.target_vendor = bb_vars['TARGET_VENDOR']
34 url = 'git://git.yoctoproject.org/dbus-wait' 39 self.target_os = bb_vars['TARGET_OS']
35 result = runCmd('git clone %s noname' % url, cwd=tempdir) 40 self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro)
36 srcdir = os.path.join(tempdir, 'noname') 41
37 result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) 42 def track_for_cleanup(self, path):
38 self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') 43 if not keep_temp_files:
39 44 super().track_for_cleanup(path)
40 recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb') 45
41 os.makedirs(os.path.dirname(recipefile)) 46 # Creates a special sstate configuration with the option to add sstate mirrors
42 srcuri = 'git://' + srcdir + ';protocol=file' 47 def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]):
43 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri]) 48 self.temp_sstate_location = temp_sstate_location
44 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) 49
45 50 if self.temp_sstate_location:
46 with open(recipefile, 'a') as f: 51 temp_sstate_path = os.path.join(self.builddir, "temp_sstate_%s" % datetime.datetime.now().strftime('%Y%m%d%H%M%S'))
47 f.write('SRCREV = "${AUTOREV}"\n') 52 config_temp_sstate = "SSTATE_DIR = \"%s\"" % temp_sstate_path
48 f.write('PV = "1.0"\n') 53 self.append_config(config_temp_sstate)
49 54 self.track_for_cleanup(temp_sstate_path)
50 bitbake("dbus-wait-test -c fetch") 55 bb_vars = get_bb_vars(['SSTATE_DIR', 'NATIVELSBSTRING'])
51 with open(os.path.join(srcdir, "bar.txt"), "w") as f: 56 self.sstate_path = bb_vars['SSTATE_DIR']
52 f.write("foo") 57 self.hostdistro = bb_vars['NATIVELSBSTRING']
53 result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir) 58 self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro)
54 bitbake("dbus-wait-test -c unpack") 59
55 60 if add_local_mirrors:
56 61 config_set_sstate_if_not_set = 'SSTATE_MIRRORS ?= ""'
57 # Test sstate files creation and their location 62 self.append_config(config_set_sstate_if_not_set)
63 for local_mirror in add_local_mirrors:
64 self.assertFalse(os.path.join(local_mirror) == os.path.join(self.sstate_path), msg='Cannot add the current sstate path as a sstate mirror')
65 config_sstate_mirror = "SSTATE_MIRRORS += \"file://.* file:///%s/PATH\"" % local_mirror
66 self.append_config(config_sstate_mirror)
67
68 # Returns a list containing sstate files
69 def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True):
70 result = []
71 for root, dirs, files in os.walk(self.sstate_path):
72 if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root):
73 for f in files:
74 if re.search(filename_regex, f):
75 result.append(f)
76 if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root):
77 for f in files:
78 if re.search(filename_regex, f):
79 result.append(f)
80 return result
81
82 # Test sstate files creation and their location and directory perms
58 def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): 83 def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True):
59 self.config_sstate(temp_sstate_location, [self.sstate_path]) 84 self.config_sstate(temp_sstate_location, [self.sstate_path])
60 85
@@ -63,12 +88,25 @@ class SStateTests(SStateBase):
63 else: 88 else:
64 bitbake(['-ccleansstate'] + targets) 89 bitbake(['-ccleansstate'] + targets)
65 90
91 # We need to test that the env umask have does not effect sstate directory creation
92 # So, first, we'll get the current umask and set it to something we know incorrect
93 # See: sstate_task_postfunc for correct umask of os.umask(0o002)
94 import os
95 def current_umask():
96 current_umask = os.umask(0)
97 os.umask(current_umask)
98 return current_umask
99
100 orig_umask = current_umask()
101 # Set it to a umask we know will be 'wrong'
102 os.umask(0o022)
103
66 bitbake(targets) 104 bitbake(targets)
67 file_tracker = [] 105 file_tracker = []
68 results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) 106 results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific)
69 if distro_nonspecific: 107 if distro_nonspecific:
70 for r in results: 108 for r in results:
71 if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo", "_fetch.tgz.siginfo", "_unpack.tgz.siginfo", "_patch.tgz.siginfo")): 109 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo", "_fetch.tar.zst.siginfo", "_unpack.tar.zst.siginfo", "_patch.tar.zst.siginfo")):
72 continue 110 continue
73 file_tracker.append(r) 111 file_tracker.append(r)
74 else: 112 else:
@@ -79,17 +117,18 @@ class SStateTests(SStateBase):
79 else: 117 else:
80 self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) 118 self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker)))
81 119
82 def test_sstate_creation_distro_specific_pass(self): 120 # Now we'll walk the tree to check the mode and see if things are incorrect.
83 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) 121 badperms = []
84 122 for root, dirs, files in os.walk(self.sstate_path):
85 def test_sstate_creation_distro_specific_fail(self): 123 for directory in dirs:
86 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) 124 if (os.stat(os.path.join(root, directory)).st_mode & 0o777) != 0o775:
125 badperms.append(os.path.join(root, directory))
87 126
88 def test_sstate_creation_distro_nonspecific_pass(self): 127 # Return to original umask
89 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) 128 os.umask(orig_umask)
90 129
91 def test_sstate_creation_distro_nonspecific_fail(self): 130 if should_pass:
92 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) 131 self.assertTrue(badperms , msg="Found sstate directories with the wrong permissions: %s (found %s)" % (', '.join(map(str, targets)), str(badperms)))
93 132
94 # Test the sstate files deletion part of the do_cleansstate task 133 # Test the sstate files deletion part of the do_cleansstate task
95 def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): 134 def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True):
@@ -98,29 +137,15 @@ class SStateTests(SStateBase):
98 bitbake(['-ccleansstate'] + targets) 137 bitbake(['-ccleansstate'] + targets)
99 138
100 bitbake(targets) 139 bitbake(targets)
101 tgz_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) 140 archives_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific)
102 self.assertTrue(tgz_created, msg="Could not find sstate .tgz files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_created))) 141 self.assertTrue(archives_created, msg="Could not find sstate .tar.zst files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_created)))
103 142
104 siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) 143 siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific)
105 self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) 144 self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created)))
106 145
107 bitbake(['-ccleansstate'] + targets) 146 bitbake(['-ccleansstate'] + targets)
108 tgz_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) 147 archives_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific)
109 self.assertTrue(not tgz_removed, msg="do_cleansstate didn't remove .tgz sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_removed))) 148 self.assertTrue(not archives_removed, msg="do_cleansstate didn't remove .tar.zst sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_removed)))
110
111 def test_cleansstate_task_distro_specific_nonspecific(self):
112 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native']
113 targets.append('linux-libc-headers')
114 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True)
115
116 def test_cleansstate_task_distro_nonspecific(self):
117 self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True)
118
119 def test_cleansstate_task_distro_specific(self):
120 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native']
121 targets.append('linux-libc-headers')
122 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True)
123
124 149
125 # Test rebuilding of distro-specific sstate files 150 # Test rebuilding of distro-specific sstate files
126 def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): 151 def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True):
@@ -129,15 +154,15 @@ class SStateTests(SStateBase):
129 bitbake(['-ccleansstate'] + targets) 154 bitbake(['-ccleansstate'] + targets)
130 155
131 bitbake(targets) 156 bitbake(targets)
132 results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=False, distro_nonspecific=True) 157 results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=False, distro_nonspecific=True)
133 filtered_results = [] 158 filtered_results = []
134 for r in results: 159 for r in results:
135 if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo")): 160 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo")):
136 continue 161 continue
137 filtered_results.append(r) 162 filtered_results.append(r)
138 self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) 163 self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results)))
139 file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=True, distro_nonspecific=False) 164 file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False)
140 self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files ware created for: %s" % ', '.join(map(str, targets))) 165 self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets)))
141 166
142 self.track_for_cleanup(self.distro_specific_sstate + "_old") 167 self.track_for_cleanup(self.distro_specific_sstate + "_old")
143 shutil.copytree(self.distro_specific_sstate, self.distro_specific_sstate + "_old") 168 shutil.copytree(self.distro_specific_sstate, self.distro_specific_sstate + "_old")
@@ -145,15 +170,114 @@ class SStateTests(SStateBase):
145 170
146 bitbake(['-cclean'] + targets) 171 bitbake(['-cclean'] + targets)
147 bitbake(targets) 172 bitbake(targets)
148 file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=True, distro_nonspecific=False) 173 file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False)
149 self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files ware created for: %s" % ', '.join(map(str, targets))) 174 self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets)))
150 175
151 not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2] 176 not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2]
152 self.assertTrue(not_recreated == [], msg="The following sstate files ware not recreated: %s" % ', '.join(map(str, not_recreated))) 177 self.assertTrue(not_recreated == [], msg="The following sstate files were not recreated: %s" % ', '.join(map(str, not_recreated)))
153 178
154 created_once = [x for x in file_tracker_2 if x not in file_tracker_1] 179 created_once = [x for x in file_tracker_2 if x not in file_tracker_1]
155 self.assertTrue(created_once == [], msg="The following sstate files ware created only in the second run: %s" % ', '.join(map(str, created_once))) 180 self.assertTrue(created_once == [], msg="The following sstate files were created only in the second run: %s" % ', '.join(map(str, created_once)))
181
182 def sstate_common_samesigs(self, configA, configB, allarch=False):
183
184 self.write_config(configA)
185 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
186 bitbake("world meta-toolchain -S none")
187 self.write_config(configB)
188 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
189 bitbake("world meta-toolchain -S none")
190
191 def get_files(d, result):
192 for root, dirs, files in os.walk(d):
193 for name in files:
194 if "meta-environment" in root or "cross-canadian" in root:
195 continue
196 if "do_build" not in name:
197 # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79
198 (_, task, _, shash) = name.rsplit(".", 3)
199 result[os.path.join(os.path.basename(root), task)] = shash
200
201 files1 = {}
202 files2 = {}
203 subdirs = sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux"))
204 if allarch:
205 subdirs.extend(sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/all-*-linux")))
206
207 for subdir in subdirs:
208 nativesdkdir = os.path.basename(subdir)
209 get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir, files1)
210 get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir, files2)
211
212 self.maxDiff = None
213 self.assertEqual(files1, files2)
214
215class SStateTests(SStateBase):
216 def test_autorev_sstate_works(self):
217 # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV}
156 218
219 tempdir = tempfile.mkdtemp(prefix='sstate_autorev')
220 tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir')
221 self.track_for_cleanup(tempdir)
222 self.track_for_cleanup(tempdldir)
223 create_temp_layer(tempdir, 'selftestrecipetool')
224 self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir)
225 self.append_config("DL_DIR = \"%s\"" % tempdldir)
226 runCmd('bitbake-layers add-layer %s' % tempdir)
227
228 # Use dbus-wait as a local git repo we can add a commit between two builds in
229 pn = 'dbus-wait'
230 srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517'
231 url = 'git://git.yoctoproject.org/dbus-wait'
232 result = runCmd('git clone %s noname' % url, cwd=tempdir)
233 srcdir = os.path.join(tempdir, 'noname')
234 result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir)
235 self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory')
236
237 recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb')
238 os.makedirs(os.path.dirname(recipefile))
239 srcuri = 'git://' + srcdir + ';protocol=file;branch=master'
240 result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri])
241 self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output)
242
243 with open(recipefile, 'a') as f:
244 f.write('SRCREV = "${AUTOREV}"\n')
245 f.write('PV = "1.0"\n')
246
247 bitbake("dbus-wait-test -c fetch")
248 with open(os.path.join(srcdir, "bar.txt"), "w") as f:
249 f.write("foo")
250 result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir)
251 bitbake("dbus-wait-test -c unpack")
252
253class SStateCreation(SStateBase):
254 def test_sstate_creation_distro_specific_pass(self):
255 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True)
256
257 def test_sstate_creation_distro_specific_fail(self):
258 self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False)
259
260 def test_sstate_creation_distro_nonspecific_pass(self):
261 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True)
262
263 def test_sstate_creation_distro_nonspecific_fail(self):
264 self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False)
265
266class SStateCleanup(SStateBase):
267 def test_cleansstate_task_distro_specific_nonspecific(self):
268 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native']
269 targets.append('linux-libc-headers')
270 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True)
271
272 def test_cleansstate_task_distro_nonspecific(self):
273 self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True)
274
275 def test_cleansstate_task_distro_specific(self):
276 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native']
277 targets.append('linux-libc-headers')
278 self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True)
279
280class SStateDistroTests(SStateBase):
157 def test_rebuild_distro_specific_sstate_cross_native_targets(self): 281 def test_rebuild_distro_specific_sstate_cross_native_targets(self):
158 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) 282 self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True)
159 283
@@ -163,48 +287,48 @@ class SStateTests(SStateBase):
163 def test_rebuild_distro_specific_sstate_native_target(self): 287 def test_rebuild_distro_specific_sstate_native_target(self):
164 self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) 288 self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True)
165 289
166 290class SStateCacheManagement(SStateBase):
167 # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list 291 # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list
168 # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.sh (such as changing the value of MACHINE) 292 # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.py (such as changing the value of MACHINE)
169 def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]): 293 def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]):
170 self.assertTrue(global_config) 294 self.assertTrue(global_config)
171 self.assertTrue(target_config) 295 self.assertTrue(target_config)
172 self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') 296 self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements')
173 self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path])
174 297
175 # If buildhistory is enabled, we need to disable version-going-backwards 298 for idx in range(len(target_config)):
176 # QA checks for this test. It may report errors otherwise. 299 self.append_config(global_config[idx])
177 self.append_config('ERROR_QA_remove = "version-going-backwards"') 300 self.append_recipeinc(target, target_config[idx])
301 bitbake(target)
302 self.remove_config(global_config[idx])
303 self.remove_recipeinc(target, target_config[idx])
304
305 self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path])
178 306
179 # For not this only checks if random sstate tasks are handled correctly as a group. 307 # For now this only checks if random sstate tasks are handled correctly as a group.
180 # In the future we should add control over what tasks we check for. 308 # In the future we should add control over what tasks we check for.
181 309
182 sstate_archs_list = []
183 expected_remaining_sstate = [] 310 expected_remaining_sstate = []
184 for idx in range(len(target_config)): 311 for idx in range(len(target_config)):
185 self.append_config(global_config[idx]) 312 self.append_config(global_config[idx])
186 self.append_recipeinc(target, target_config[idx]) 313 self.append_recipeinc(target, target_config[idx])
187 sstate_arch = get_bb_var('SSTATE_PKGARCH', target)
188 if not sstate_arch in sstate_archs_list:
189 sstate_archs_list.append(sstate_arch)
190 if target_config[idx] == target_config[-1]: 314 if target_config[idx] == target_config[-1]:
191 target_sstate_before_build = self.search_sstate(target + r'.*?\.tgz$') 315 target_sstate_before_build = self.search_sstate(target + r'.*?\.tar.zst$')
192 bitbake("-cclean %s" % target) 316 bitbake("-cclean %s" % target)
193 result = bitbake(target, ignore_status=True) 317 result = bitbake(target, ignore_status=True)
194 if target_config[idx] == target_config[-1]: 318 if target_config[idx] == target_config[-1]:
195 target_sstate_after_build = self.search_sstate(target + r'.*?\.tgz$') 319 target_sstate_after_build = self.search_sstate(target + r'.*?\.tar.zst$')
196 expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)] 320 expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)]
197 self.remove_config(global_config[idx]) 321 self.remove_config(global_config[idx])
198 self.remove_recipeinc(target, target_config[idx]) 322 self.remove_recipeinc(target, target_config[idx])
199 self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) 323 self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output))
200 324
201 runCmd("sstate-cache-management.sh -y --cache-dir=%s --remove-duplicated --extra-archs=%s" % (self.sstate_path, ','.join(map(str, sstate_archs_list)))) 325 runCmd("sstate-cache-management.py -y --cache-dir=%s --remove-duplicated" % (self.sstate_path))
202 actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tgz$') if not any(pattern in x for pattern in ignore_patterns)] 326 actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tar.zst$') if not any(pattern in x for pattern in ignore_patterns)]
203 327
204 actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate] 328 actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate]
205 self.assertFalse(actual_not_expected, msg="Files should have been removed but ware not: %s" % ', '.join(map(str, actual_not_expected))) 329 self.assertFalse(actual_not_expected, msg="Files should have been removed but were not: %s" % ', '.join(map(str, actual_not_expected)))
206 expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate] 330 expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate]
207 self.assertFalse(expected_not_actual, msg="Extra files ware removed: %s" ', '.join(map(str, expected_not_actual))) 331 self.assertFalse(expected_not_actual, msg="Extra files were removed: %s" ', '.join(map(str, expected_not_actual)))
208 332
209 def test_sstate_cache_management_script_using_pr_1(self): 333 def test_sstate_cache_management_script_using_pr_1(self):
210 global_config = [] 334 global_config = []
@@ -242,6 +366,7 @@ class SStateTests(SStateBase):
242 target_config.append('') 366 target_config.append('')
243 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) 367 self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic'])
244 368
369class SStateHashSameSigs(SStateBase):
245 def test_sstate_32_64_same_hash(self): 370 def test_sstate_32_64_same_hash(self):
246 """ 371 """
247 The sstate checksums for both native and target should not vary whether 372 The sstate checksums for both native and target should not vary whether
@@ -261,7 +386,7 @@ PACKAGE_CLASSES = "package_rpm package_ipk package_deb"
261BB_SIGNATURE_HANDLER = "OEBasicHash" 386BB_SIGNATURE_HANDLER = "OEBasicHash"
262""") 387""")
263 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 388 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
264 bitbake("core-image-sato -S none") 389 bitbake("core-image-weston -S none")
265 self.write_config(""" 390 self.write_config("""
266MACHINE = "qemux86" 391MACHINE = "qemux86"
267TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" 392TMPDIR = "${TOPDIR}/tmp-sstatesamehash2"
@@ -273,12 +398,12 @@ PACKAGE_CLASSES = "package_rpm package_ipk package_deb"
273BB_SIGNATURE_HANDLER = "OEBasicHash" 398BB_SIGNATURE_HANDLER = "OEBasicHash"
274""") 399""")
275 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 400 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
276 bitbake("core-image-sato -S none") 401 bitbake("core-image-weston -S none")
277 402
278 def get_files(d): 403 def get_files(d):
279 f = [] 404 f = []
280 for root, dirs, files in os.walk(d): 405 for root, dirs, files in os.walk(d):
281 if "core-image-sato" in root: 406 if "core-image-weston" in root:
282 # SDKMACHINE changing will change 407 # SDKMACHINE changing will change
283 # do_rootfs/do_testimage/do_build stamps of images which 408 # do_rootfs/do_testimage/do_build stamps of images which
284 # is safe to ignore. 409 # is safe to ignore.
@@ -306,7 +431,7 @@ NATIVELSBSTRING = \"DistroA\"
306BB_SIGNATURE_HANDLER = "OEBasicHash" 431BB_SIGNATURE_HANDLER = "OEBasicHash"
307""") 432""")
308 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 433 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
309 bitbake("core-image-sato -S none") 434 bitbake("core-image-weston -S none")
310 self.write_config(""" 435 self.write_config("""
311TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 436TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
312TCLIBCAPPEND = \"\" 437TCLIBCAPPEND = \"\"
@@ -314,7 +439,7 @@ NATIVELSBSTRING = \"DistroB\"
314BB_SIGNATURE_HANDLER = "OEBasicHash" 439BB_SIGNATURE_HANDLER = "OEBasicHash"
315""") 440""")
316 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 441 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
317 bitbake("core-image-sato -S none") 442 bitbake("core-image-weston -S none")
318 443
319 def get_files(d): 444 def get_files(d):
320 f = [] 445 f = []
@@ -327,6 +452,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash"
327 self.maxDiff = None 452 self.maxDiff = None
328 self.assertCountEqual(files1, files2) 453 self.assertCountEqual(files1, files2)
329 454
455class SStateHashSameSigs2(SStateBase):
330 def test_sstate_allarch_samesigs(self): 456 def test_sstate_allarch_samesigs(self):
331 """ 457 """
332 The sstate checksums of allarch packages should be independent of whichever 458 The sstate checksums of allarch packages should be independent of whichever
@@ -341,13 +467,15 @@ TCLIBCAPPEND = \"\"
341MACHINE = \"qemux86-64\" 467MACHINE = \"qemux86-64\"
342BB_SIGNATURE_HANDLER = "OEBasicHash" 468BB_SIGNATURE_HANDLER = "OEBasicHash"
343""" 469"""
470 #OLDEST_KERNEL is arch specific so set to a different value here for testing
344 configB = """ 471 configB = """
345TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" 472TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\"
346TCLIBCAPPEND = \"\" 473TCLIBCAPPEND = \"\"
347MACHINE = \"qemuarm\" 474MACHINE = \"qemuarm\"
475OLDEST_KERNEL = \"3.3.0\"
348BB_SIGNATURE_HANDLER = "OEBasicHash" 476BB_SIGNATURE_HANDLER = "OEBasicHash"
349""" 477"""
350 self.sstate_allarch_samesigs(configA, configB) 478 self.sstate_common_samesigs(configA, configB, allarch=True)
351 479
352 def test_sstate_nativesdk_samesigs_multilib(self): 480 def test_sstate_nativesdk_samesigs_multilib(self):
353 """ 481 """
@@ -360,7 +488,7 @@ TCLIBCAPPEND = \"\"
360MACHINE = \"qemux86-64\" 488MACHINE = \"qemux86-64\"
361require conf/multilib.conf 489require conf/multilib.conf
362MULTILIBS = \"multilib:lib32\" 490MULTILIBS = \"multilib:lib32\"
363DEFAULTTUNE_virtclass-multilib-lib32 = \"x86\" 491DEFAULTTUNE:virtclass-multilib-lib32 = \"x86\"
364BB_SIGNATURE_HANDLER = "OEBasicHash" 492BB_SIGNATURE_HANDLER = "OEBasicHash"
365""" 493"""
366 configB = """ 494 configB = """
@@ -371,36 +499,9 @@ require conf/multilib.conf
371MULTILIBS = \"\" 499MULTILIBS = \"\"
372BB_SIGNATURE_HANDLER = "OEBasicHash" 500BB_SIGNATURE_HANDLER = "OEBasicHash"
373""" 501"""
374 self.sstate_allarch_samesigs(configA, configB) 502 self.sstate_common_samesigs(configA, configB)
375
376 def sstate_allarch_samesigs(self, configA, configB):
377
378 self.write_config(configA)
379 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
380 bitbake("world meta-toolchain -S none")
381 self.write_config(configB)
382 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
383 bitbake("world meta-toolchain -S none")
384
385 def get_files(d):
386 f = {}
387 for root, dirs, files in os.walk(d):
388 for name in files:
389 if "meta-environment" in root or "cross-canadian" in root:
390 continue
391 if "do_build" not in name:
392 # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79
393 (_, task, _, shash) = name.rsplit(".", 3)
394 f[os.path.join(os.path.basename(root), task)] = shash
395 return f
396
397 nativesdkdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0])
398
399 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir)
400 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir)
401 self.maxDiff = None
402 self.assertEqual(files1, files2)
403 503
504class SStateHashSameSigs3(SStateBase):
404 def test_sstate_sametune_samesigs(self): 505 def test_sstate_sametune_samesigs(self):
405 """ 506 """
406 The sstate checksums of two identical machines (using the same tune) should be the 507 The sstate checksums of two identical machines (using the same tune) should be the
@@ -414,7 +515,7 @@ TCLIBCAPPEND = \"\"
414MACHINE = \"qemux86\" 515MACHINE = \"qemux86\"
415require conf/multilib.conf 516require conf/multilib.conf
416MULTILIBS = "multilib:lib32" 517MULTILIBS = "multilib:lib32"
417DEFAULTTUNE_virtclass-multilib-lib32 = "x86" 518DEFAULTTUNE:virtclass-multilib-lib32 = "x86"
418BB_SIGNATURE_HANDLER = "OEBasicHash" 519BB_SIGNATURE_HANDLER = "OEBasicHash"
419""") 520""")
420 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 521 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
@@ -425,7 +526,7 @@ TCLIBCAPPEND = \"\"
425MACHINE = \"qemux86copy\" 526MACHINE = \"qemux86copy\"
426require conf/multilib.conf 527require conf/multilib.conf
427MULTILIBS = "multilib:lib32" 528MULTILIBS = "multilib:lib32"
428DEFAULTTUNE_virtclass-multilib-lib32 = "x86" 529DEFAULTTUNE:virtclass-multilib-lib32 = "x86"
429BB_SIGNATURE_HANDLER = "OEBasicHash" 530BB_SIGNATURE_HANDLER = "OEBasicHash"
430""") 531""")
431 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") 532 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
@@ -435,7 +536,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash"
435 f = [] 536 f = []
436 for root, dirs, files in os.walk(d): 537 for root, dirs, files in os.walk(d):
437 for name in files: 538 for name in files:
438 if "meta-environment" in root or "cross-canadian" in root: 539 if "meta-environment" in root or "cross-canadian" in root or 'meta-ide-support' in root:
439 continue 540 continue
440 if "qemux86copy-" in root or "qemux86-" in root: 541 if "qemux86copy-" in root or "qemux86-" in root:
441 continue 542 continue
@@ -462,7 +563,7 @@ TCLIBCAPPEND = \"\"
462MACHINE = \"qemux86\" 563MACHINE = \"qemux86\"
463require conf/multilib.conf 564require conf/multilib.conf
464MULTILIBS = "multilib:lib32" 565MULTILIBS = "multilib:lib32"
465DEFAULTTUNE_virtclass-multilib-lib32 = "x86" 566DEFAULTTUNE:virtclass-multilib-lib32 = "x86"
466BB_SIGNATURE_HANDLER = "OEBasicHash" 567BB_SIGNATURE_HANDLER = "OEBasicHash"
467""") 568""")
468 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") 569 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
@@ -488,7 +589,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash"
488 self.maxDiff = None 589 self.maxDiff = None
489 self.assertCountEqual(files1, files2) 590 self.assertCountEqual(files1, files2)
490 591
491 592class SStateHashSameSigs4(SStateBase):
492 def test_sstate_noop_samesigs(self): 593 def test_sstate_noop_samesigs(self):
493 """ 594 """
494 The sstate checksums of two builds with these variables changed or 595 The sstate checksums of two builds with these variables changed or
@@ -503,7 +604,7 @@ PARALLEL_MAKE = "-j 1"
503DL_DIR = "${TOPDIR}/download1" 604DL_DIR = "${TOPDIR}/download1"
504TIME = "111111" 605TIME = "111111"
505DATE = "20161111" 606DATE = "20161111"
506INHERIT_remove = "buildstats-summary buildhistory uninative" 607INHERIT:remove = "buildstats-summary buildhistory uninative"
507http_proxy = "" 608http_proxy = ""
508BB_SIGNATURE_HANDLER = "OEBasicHash" 609BB_SIGNATURE_HANDLER = "OEBasicHash"
509""") 610""")
@@ -519,7 +620,7 @@ DL_DIR = "${TOPDIR}/download2"
519TIME = "222222" 620TIME = "222222"
520DATE = "20161212" 621DATE = "20161212"
521# Always remove uninative as we're changing proxies 622# Always remove uninative as we're changing proxies
522INHERIT_remove = "uninative" 623INHERIT:remove = "uninative"
523INHERIT += "buildstats-summary buildhistory" 624INHERIT += "buildstats-summary buildhistory"
524http_proxy = "http://example.com/" 625http_proxy = "http://example.com/"
525BB_SIGNATURE_HANDLER = "OEBasicHash" 626BB_SIGNATURE_HANDLER = "OEBasicHash"
@@ -573,3 +674,335 @@ BB_SIGNATURE_HANDLER = "OEBasicHash"
573 compare_sigfiles(rest, files1, files2, compare=False) 674 compare_sigfiles(rest, files1, files2, compare=False)
574 675
575 self.fail("sstate hashes not identical.") 676 self.fail("sstate hashes not identical.")
677
678 def test_sstate_movelayer_samesigs(self):
679 """
680 The sstate checksums of two builds with the same oe-core layer in two
681 different locations should be the same.
682 """
683 core_layer = os.path.join(
684 self.tc.td["COREBASE"], 'meta')
685 copy_layer_1 = self.topdir + "/meta-copy1/meta"
686 copy_layer_2 = self.topdir + "/meta-copy2/meta"
687
688 oe.path.copytree(core_layer, copy_layer_1)
689 os.symlink(os.path.dirname(core_layer) + "/scripts", self.topdir + "/meta-copy1/scripts")
690 self.write_config("""
691TMPDIR = "${TOPDIR}/tmp-sstatesamehash"
692""")
693 bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_1, core_layer)
694 self.write_bblayers_config(bblayers_conf)
695 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash")
696 bitbake("bash -S none")
697
698 oe.path.copytree(core_layer, copy_layer_2)
699 os.symlink(os.path.dirname(core_layer) + "/scripts", self.topdir + "/meta-copy2/scripts")
700 self.write_config("""
701TMPDIR = "${TOPDIR}/tmp-sstatesamehash2"
702""")
703 bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_2, core_layer)
704 self.write_bblayers_config(bblayers_conf)
705 self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2")
706 bitbake("bash -S none")
707
708 def get_files(d):
709 f = []
710 for root, dirs, files in os.walk(d):
711 for name in files:
712 f.append(os.path.join(root, name))
713 return f
714 files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps")
715 files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps")
716 files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2]
717 self.maxDiff = None
718 self.assertCountEqual(files1, files2)
719
720class SStateFindSiginfo(SStateBase):
721 def test_sstate_compare_sigfiles_and_find_siginfo(self):
722 """
723 Test the functionality of the find_siginfo: basic function and callback in compare_sigfiles
724 """
725 self.write_config("""
726TMPDIR = \"${TOPDIR}/tmp-sstates-findsiginfo\"
727TCLIBCAPPEND = \"\"
728MACHINE = \"qemux86-64\"
729require conf/multilib.conf
730MULTILIBS = "multilib:lib32"
731DEFAULTTUNE:virtclass-multilib-lib32 = "x86"
732BB_SIGNATURE_HANDLER = "OEBasicHash"
733""")
734 self.track_for_cleanup(self.topdir + "/tmp-sstates-findsiginfo")
735
736 pns = ["binutils", "binutils-native", "lib32-binutils"]
737 target_configs = [
738"""
739TMPVAL1 = "tmpval1"
740TMPVAL2 = "tmpval2"
741do_tmptask1() {
742 echo ${TMPVAL1}
743}
744do_tmptask2() {
745 echo ${TMPVAL2}
746}
747addtask do_tmptask1
748addtask tmptask2 before do_tmptask1
749""",
750"""
751TMPVAL3 = "tmpval3"
752TMPVAL4 = "tmpval4"
753do_tmptask1() {
754 echo ${TMPVAL3}
755}
756do_tmptask2() {
757 echo ${TMPVAL4}
758}
759addtask do_tmptask1
760addtask tmptask2 before do_tmptask1
761"""
762 ]
763
764 for target_config in target_configs:
765 self.write_recipeinc("binutils", target_config)
766 for pn in pns:
767 bitbake("%s -c do_tmptask1 -S none" % pn)
768 self.delete_recipeinc("binutils")
769
770 with bb.tinfoil.Tinfoil() as tinfoil:
771 tinfoil.prepare(config_only=True)
772
773 def find_siginfo(pn, taskname, sigs=None):
774 result = None
775 command_complete = False
776 tinfoil.set_event_mask(["bb.event.FindSigInfoResult",
777 "bb.command.CommandCompleted"])
778 ret = tinfoil.run_command("findSigInfo", pn, taskname, sigs)
779 if ret:
780 while result is None or not command_complete:
781 event = tinfoil.wait_event(1)
782 if event:
783 if isinstance(event, bb.command.CommandCompleted):
784 command_complete = True
785 elif isinstance(event, bb.event.FindSigInfoResult):
786 result = event.result
787 return result
788
789 def recursecb(key, hash1, hash2):
790 nonlocal recursecb_count
791 recursecb_count += 1
792 hashes = [hash1, hash2]
793 hashfiles = find_siginfo(key, None, hashes)
794 self.assertCountEqual(hashes, hashfiles)
795 bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb)
796
797 for pn in pns:
798 recursecb_count = 0
799 matches = find_siginfo(pn, "do_tmptask1")
800 self.assertGreaterEqual(len(matches), 2)
801 latesthashes = sorted(matches.keys(), key=lambda h: matches[h]['time'])[-2:]
802 bb.siggen.compare_sigfiles(matches[latesthashes[-2]]['path'], matches[latesthashes[-1]]['path'], recursecb)
803 self.assertEqual(recursecb_count,1)
804
805class SStatePrintdiff(SStateBase):
806 def run_test_printdiff_changerecipe(self, target, change_recipe, change_bbtask, change_content, expected_sametmp_output, expected_difftmp_output):
807 import time
808 self.write_config("""
809TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}"
810""".format(time.time()))
811 # Use runall do_build to ensure any indirect sstate is created, e.g. tzcode-native on both x86 and
812 # aarch64 hosts since only allarch target recipes depend upon it and it may not be built otherwise.
813 # A bitbake -c cleansstate tzcode-native would cause some of these tests to error for example.
814 bitbake("--runall build --runall deploy_source_date_epoch {}".format(target))
815 bitbake("-S none {}".format(target))
816 bitbake(change_bbtask)
817 self.write_recipeinc(change_recipe, change_content)
818 result_sametmp = bitbake("-S printdiff {}".format(target))
819
820 self.write_config("""
821TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}"
822""".format(time.time()))
823 result_difftmp = bitbake("-S printdiff {}".format(target))
824
825 self.delete_recipeinc(change_recipe)
826 for item in expected_sametmp_output:
827 self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output))
828 for item in expected_difftmp_output:
829 self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output))
830
831 def run_test_printdiff_changeconfig(self, target, change_bbtasks, change_content, expected_sametmp_output, expected_difftmp_output):
832 import time
833 self.write_config("""
834TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}"
835""".format(time.time()))
836 bitbake("--runall build --runall deploy_source_date_epoch {}".format(target))
837 bitbake("-S none {}".format(target))
838 bitbake(" ".join(change_bbtasks))
839 self.append_config(change_content)
840 result_sametmp = bitbake("-S printdiff {}".format(target))
841
842 self.write_config("""
843TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}"
844""".format(time.time()))
845 self.append_config(change_content)
846 result_difftmp = bitbake("-S printdiff {}".format(target))
847
848 for item in expected_sametmp_output:
849 self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output))
850 for item in expected_difftmp_output:
851 self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output))
852
853
854 # Check if printdiff walks the full dependency chain from the image target to where the change is in a specific recipe
855 def test_image_minimal_vs_perlcross(self):
856 expected_output = ("Task perlcross-native:do_install couldn't be used from the cache because:",
857"We need hash",
858"most recent matching task was")
859 expected_sametmp_output = expected_output + (
860"Variable do_install value changed",
861'+ echo "this changes the task signature"')
862 expected_difftmp_output = expected_output
863
864 self.run_test_printdiff_changerecipe("core-image-minimal", "perlcross", "-c do_install perlcross-native",
865"""
866do_install:append() {
867 echo "this changes the task signature"
868}
869""",
870expected_sametmp_output, expected_difftmp_output)
871
872 # Check if changes to gcc-source (which uses tmp/work-shared) are correctly discovered
873 def test_gcc_runtime_vs_gcc_source(self):
874 gcc_source_pn = 'gcc-source-%s' % get_bb_vars(['PV'], 'gcc')['PV']
875
876 expected_output = ("Task {}:do_preconfigure couldn't be used from the cache because:".format(gcc_source_pn),
877"We need hash",
878"most recent matching task was")
879 expected_sametmp_output = expected_output + (
880"Variable do_preconfigure value changed",
881'+ print("this changes the task signature")')
882 expected_difftmp_output = expected_output
883
884 self.run_test_printdiff_changerecipe("gcc-runtime", "gcc-source", "-c do_preconfigure {}".format(gcc_source_pn),
885"""
886python do_preconfigure:append() {
887 print("this changes the task signature")
888}
889""",
890expected_sametmp_output, expected_difftmp_output)
891
892 # Check if changing a really base task definiton is reported against multiple core recipes using it
893 def test_image_minimal_vs_base_do_configure(self):
894 change_bbtasks = ('zstd-native:do_configure',
895'texinfo-dummy-native:do_configure',
896'ldconfig-native:do_configure',
897'gettext-minimal-native:do_configure',
898'tzcode-native:do_configure',
899'makedevs-native:do_configure',
900'pigz-native:do_configure',
901'update-rc.d-native:do_configure',
902'unzip-native:do_configure',
903'gnu-config-native:do_configure')
904
905 expected_output = ["Task {} couldn't be used from the cache because:".format(t) for t in change_bbtasks] + [
906"We need hash",
907"most recent matching task was"]
908
909 expected_sametmp_output = expected_output + [
910"Variable base_do_configure value changed",
911'+ echo "this changes base_do_configure() definiton "']
912 expected_difftmp_output = expected_output
913
914 self.run_test_printdiff_changeconfig("core-image-minimal",change_bbtasks,
915"""
916INHERIT += "base-do-configure-modified"
917""",
918expected_sametmp_output, expected_difftmp_output)
919
920@OETestTag("yocto-mirrors")
921class SStateMirrors(SStateBase):
922 def check_bb_output(self, output, exceptions, check_cdn):
923 def is_exception(object, exceptions):
924 for e in exceptions:
925 if re.search(e, object):
926 return True
927 return False
928
929 output_l = output.splitlines()
930 for l in output_l:
931 if l.startswith("Sstate summary"):
932 for idx, item in enumerate(l.split()):
933 if item == 'Missed':
934 missing_objects = int(l.split()[idx+1])
935 break
936 else:
937 self.fail("Did not find missing objects amount in sstate summary: {}".format(l))
938 break
939 else:
940 self.fail("Did not find 'Sstate summary' line in bitbake output")
941
942 failed_urls = []
943 failed_urls_extrainfo = []
944 for l in output_l:
945 if "SState: Unsuccessful fetch test for" in l and check_cdn:
946 missing_object = l.split()[6]
947 elif "SState: Looked for but didn't find file" in l and not check_cdn:
948 missing_object = l.split()[8]
949 else:
950 missing_object = None
951 if missing_object:
952 if not is_exception(missing_object, exceptions):
953 failed_urls.append(missing_object)
954 else:
955 missing_objects -= 1
956
957 if "urlopen failed for" in l and not is_exception(l, exceptions):
958 failed_urls_extrainfo.append(l)
959
960 self.assertEqual(len(failed_urls), missing_objects, "Amount of reported missing objects does not match failed URLs: {}\nFailed URLs:\n{}\nFetcher diagnostics:\n{}".format(missing_objects, "\n".join(failed_urls), "\n".join(failed_urls_extrainfo)))
961 self.assertEqual(len(failed_urls), 0, "Missing objects in the cache:\n{}\nFetcher diagnostics:\n{}".format("\n".join(failed_urls), "\n".join(failed_urls_extrainfo)))
962
963 def run_test(self, machine, targets, exceptions, check_cdn = True, ignore_errors = False):
964 # sstate is checked for existence of these, but they never get written out to begin with
965 exceptions += ["{}.*image_qa".format(t) for t in targets.split()]
966 exceptions += ["{}.*deploy_source_date_epoch".format(t) for t in targets.split()]
967 exceptions += ["{}.*image_complete".format(t) for t in targets.split()]
968 exceptions += ["linux-yocto.*shared_workdir"]
969 # these get influnced by IMAGE_FSTYPES tweaks in yocto-autobuilder-helper's config.json (on x86-64)
970 # additionally, they depend on noexec (thus, absent stamps) package, install, etc. image tasks,
971 # which makes tracing other changes difficult
972 exceptions += ["{}.*create_spdx".format(t) for t in targets.split()]
973 exceptions += ["{}.*create_runtime_spdx".format(t) for t in targets.split()]
974
975 if check_cdn:
976 self.config_sstate(True)
977 self.append_config("""
978MACHINE = "{}"
979BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687"
980SSTATE_MIRRORS ?= "file://.* http://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH"
981""".format(machine))
982 else:
983 self.append_config("""
984MACHINE = "{}"
985""".format(machine))
986 result = bitbake("-DD -n {}".format(targets))
987 bitbake("-S none {}".format(targets))
988 if ignore_errors:
989 return
990 self.check_bb_output(result.output, exceptions, check_cdn)
991
992 def test_cdn_mirror_qemux86_64(self):
993 exceptions = []
994 self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, ignore_errors = True)
995 self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions)
996
997 def test_cdn_mirror_qemuarm64(self):
998 exceptions = []
999 self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, ignore_errors = True)
1000 self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions)
1001
1002 def test_local_cache_qemux86_64(self):
1003 exceptions = []
1004 self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False)
1005
1006 def test_local_cache_qemuarm64(self):
1007 exceptions = []
1008 self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False)