diff options
Diffstat (limited to 'meta/lib/oeqa/selftest/cases/sstatetests.py')
-rw-r--r-- | meta/lib/oeqa/selftest/cases/sstatetests.py | 711 |
1 files changed, 572 insertions, 139 deletions
diff --git a/meta/lib/oeqa/selftest/cases/sstatetests.py b/meta/lib/oeqa/selftest/cases/sstatetests.py index c46e8ba489..86d6cd7464 100644 --- a/meta/lib/oeqa/selftest/cases/sstatetests.py +++ b/meta/lib/oeqa/selftest/cases/sstatetests.py | |||
@@ -1,4 +1,6 @@ | |||
1 | # | 1 | # |
2 | # Copyright OpenEmbedded Contributors | ||
3 | # | ||
2 | # SPDX-License-Identifier: MIT | 4 | # SPDX-License-Identifier: MIT |
3 | # | 5 | # |
4 | 6 | ||
@@ -7,54 +9,77 @@ import shutil | |||
7 | import glob | 9 | import glob |
8 | import subprocess | 10 | import subprocess |
9 | import tempfile | 11 | import tempfile |
12 | import datetime | ||
13 | import re | ||
10 | 14 | ||
15 | from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer, get_bb_vars | ||
11 | from oeqa.selftest.case import OESelftestTestCase | 16 | from oeqa.selftest.case import OESelftestTestCase |
12 | from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer, create_temp_layer | 17 | from oeqa.core.decorator import OETestTag |
13 | from oeqa.selftest.cases.sstate import SStateBase | ||
14 | 18 | ||
19 | import oe | ||
15 | import bb.siggen | 20 | import bb.siggen |
16 | 21 | ||
17 | class SStateTests(SStateBase): | 22 | # Set to True to preserve stamp files after test execution for debugging failures |
18 | def test_autorev_sstate_works(self): | 23 | keep_temp_files = False |
19 | # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} | 24 | |
20 | # when PV does not contain SRCPV | 25 | class SStateBase(OESelftestTestCase): |
21 | 26 | ||
22 | tempdir = tempfile.mkdtemp(prefix='sstate_autorev') | 27 | def setUpLocal(self): |
23 | tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') | 28 | super(SStateBase, self).setUpLocal() |
24 | self.track_for_cleanup(tempdir) | 29 | self.temp_sstate_location = None |
25 | self.track_for_cleanup(tempdldir) | 30 | needed_vars = ['SSTATE_DIR', 'NATIVELSBSTRING', 'TCLIBC', 'TUNE_ARCH', |
26 | create_temp_layer(tempdir, 'selftestrecipetool') | 31 | 'TOPDIR', 'TARGET_VENDOR', 'TARGET_OS'] |
27 | self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) | 32 | bb_vars = get_bb_vars(needed_vars) |
28 | self.append_config("DL_DIR = \"%s\"" % tempdldir) | 33 | self.sstate_path = bb_vars['SSTATE_DIR'] |
29 | runCmd('bitbake-layers add-layer %s' % tempdir) | 34 | self.hostdistro = bb_vars['NATIVELSBSTRING'] |
30 | 35 | self.tclibc = bb_vars['TCLIBC'] | |
31 | # Use dbus-wait as a local git repo we can add a commit between two builds in | 36 | self.tune_arch = bb_vars['TUNE_ARCH'] |
32 | pn = 'dbus-wait' | 37 | self.topdir = bb_vars['TOPDIR'] |
33 | srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' | 38 | self.target_vendor = bb_vars['TARGET_VENDOR'] |
34 | url = 'git://git.yoctoproject.org/dbus-wait' | 39 | self.target_os = bb_vars['TARGET_OS'] |
35 | result = runCmd('git clone %s noname' % url, cwd=tempdir) | 40 | self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) |
36 | srcdir = os.path.join(tempdir, 'noname') | 41 | |
37 | result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) | 42 | def track_for_cleanup(self, path): |
38 | self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') | 43 | if not keep_temp_files: |
39 | 44 | super().track_for_cleanup(path) | |
40 | recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb') | 45 | |
41 | os.makedirs(os.path.dirname(recipefile)) | 46 | # Creates a special sstate configuration with the option to add sstate mirrors |
42 | srcuri = 'git://' + srcdir + ';protocol=file' | 47 | def config_sstate(self, temp_sstate_location=False, add_local_mirrors=[]): |
43 | result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri]) | 48 | self.temp_sstate_location = temp_sstate_location |
44 | self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) | 49 | |
45 | 50 | if self.temp_sstate_location: | |
46 | with open(recipefile, 'a') as f: | 51 | temp_sstate_path = os.path.join(self.builddir, "temp_sstate_%s" % datetime.datetime.now().strftime('%Y%m%d%H%M%S')) |
47 | f.write('SRCREV = "${AUTOREV}"\n') | 52 | config_temp_sstate = "SSTATE_DIR = \"%s\"" % temp_sstate_path |
48 | f.write('PV = "1.0"\n') | 53 | self.append_config(config_temp_sstate) |
49 | 54 | self.track_for_cleanup(temp_sstate_path) | |
50 | bitbake("dbus-wait-test -c fetch") | 55 | bb_vars = get_bb_vars(['SSTATE_DIR', 'NATIVELSBSTRING']) |
51 | with open(os.path.join(srcdir, "bar.txt"), "w") as f: | 56 | self.sstate_path = bb_vars['SSTATE_DIR'] |
52 | f.write("foo") | 57 | self.hostdistro = bb_vars['NATIVELSBSTRING'] |
53 | result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir) | 58 | self.distro_specific_sstate = os.path.join(self.sstate_path, self.hostdistro) |
54 | bitbake("dbus-wait-test -c unpack") | 59 | |
55 | 60 | if add_local_mirrors: | |
56 | 61 | config_set_sstate_if_not_set = 'SSTATE_MIRRORS ?= ""' | |
57 | # Test sstate files creation and their location | 62 | self.append_config(config_set_sstate_if_not_set) |
63 | for local_mirror in add_local_mirrors: | ||
64 | self.assertFalse(os.path.join(local_mirror) == os.path.join(self.sstate_path), msg='Cannot add the current sstate path as a sstate mirror') | ||
65 | config_sstate_mirror = "SSTATE_MIRRORS += \"file://.* file:///%s/PATH\"" % local_mirror | ||
66 | self.append_config(config_sstate_mirror) | ||
67 | |||
68 | # Returns a list containing sstate files | ||
69 | def search_sstate(self, filename_regex, distro_specific=True, distro_nonspecific=True): | ||
70 | result = [] | ||
71 | for root, dirs, files in os.walk(self.sstate_path): | ||
72 | if distro_specific and re.search(r"%s/%s/[a-z0-9]{2}/[a-z0-9]{2}$" % (self.sstate_path, self.hostdistro), root): | ||
73 | for f in files: | ||
74 | if re.search(filename_regex, f): | ||
75 | result.append(f) | ||
76 | if distro_nonspecific and re.search(r"%s/[a-z0-9]{2}/[a-z0-9]{2}$" % self.sstate_path, root): | ||
77 | for f in files: | ||
78 | if re.search(filename_regex, f): | ||
79 | result.append(f) | ||
80 | return result | ||
81 | |||
82 | # Test sstate files creation and their location and directory perms | ||
58 | def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): | 83 | def run_test_sstate_creation(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True, should_pass=True): |
59 | self.config_sstate(temp_sstate_location, [self.sstate_path]) | 84 | self.config_sstate(temp_sstate_location, [self.sstate_path]) |
60 | 85 | ||
@@ -63,12 +88,25 @@ class SStateTests(SStateBase): | |||
63 | else: | 88 | else: |
64 | bitbake(['-ccleansstate'] + targets) | 89 | bitbake(['-ccleansstate'] + targets) |
65 | 90 | ||
91 | # We need to test that the env umask have does not effect sstate directory creation | ||
92 | # So, first, we'll get the current umask and set it to something we know incorrect | ||
93 | # See: sstate_task_postfunc for correct umask of os.umask(0o002) | ||
94 | import os | ||
95 | def current_umask(): | ||
96 | current_umask = os.umask(0) | ||
97 | os.umask(current_umask) | ||
98 | return current_umask | ||
99 | |||
100 | orig_umask = current_umask() | ||
101 | # Set it to a umask we know will be 'wrong' | ||
102 | os.umask(0o022) | ||
103 | |||
66 | bitbake(targets) | 104 | bitbake(targets) |
67 | file_tracker = [] | 105 | file_tracker = [] |
68 | results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) | 106 | results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) |
69 | if distro_nonspecific: | 107 | if distro_nonspecific: |
70 | for r in results: | 108 | for r in results: |
71 | if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo", "_fetch.tgz.siginfo", "_unpack.tgz.siginfo", "_patch.tgz.siginfo")): | 109 | if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo", "_fetch.tar.zst.siginfo", "_unpack.tar.zst.siginfo", "_patch.tar.zst.siginfo")): |
72 | continue | 110 | continue |
73 | file_tracker.append(r) | 111 | file_tracker.append(r) |
74 | else: | 112 | else: |
@@ -79,17 +117,18 @@ class SStateTests(SStateBase): | |||
79 | else: | 117 | else: |
80 | self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) | 118 | self.assertTrue(not file_tracker , msg="Found sstate files in the wrong place for: %s (found %s)" % (', '.join(map(str, targets)), str(file_tracker))) |
81 | 119 | ||
82 | def test_sstate_creation_distro_specific_pass(self): | 120 | # Now we'll walk the tree to check the mode and see if things are incorrect. |
83 | self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) | 121 | badperms = [] |
84 | 122 | for root, dirs, files in os.walk(self.sstate_path): | |
85 | def test_sstate_creation_distro_specific_fail(self): | 123 | for directory in dirs: |
86 | self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) | 124 | if (os.stat(os.path.join(root, directory)).st_mode & 0o777) != 0o775: |
125 | badperms.append(os.path.join(root, directory)) | ||
87 | 126 | ||
88 | def test_sstate_creation_distro_nonspecific_pass(self): | 127 | # Return to original umask |
89 | self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) | 128 | os.umask(orig_umask) |
90 | 129 | ||
91 | def test_sstate_creation_distro_nonspecific_fail(self): | 130 | if should_pass: |
92 | self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) | 131 | self.assertTrue(badperms , msg="Found sstate directories with the wrong permissions: %s (found %s)" % (', '.join(map(str, targets)), str(badperms))) |
93 | 132 | ||
94 | # Test the sstate files deletion part of the do_cleansstate task | 133 | # Test the sstate files deletion part of the do_cleansstate task |
95 | def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): | 134 | def run_test_cleansstate_task(self, targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True): |
@@ -98,29 +137,15 @@ class SStateTests(SStateBase): | |||
98 | bitbake(['-ccleansstate'] + targets) | 137 | bitbake(['-ccleansstate'] + targets) |
99 | 138 | ||
100 | bitbake(targets) | 139 | bitbake(targets) |
101 | tgz_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) | 140 | archives_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) |
102 | self.assertTrue(tgz_created, msg="Could not find sstate .tgz files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_created))) | 141 | self.assertTrue(archives_created, msg="Could not find sstate .tar.zst files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_created))) |
103 | 142 | ||
104 | siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) | 143 | siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) |
105 | self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) | 144 | self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) |
106 | 145 | ||
107 | bitbake(['-ccleansstate'] + targets) | 146 | bitbake(['-ccleansstate'] + targets) |
108 | tgz_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) | 147 | archives_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific) |
109 | self.assertTrue(not tgz_removed, msg="do_cleansstate didn't remove .tgz sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_removed))) | 148 | self.assertTrue(not archives_removed, msg="do_cleansstate didn't remove .tar.zst sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_removed))) |
110 | |||
111 | def test_cleansstate_task_distro_specific_nonspecific(self): | ||
112 | targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] | ||
113 | targets.append('linux-libc-headers') | ||
114 | self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) | ||
115 | |||
116 | def test_cleansstate_task_distro_nonspecific(self): | ||
117 | self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) | ||
118 | |||
119 | def test_cleansstate_task_distro_specific(self): | ||
120 | targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] | ||
121 | targets.append('linux-libc-headers') | ||
122 | self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) | ||
123 | |||
124 | 149 | ||
125 | # Test rebuilding of distro-specific sstate files | 150 | # Test rebuilding of distro-specific sstate files |
126 | def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): | 151 | def run_test_rebuild_distro_specific_sstate(self, targets, temp_sstate_location=True): |
@@ -129,15 +154,15 @@ class SStateTests(SStateBase): | |||
129 | bitbake(['-ccleansstate'] + targets) | 154 | bitbake(['-ccleansstate'] + targets) |
130 | 155 | ||
131 | bitbake(targets) | 156 | bitbake(targets) |
132 | results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=False, distro_nonspecific=True) | 157 | results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=False, distro_nonspecific=True) |
133 | filtered_results = [] | 158 | filtered_results = [] |
134 | for r in results: | 159 | for r in results: |
135 | if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo")): | 160 | if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo")): |
136 | continue | 161 | continue |
137 | filtered_results.append(r) | 162 | filtered_results.append(r) |
138 | self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) | 163 | self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) |
139 | file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=True, distro_nonspecific=False) | 164 | file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False) |
140 | self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files ware created for: %s" % ', '.join(map(str, targets))) | 165 | self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) |
141 | 166 | ||
142 | self.track_for_cleanup(self.distro_specific_sstate + "_old") | 167 | self.track_for_cleanup(self.distro_specific_sstate + "_old") |
143 | shutil.copytree(self.distro_specific_sstate, self.distro_specific_sstate + "_old") | 168 | shutil.copytree(self.distro_specific_sstate, self.distro_specific_sstate + "_old") |
@@ -145,15 +170,114 @@ class SStateTests(SStateBase): | |||
145 | 170 | ||
146 | bitbake(['-cclean'] + targets) | 171 | bitbake(['-cclean'] + targets) |
147 | bitbake(targets) | 172 | bitbake(targets) |
148 | file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=True, distro_nonspecific=False) | 173 | file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False) |
149 | self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files ware created for: %s" % ', '.join(map(str, targets))) | 174 | self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) |
150 | 175 | ||
151 | not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2] | 176 | not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2] |
152 | self.assertTrue(not_recreated == [], msg="The following sstate files ware not recreated: %s" % ', '.join(map(str, not_recreated))) | 177 | self.assertTrue(not_recreated == [], msg="The following sstate files were not recreated: %s" % ', '.join(map(str, not_recreated))) |
153 | 178 | ||
154 | created_once = [x for x in file_tracker_2 if x not in file_tracker_1] | 179 | created_once = [x for x in file_tracker_2 if x not in file_tracker_1] |
155 | self.assertTrue(created_once == [], msg="The following sstate files ware created only in the second run: %s" % ', '.join(map(str, created_once))) | 180 | self.assertTrue(created_once == [], msg="The following sstate files were created only in the second run: %s" % ', '.join(map(str, created_once))) |
181 | |||
182 | def sstate_common_samesigs(self, configA, configB, allarch=False): | ||
183 | |||
184 | self.write_config(configA) | ||
185 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") | ||
186 | bitbake("world meta-toolchain -S none") | ||
187 | self.write_config(configB) | ||
188 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") | ||
189 | bitbake("world meta-toolchain -S none") | ||
190 | |||
191 | def get_files(d, result): | ||
192 | for root, dirs, files in os.walk(d): | ||
193 | for name in files: | ||
194 | if "meta-environment" in root or "cross-canadian" in root: | ||
195 | continue | ||
196 | if "do_build" not in name: | ||
197 | # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 | ||
198 | (_, task, _, shash) = name.rsplit(".", 3) | ||
199 | result[os.path.join(os.path.basename(root), task)] = shash | ||
200 | |||
201 | files1 = {} | ||
202 | files2 = {} | ||
203 | subdirs = sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")) | ||
204 | if allarch: | ||
205 | subdirs.extend(sorted(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/all-*-linux"))) | ||
206 | |||
207 | for subdir in subdirs: | ||
208 | nativesdkdir = os.path.basename(subdir) | ||
209 | get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir, files1) | ||
210 | get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir, files2) | ||
211 | |||
212 | self.maxDiff = None | ||
213 | self.assertEqual(files1, files2) | ||
214 | |||
215 | class SStateTests(SStateBase): | ||
216 | def test_autorev_sstate_works(self): | ||
217 | # Test that a git repository which changes is correctly handled by SRCREV = ${AUTOREV} | ||
156 | 218 | ||
219 | tempdir = tempfile.mkdtemp(prefix='sstate_autorev') | ||
220 | tempdldir = tempfile.mkdtemp(prefix='sstate_autorev_dldir') | ||
221 | self.track_for_cleanup(tempdir) | ||
222 | self.track_for_cleanup(tempdldir) | ||
223 | create_temp_layer(tempdir, 'selftestrecipetool') | ||
224 | self.add_command_to_tearDown('bitbake-layers remove-layer %s' % tempdir) | ||
225 | self.append_config("DL_DIR = \"%s\"" % tempdldir) | ||
226 | runCmd('bitbake-layers add-layer %s' % tempdir) | ||
227 | |||
228 | # Use dbus-wait as a local git repo we can add a commit between two builds in | ||
229 | pn = 'dbus-wait' | ||
230 | srcrev = '6cc6077a36fe2648a5f993fe7c16c9632f946517' | ||
231 | url = 'git://git.yoctoproject.org/dbus-wait' | ||
232 | result = runCmd('git clone %s noname' % url, cwd=tempdir) | ||
233 | srcdir = os.path.join(tempdir, 'noname') | ||
234 | result = runCmd('git reset --hard %s' % srcrev, cwd=srcdir) | ||
235 | self.assertTrue(os.path.isfile(os.path.join(srcdir, 'configure.ac')), 'Unable to find configure script in source directory') | ||
236 | |||
237 | recipefile = os.path.join(tempdir, "recipes-test", "dbus-wait-test", 'dbus-wait-test_git.bb') | ||
238 | os.makedirs(os.path.dirname(recipefile)) | ||
239 | srcuri = 'git://' + srcdir + ';protocol=file;branch=master' | ||
240 | result = runCmd(['recipetool', 'create', '-o', recipefile, srcuri]) | ||
241 | self.assertTrue(os.path.isfile(recipefile), 'recipetool did not create recipe file; output:\n%s' % result.output) | ||
242 | |||
243 | with open(recipefile, 'a') as f: | ||
244 | f.write('SRCREV = "${AUTOREV}"\n') | ||
245 | f.write('PV = "1.0"\n') | ||
246 | |||
247 | bitbake("dbus-wait-test -c fetch") | ||
248 | with open(os.path.join(srcdir, "bar.txt"), "w") as f: | ||
249 | f.write("foo") | ||
250 | result = runCmd('git add bar.txt; git commit -asm "add bar"', cwd=srcdir) | ||
251 | bitbake("dbus-wait-test -c unpack") | ||
252 | |||
253 | class SStateCreation(SStateBase): | ||
254 | def test_sstate_creation_distro_specific_pass(self): | ||
255 | self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) | ||
256 | |||
257 | def test_sstate_creation_distro_specific_fail(self): | ||
258 | self.run_test_sstate_creation(['binutils-cross-'+ self.tune_arch, 'binutils-native'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True, should_pass=False) | ||
259 | |||
260 | def test_sstate_creation_distro_nonspecific_pass(self): | ||
261 | self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) | ||
262 | |||
263 | def test_sstate_creation_distro_nonspecific_fail(self): | ||
264 | self.run_test_sstate_creation(['linux-libc-headers'], distro_specific=True, distro_nonspecific=False, temp_sstate_location=True, should_pass=False) | ||
265 | |||
266 | class SStateCleanup(SStateBase): | ||
267 | def test_cleansstate_task_distro_specific_nonspecific(self): | ||
268 | targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] | ||
269 | targets.append('linux-libc-headers') | ||
270 | self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=True, temp_sstate_location=True) | ||
271 | |||
272 | def test_cleansstate_task_distro_nonspecific(self): | ||
273 | self.run_test_cleansstate_task(['linux-libc-headers'], distro_specific=False, distro_nonspecific=True, temp_sstate_location=True) | ||
274 | |||
275 | def test_cleansstate_task_distro_specific(self): | ||
276 | targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] | ||
277 | targets.append('linux-libc-headers') | ||
278 | self.run_test_cleansstate_task(targets, distro_specific=True, distro_nonspecific=False, temp_sstate_location=True) | ||
279 | |||
280 | class SStateDistroTests(SStateBase): | ||
157 | def test_rebuild_distro_specific_sstate_cross_native_targets(self): | 281 | def test_rebuild_distro_specific_sstate_cross_native_targets(self): |
158 | self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) | 282 | self.run_test_rebuild_distro_specific_sstate(['binutils-cross-' + self.tune_arch, 'binutils-native'], temp_sstate_location=True) |
159 | 283 | ||
@@ -163,48 +287,48 @@ class SStateTests(SStateBase): | |||
163 | def test_rebuild_distro_specific_sstate_native_target(self): | 287 | def test_rebuild_distro_specific_sstate_native_target(self): |
164 | self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) | 288 | self.run_test_rebuild_distro_specific_sstate(['binutils-native'], temp_sstate_location=True) |
165 | 289 | ||
166 | 290 | class SStateCacheManagement(SStateBase): | |
167 | # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list | 291 | # Test the sstate-cache-management script. Each element in the global_config list is used with the corresponding element in the target_config list |
168 | # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.sh (such as changing the value of MACHINE) | 292 | # global_config elements are expected to not generate any sstate files that would be removed by sstate-cache-management.py (such as changing the value of MACHINE) |
169 | def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]): | 293 | def run_test_sstate_cache_management_script(self, target, global_config=[''], target_config=[''], ignore_patterns=[]): |
170 | self.assertTrue(global_config) | 294 | self.assertTrue(global_config) |
171 | self.assertTrue(target_config) | 295 | self.assertTrue(target_config) |
172 | self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') | 296 | self.assertTrue(len(global_config) == len(target_config), msg='Lists global_config and target_config should have the same number of elements') |
173 | self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) | ||
174 | 297 | ||
175 | # If buildhistory is enabled, we need to disable version-going-backwards | 298 | for idx in range(len(target_config)): |
176 | # QA checks for this test. It may report errors otherwise. | 299 | self.append_config(global_config[idx]) |
177 | self.append_config('ERROR_QA_remove = "version-going-backwards"') | 300 | self.append_recipeinc(target, target_config[idx]) |
301 | bitbake(target) | ||
302 | self.remove_config(global_config[idx]) | ||
303 | self.remove_recipeinc(target, target_config[idx]) | ||
304 | |||
305 | self.config_sstate(temp_sstate_location=True, add_local_mirrors=[self.sstate_path]) | ||
178 | 306 | ||
179 | # For not this only checks if random sstate tasks are handled correctly as a group. | 307 | # For now this only checks if random sstate tasks are handled correctly as a group. |
180 | # In the future we should add control over what tasks we check for. | 308 | # In the future we should add control over what tasks we check for. |
181 | 309 | ||
182 | sstate_archs_list = [] | ||
183 | expected_remaining_sstate = [] | 310 | expected_remaining_sstate = [] |
184 | for idx in range(len(target_config)): | 311 | for idx in range(len(target_config)): |
185 | self.append_config(global_config[idx]) | 312 | self.append_config(global_config[idx]) |
186 | self.append_recipeinc(target, target_config[idx]) | 313 | self.append_recipeinc(target, target_config[idx]) |
187 | sstate_arch = get_bb_var('SSTATE_PKGARCH', target) | ||
188 | if not sstate_arch in sstate_archs_list: | ||
189 | sstate_archs_list.append(sstate_arch) | ||
190 | if target_config[idx] == target_config[-1]: | 314 | if target_config[idx] == target_config[-1]: |
191 | target_sstate_before_build = self.search_sstate(target + r'.*?\.tgz$') | 315 | target_sstate_before_build = self.search_sstate(target + r'.*?\.tar.zst$') |
192 | bitbake("-cclean %s" % target) | 316 | bitbake("-cclean %s" % target) |
193 | result = bitbake(target, ignore_status=True) | 317 | result = bitbake(target, ignore_status=True) |
194 | if target_config[idx] == target_config[-1]: | 318 | if target_config[idx] == target_config[-1]: |
195 | target_sstate_after_build = self.search_sstate(target + r'.*?\.tgz$') | 319 | target_sstate_after_build = self.search_sstate(target + r'.*?\.tar.zst$') |
196 | expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)] | 320 | expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)] |
197 | self.remove_config(global_config[idx]) | 321 | self.remove_config(global_config[idx]) |
198 | self.remove_recipeinc(target, target_config[idx]) | 322 | self.remove_recipeinc(target, target_config[idx]) |
199 | self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) | 323 | self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) |
200 | 324 | ||
201 | runCmd("sstate-cache-management.sh -y --cache-dir=%s --remove-duplicated --extra-archs=%s" % (self.sstate_path, ','.join(map(str, sstate_archs_list)))) | 325 | runCmd("sstate-cache-management.py -y --cache-dir=%s --remove-duplicated" % (self.sstate_path)) |
202 | actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tgz$') if not any(pattern in x for pattern in ignore_patterns)] | 326 | actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tar.zst$') if not any(pattern in x for pattern in ignore_patterns)] |
203 | 327 | ||
204 | actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate] | 328 | actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate] |
205 | self.assertFalse(actual_not_expected, msg="Files should have been removed but ware not: %s" % ', '.join(map(str, actual_not_expected))) | 329 | self.assertFalse(actual_not_expected, msg="Files should have been removed but were not: %s" % ', '.join(map(str, actual_not_expected))) |
206 | expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate] | 330 | expected_not_actual = [x for x in expected_remaining_sstate if x not in actual_remaining_sstate] |
207 | self.assertFalse(expected_not_actual, msg="Extra files ware removed: %s" ', '.join(map(str, expected_not_actual))) | 331 | self.assertFalse(expected_not_actual, msg="Extra files were removed: %s" ', '.join(map(str, expected_not_actual))) |
208 | 332 | ||
209 | def test_sstate_cache_management_script_using_pr_1(self): | 333 | def test_sstate_cache_management_script_using_pr_1(self): |
210 | global_config = [] | 334 | global_config = [] |
@@ -242,6 +366,7 @@ class SStateTests(SStateBase): | |||
242 | target_config.append('') | 366 | target_config.append('') |
243 | self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) | 367 | self.run_test_sstate_cache_management_script('m4', global_config, target_config, ignore_patterns=['populate_lic']) |
244 | 368 | ||
369 | class SStateHashSameSigs(SStateBase): | ||
245 | def test_sstate_32_64_same_hash(self): | 370 | def test_sstate_32_64_same_hash(self): |
246 | """ | 371 | """ |
247 | The sstate checksums for both native and target should not vary whether | 372 | The sstate checksums for both native and target should not vary whether |
@@ -261,7 +386,7 @@ PACKAGE_CLASSES = "package_rpm package_ipk package_deb" | |||
261 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 386 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
262 | """) | 387 | """) |
263 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") | 388 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") |
264 | bitbake("core-image-sato -S none") | 389 | bitbake("core-image-weston -S none") |
265 | self.write_config(""" | 390 | self.write_config(""" |
266 | MACHINE = "qemux86" | 391 | MACHINE = "qemux86" |
267 | TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" | 392 | TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" |
@@ -273,12 +398,12 @@ PACKAGE_CLASSES = "package_rpm package_ipk package_deb" | |||
273 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 398 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
274 | """) | 399 | """) |
275 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") | 400 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") |
276 | bitbake("core-image-sato -S none") | 401 | bitbake("core-image-weston -S none") |
277 | 402 | ||
278 | def get_files(d): | 403 | def get_files(d): |
279 | f = [] | 404 | f = [] |
280 | for root, dirs, files in os.walk(d): | 405 | for root, dirs, files in os.walk(d): |
281 | if "core-image-sato" in root: | 406 | if "core-image-weston" in root: |
282 | # SDKMACHINE changing will change | 407 | # SDKMACHINE changing will change |
283 | # do_rootfs/do_testimage/do_build stamps of images which | 408 | # do_rootfs/do_testimage/do_build stamps of images which |
284 | # is safe to ignore. | 409 | # is safe to ignore. |
@@ -306,7 +431,7 @@ NATIVELSBSTRING = \"DistroA\" | |||
306 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 431 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
307 | """) | 432 | """) |
308 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") | 433 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") |
309 | bitbake("core-image-sato -S none") | 434 | bitbake("core-image-weston -S none") |
310 | self.write_config(""" | 435 | self.write_config(""" |
311 | TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" | 436 | TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" |
312 | TCLIBCAPPEND = \"\" | 437 | TCLIBCAPPEND = \"\" |
@@ -314,7 +439,7 @@ NATIVELSBSTRING = \"DistroB\" | |||
314 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 439 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
315 | """) | 440 | """) |
316 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") | 441 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") |
317 | bitbake("core-image-sato -S none") | 442 | bitbake("core-image-weston -S none") |
318 | 443 | ||
319 | def get_files(d): | 444 | def get_files(d): |
320 | f = [] | 445 | f = [] |
@@ -327,6 +452,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" | |||
327 | self.maxDiff = None | 452 | self.maxDiff = None |
328 | self.assertCountEqual(files1, files2) | 453 | self.assertCountEqual(files1, files2) |
329 | 454 | ||
455 | class SStateHashSameSigs2(SStateBase): | ||
330 | def test_sstate_allarch_samesigs(self): | 456 | def test_sstate_allarch_samesigs(self): |
331 | """ | 457 | """ |
332 | The sstate checksums of allarch packages should be independent of whichever | 458 | The sstate checksums of allarch packages should be independent of whichever |
@@ -341,13 +467,15 @@ TCLIBCAPPEND = \"\" | |||
341 | MACHINE = \"qemux86-64\" | 467 | MACHINE = \"qemux86-64\" |
342 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 468 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
343 | """ | 469 | """ |
470 | #OLDEST_KERNEL is arch specific so set to a different value here for testing | ||
344 | configB = """ | 471 | configB = """ |
345 | TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" | 472 | TMPDIR = \"${TOPDIR}/tmp-sstatesamehash2\" |
346 | TCLIBCAPPEND = \"\" | 473 | TCLIBCAPPEND = \"\" |
347 | MACHINE = \"qemuarm\" | 474 | MACHINE = \"qemuarm\" |
475 | OLDEST_KERNEL = \"3.3.0\" | ||
348 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 476 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
349 | """ | 477 | """ |
350 | self.sstate_allarch_samesigs(configA, configB) | 478 | self.sstate_common_samesigs(configA, configB, allarch=True) |
351 | 479 | ||
352 | def test_sstate_nativesdk_samesigs_multilib(self): | 480 | def test_sstate_nativesdk_samesigs_multilib(self): |
353 | """ | 481 | """ |
@@ -360,7 +488,7 @@ TCLIBCAPPEND = \"\" | |||
360 | MACHINE = \"qemux86-64\" | 488 | MACHINE = \"qemux86-64\" |
361 | require conf/multilib.conf | 489 | require conf/multilib.conf |
362 | MULTILIBS = \"multilib:lib32\" | 490 | MULTILIBS = \"multilib:lib32\" |
363 | DEFAULTTUNE_virtclass-multilib-lib32 = \"x86\" | 491 | DEFAULTTUNE:virtclass-multilib-lib32 = \"x86\" |
364 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 492 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
365 | """ | 493 | """ |
366 | configB = """ | 494 | configB = """ |
@@ -371,36 +499,9 @@ require conf/multilib.conf | |||
371 | MULTILIBS = \"\" | 499 | MULTILIBS = \"\" |
372 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 500 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
373 | """ | 501 | """ |
374 | self.sstate_allarch_samesigs(configA, configB) | 502 | self.sstate_common_samesigs(configA, configB) |
375 | |||
376 | def sstate_allarch_samesigs(self, configA, configB): | ||
377 | |||
378 | self.write_config(configA) | ||
379 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") | ||
380 | bitbake("world meta-toolchain -S none") | ||
381 | self.write_config(configB) | ||
382 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") | ||
383 | bitbake("world meta-toolchain -S none") | ||
384 | |||
385 | def get_files(d): | ||
386 | f = {} | ||
387 | for root, dirs, files in os.walk(d): | ||
388 | for name in files: | ||
389 | if "meta-environment" in root or "cross-canadian" in root: | ||
390 | continue | ||
391 | if "do_build" not in name: | ||
392 | # 1.4.1+gitAUTOINC+302fca9f4c-r0.do_package_write_ipk.sigdata.f3a2a38697da743f0dbed8b56aafcf79 | ||
393 | (_, task, _, shash) = name.rsplit(".", 3) | ||
394 | f[os.path.join(os.path.basename(root), task)] = shash | ||
395 | return f | ||
396 | |||
397 | nativesdkdir = os.path.basename(glob.glob(self.topdir + "/tmp-sstatesamehash/stamps/*-nativesdk*-linux")[0]) | ||
398 | |||
399 | files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps/" + nativesdkdir) | ||
400 | files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps/" + nativesdkdir) | ||
401 | self.maxDiff = None | ||
402 | self.assertEqual(files1, files2) | ||
403 | 503 | ||
504 | class SStateHashSameSigs3(SStateBase): | ||
404 | def test_sstate_sametune_samesigs(self): | 505 | def test_sstate_sametune_samesigs(self): |
405 | """ | 506 | """ |
406 | The sstate checksums of two identical machines (using the same tune) should be the | 507 | The sstate checksums of two identical machines (using the same tune) should be the |
@@ -414,7 +515,7 @@ TCLIBCAPPEND = \"\" | |||
414 | MACHINE = \"qemux86\" | 515 | MACHINE = \"qemux86\" |
415 | require conf/multilib.conf | 516 | require conf/multilib.conf |
416 | MULTILIBS = "multilib:lib32" | 517 | MULTILIBS = "multilib:lib32" |
417 | DEFAULTTUNE_virtclass-multilib-lib32 = "x86" | 518 | DEFAULTTUNE:virtclass-multilib-lib32 = "x86" |
418 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 519 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
419 | """) | 520 | """) |
420 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") | 521 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") |
@@ -425,7 +526,7 @@ TCLIBCAPPEND = \"\" | |||
425 | MACHINE = \"qemux86copy\" | 526 | MACHINE = \"qemux86copy\" |
426 | require conf/multilib.conf | 527 | require conf/multilib.conf |
427 | MULTILIBS = "multilib:lib32" | 528 | MULTILIBS = "multilib:lib32" |
428 | DEFAULTTUNE_virtclass-multilib-lib32 = "x86" | 529 | DEFAULTTUNE:virtclass-multilib-lib32 = "x86" |
429 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 530 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
430 | """) | 531 | """) |
431 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") | 532 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") |
@@ -435,7 +536,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" | |||
435 | f = [] | 536 | f = [] |
436 | for root, dirs, files in os.walk(d): | 537 | for root, dirs, files in os.walk(d): |
437 | for name in files: | 538 | for name in files: |
438 | if "meta-environment" in root or "cross-canadian" in root: | 539 | if "meta-environment" in root or "cross-canadian" in root or 'meta-ide-support' in root: |
439 | continue | 540 | continue |
440 | if "qemux86copy-" in root or "qemux86-" in root: | 541 | if "qemux86copy-" in root or "qemux86-" in root: |
441 | continue | 542 | continue |
@@ -462,7 +563,7 @@ TCLIBCAPPEND = \"\" | |||
462 | MACHINE = \"qemux86\" | 563 | MACHINE = \"qemux86\" |
463 | require conf/multilib.conf | 564 | require conf/multilib.conf |
464 | MULTILIBS = "multilib:lib32" | 565 | MULTILIBS = "multilib:lib32" |
465 | DEFAULTTUNE_virtclass-multilib-lib32 = "x86" | 566 | DEFAULTTUNE:virtclass-multilib-lib32 = "x86" |
466 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 567 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
467 | """) | 568 | """) |
468 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") | 569 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") |
@@ -488,7 +589,7 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" | |||
488 | self.maxDiff = None | 589 | self.maxDiff = None |
489 | self.assertCountEqual(files1, files2) | 590 | self.assertCountEqual(files1, files2) |
490 | 591 | ||
491 | 592 | class SStateHashSameSigs4(SStateBase): | |
492 | def test_sstate_noop_samesigs(self): | 593 | def test_sstate_noop_samesigs(self): |
493 | """ | 594 | """ |
494 | The sstate checksums of two builds with these variables changed or | 595 | The sstate checksums of two builds with these variables changed or |
@@ -503,7 +604,7 @@ PARALLEL_MAKE = "-j 1" | |||
503 | DL_DIR = "${TOPDIR}/download1" | 604 | DL_DIR = "${TOPDIR}/download1" |
504 | TIME = "111111" | 605 | TIME = "111111" |
505 | DATE = "20161111" | 606 | DATE = "20161111" |
506 | INHERIT_remove = "buildstats-summary buildhistory uninative" | 607 | INHERIT:remove = "buildstats-summary buildhistory uninative" |
507 | http_proxy = "" | 608 | http_proxy = "" |
508 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 609 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
509 | """) | 610 | """) |
@@ -519,7 +620,7 @@ DL_DIR = "${TOPDIR}/download2" | |||
519 | TIME = "222222" | 620 | TIME = "222222" |
520 | DATE = "20161212" | 621 | DATE = "20161212" |
521 | # Always remove uninative as we're changing proxies | 622 | # Always remove uninative as we're changing proxies |
522 | INHERIT_remove = "uninative" | 623 | INHERIT:remove = "uninative" |
523 | INHERIT += "buildstats-summary buildhistory" | 624 | INHERIT += "buildstats-summary buildhistory" |
524 | http_proxy = "http://example.com/" | 625 | http_proxy = "http://example.com/" |
525 | BB_SIGNATURE_HANDLER = "OEBasicHash" | 626 | BB_SIGNATURE_HANDLER = "OEBasicHash" |
@@ -573,3 +674,335 @@ BB_SIGNATURE_HANDLER = "OEBasicHash" | |||
573 | compare_sigfiles(rest, files1, files2, compare=False) | 674 | compare_sigfiles(rest, files1, files2, compare=False) |
574 | 675 | ||
575 | self.fail("sstate hashes not identical.") | 676 | self.fail("sstate hashes not identical.") |
677 | |||
678 | def test_sstate_movelayer_samesigs(self): | ||
679 | """ | ||
680 | The sstate checksums of two builds with the same oe-core layer in two | ||
681 | different locations should be the same. | ||
682 | """ | ||
683 | core_layer = os.path.join( | ||
684 | self.tc.td["COREBASE"], 'meta') | ||
685 | copy_layer_1 = self.topdir + "/meta-copy1/meta" | ||
686 | copy_layer_2 = self.topdir + "/meta-copy2/meta" | ||
687 | |||
688 | oe.path.copytree(core_layer, copy_layer_1) | ||
689 | os.symlink(os.path.dirname(core_layer) + "/scripts", self.topdir + "/meta-copy1/scripts") | ||
690 | self.write_config(""" | ||
691 | TMPDIR = "${TOPDIR}/tmp-sstatesamehash" | ||
692 | """) | ||
693 | bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_1, core_layer) | ||
694 | self.write_bblayers_config(bblayers_conf) | ||
695 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash") | ||
696 | bitbake("bash -S none") | ||
697 | |||
698 | oe.path.copytree(core_layer, copy_layer_2) | ||
699 | os.symlink(os.path.dirname(core_layer) + "/scripts", self.topdir + "/meta-copy2/scripts") | ||
700 | self.write_config(""" | ||
701 | TMPDIR = "${TOPDIR}/tmp-sstatesamehash2" | ||
702 | """) | ||
703 | bblayers_conf = 'BBLAYERS += "%s"\nBBLAYERS:remove = "%s"' % (copy_layer_2, core_layer) | ||
704 | self.write_bblayers_config(bblayers_conf) | ||
705 | self.track_for_cleanup(self.topdir + "/tmp-sstatesamehash2") | ||
706 | bitbake("bash -S none") | ||
707 | |||
708 | def get_files(d): | ||
709 | f = [] | ||
710 | for root, dirs, files in os.walk(d): | ||
711 | for name in files: | ||
712 | f.append(os.path.join(root, name)) | ||
713 | return f | ||
714 | files1 = get_files(self.topdir + "/tmp-sstatesamehash/stamps") | ||
715 | files2 = get_files(self.topdir + "/tmp-sstatesamehash2/stamps") | ||
716 | files2 = [x.replace("tmp-sstatesamehash2", "tmp-sstatesamehash") for x in files2] | ||
717 | self.maxDiff = None | ||
718 | self.assertCountEqual(files1, files2) | ||
719 | |||
720 | class SStateFindSiginfo(SStateBase): | ||
721 | def test_sstate_compare_sigfiles_and_find_siginfo(self): | ||
722 | """ | ||
723 | Test the functionality of the find_siginfo: basic function and callback in compare_sigfiles | ||
724 | """ | ||
725 | self.write_config(""" | ||
726 | TMPDIR = \"${TOPDIR}/tmp-sstates-findsiginfo\" | ||
727 | TCLIBCAPPEND = \"\" | ||
728 | MACHINE = \"qemux86-64\" | ||
729 | require conf/multilib.conf | ||
730 | MULTILIBS = "multilib:lib32" | ||
731 | DEFAULTTUNE:virtclass-multilib-lib32 = "x86" | ||
732 | BB_SIGNATURE_HANDLER = "OEBasicHash" | ||
733 | """) | ||
734 | self.track_for_cleanup(self.topdir + "/tmp-sstates-findsiginfo") | ||
735 | |||
736 | pns = ["binutils", "binutils-native", "lib32-binutils"] | ||
737 | target_configs = [ | ||
738 | """ | ||
739 | TMPVAL1 = "tmpval1" | ||
740 | TMPVAL2 = "tmpval2" | ||
741 | do_tmptask1() { | ||
742 | echo ${TMPVAL1} | ||
743 | } | ||
744 | do_tmptask2() { | ||
745 | echo ${TMPVAL2} | ||
746 | } | ||
747 | addtask do_tmptask1 | ||
748 | addtask tmptask2 before do_tmptask1 | ||
749 | """, | ||
750 | """ | ||
751 | TMPVAL3 = "tmpval3" | ||
752 | TMPVAL4 = "tmpval4" | ||
753 | do_tmptask1() { | ||
754 | echo ${TMPVAL3} | ||
755 | } | ||
756 | do_tmptask2() { | ||
757 | echo ${TMPVAL4} | ||
758 | } | ||
759 | addtask do_tmptask1 | ||
760 | addtask tmptask2 before do_tmptask1 | ||
761 | """ | ||
762 | ] | ||
763 | |||
764 | for target_config in target_configs: | ||
765 | self.write_recipeinc("binutils", target_config) | ||
766 | for pn in pns: | ||
767 | bitbake("%s -c do_tmptask1 -S none" % pn) | ||
768 | self.delete_recipeinc("binutils") | ||
769 | |||
770 | with bb.tinfoil.Tinfoil() as tinfoil: | ||
771 | tinfoil.prepare(config_only=True) | ||
772 | |||
773 | def find_siginfo(pn, taskname, sigs=None): | ||
774 | result = None | ||
775 | command_complete = False | ||
776 | tinfoil.set_event_mask(["bb.event.FindSigInfoResult", | ||
777 | "bb.command.CommandCompleted"]) | ||
778 | ret = tinfoil.run_command("findSigInfo", pn, taskname, sigs) | ||
779 | if ret: | ||
780 | while result is None or not command_complete: | ||
781 | event = tinfoil.wait_event(1) | ||
782 | if event: | ||
783 | if isinstance(event, bb.command.CommandCompleted): | ||
784 | command_complete = True | ||
785 | elif isinstance(event, bb.event.FindSigInfoResult): | ||
786 | result = event.result | ||
787 | return result | ||
788 | |||
789 | def recursecb(key, hash1, hash2): | ||
790 | nonlocal recursecb_count | ||
791 | recursecb_count += 1 | ||
792 | hashes = [hash1, hash2] | ||
793 | hashfiles = find_siginfo(key, None, hashes) | ||
794 | self.assertCountEqual(hashes, hashfiles) | ||
795 | bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb) | ||
796 | |||
797 | for pn in pns: | ||
798 | recursecb_count = 0 | ||
799 | matches = find_siginfo(pn, "do_tmptask1") | ||
800 | self.assertGreaterEqual(len(matches), 2) | ||
801 | latesthashes = sorted(matches.keys(), key=lambda h: matches[h]['time'])[-2:] | ||
802 | bb.siggen.compare_sigfiles(matches[latesthashes[-2]]['path'], matches[latesthashes[-1]]['path'], recursecb) | ||
803 | self.assertEqual(recursecb_count,1) | ||
804 | |||
805 | class SStatePrintdiff(SStateBase): | ||
806 | def run_test_printdiff_changerecipe(self, target, change_recipe, change_bbtask, change_content, expected_sametmp_output, expected_difftmp_output): | ||
807 | import time | ||
808 | self.write_config(""" | ||
809 | TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}" | ||
810 | """.format(time.time())) | ||
811 | # Use runall do_build to ensure any indirect sstate is created, e.g. tzcode-native on both x86 and | ||
812 | # aarch64 hosts since only allarch target recipes depend upon it and it may not be built otherwise. | ||
813 | # A bitbake -c cleansstate tzcode-native would cause some of these tests to error for example. | ||
814 | bitbake("--runall build --runall deploy_source_date_epoch {}".format(target)) | ||
815 | bitbake("-S none {}".format(target)) | ||
816 | bitbake(change_bbtask) | ||
817 | self.write_recipeinc(change_recipe, change_content) | ||
818 | result_sametmp = bitbake("-S printdiff {}".format(target)) | ||
819 | |||
820 | self.write_config(""" | ||
821 | TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}" | ||
822 | """.format(time.time())) | ||
823 | result_difftmp = bitbake("-S printdiff {}".format(target)) | ||
824 | |||
825 | self.delete_recipeinc(change_recipe) | ||
826 | for item in expected_sametmp_output: | ||
827 | self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output)) | ||
828 | for item in expected_difftmp_output: | ||
829 | self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output)) | ||
830 | |||
831 | def run_test_printdiff_changeconfig(self, target, change_bbtasks, change_content, expected_sametmp_output, expected_difftmp_output): | ||
832 | import time | ||
833 | self.write_config(""" | ||
834 | TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-sametmp-{}" | ||
835 | """.format(time.time())) | ||
836 | bitbake("--runall build --runall deploy_source_date_epoch {}".format(target)) | ||
837 | bitbake("-S none {}".format(target)) | ||
838 | bitbake(" ".join(change_bbtasks)) | ||
839 | self.append_config(change_content) | ||
840 | result_sametmp = bitbake("-S printdiff {}".format(target)) | ||
841 | |||
842 | self.write_config(""" | ||
843 | TMPDIR = "${{TOPDIR}}/tmp-sstateprintdiff-difftmp-{}" | ||
844 | """.format(time.time())) | ||
845 | self.append_config(change_content) | ||
846 | result_difftmp = bitbake("-S printdiff {}".format(target)) | ||
847 | |||
848 | for item in expected_sametmp_output: | ||
849 | self.assertIn(item, result_sametmp.output, msg = "Item {} not found in output:\n{}".format(item, result_sametmp.output)) | ||
850 | for item in expected_difftmp_output: | ||
851 | self.assertIn(item, result_difftmp.output, msg = "Item {} not found in output:\n{}".format(item, result_difftmp.output)) | ||
852 | |||
853 | |||
854 | # Check if printdiff walks the full dependency chain from the image target to where the change is in a specific recipe | ||
855 | def test_image_minimal_vs_perlcross(self): | ||
856 | expected_output = ("Task perlcross-native:do_install couldn't be used from the cache because:", | ||
857 | "We need hash", | ||
858 | "most recent matching task was") | ||
859 | expected_sametmp_output = expected_output + ( | ||
860 | "Variable do_install value changed", | ||
861 | '+ echo "this changes the task signature"') | ||
862 | expected_difftmp_output = expected_output | ||
863 | |||
864 | self.run_test_printdiff_changerecipe("core-image-minimal", "perlcross", "-c do_install perlcross-native", | ||
865 | """ | ||
866 | do_install:append() { | ||
867 | echo "this changes the task signature" | ||
868 | } | ||
869 | """, | ||
870 | expected_sametmp_output, expected_difftmp_output) | ||
871 | |||
872 | # Check if changes to gcc-source (which uses tmp/work-shared) are correctly discovered | ||
873 | def test_gcc_runtime_vs_gcc_source(self): | ||
874 | gcc_source_pn = 'gcc-source-%s' % get_bb_vars(['PV'], 'gcc')['PV'] | ||
875 | |||
876 | expected_output = ("Task {}:do_preconfigure couldn't be used from the cache because:".format(gcc_source_pn), | ||
877 | "We need hash", | ||
878 | "most recent matching task was") | ||
879 | expected_sametmp_output = expected_output + ( | ||
880 | "Variable do_preconfigure value changed", | ||
881 | '+ print("this changes the task signature")') | ||
882 | expected_difftmp_output = expected_output | ||
883 | |||
884 | self.run_test_printdiff_changerecipe("gcc-runtime", "gcc-source", "-c do_preconfigure {}".format(gcc_source_pn), | ||
885 | """ | ||
886 | python do_preconfigure:append() { | ||
887 | print("this changes the task signature") | ||
888 | } | ||
889 | """, | ||
890 | expected_sametmp_output, expected_difftmp_output) | ||
891 | |||
892 | # Check if changing a really base task definiton is reported against multiple core recipes using it | ||
893 | def test_image_minimal_vs_base_do_configure(self): | ||
894 | change_bbtasks = ('zstd-native:do_configure', | ||
895 | 'texinfo-dummy-native:do_configure', | ||
896 | 'ldconfig-native:do_configure', | ||
897 | 'gettext-minimal-native:do_configure', | ||
898 | 'tzcode-native:do_configure', | ||
899 | 'makedevs-native:do_configure', | ||
900 | 'pigz-native:do_configure', | ||
901 | 'update-rc.d-native:do_configure', | ||
902 | 'unzip-native:do_configure', | ||
903 | 'gnu-config-native:do_configure') | ||
904 | |||
905 | expected_output = ["Task {} couldn't be used from the cache because:".format(t) for t in change_bbtasks] + [ | ||
906 | "We need hash", | ||
907 | "most recent matching task was"] | ||
908 | |||
909 | expected_sametmp_output = expected_output + [ | ||
910 | "Variable base_do_configure value changed", | ||
911 | '+ echo "this changes base_do_configure() definiton "'] | ||
912 | expected_difftmp_output = expected_output | ||
913 | |||
914 | self.run_test_printdiff_changeconfig("core-image-minimal",change_bbtasks, | ||
915 | """ | ||
916 | INHERIT += "base-do-configure-modified" | ||
917 | """, | ||
918 | expected_sametmp_output, expected_difftmp_output) | ||
919 | |||
920 | @OETestTag("yocto-mirrors") | ||
921 | class SStateMirrors(SStateBase): | ||
922 | def check_bb_output(self, output, exceptions, check_cdn): | ||
923 | def is_exception(object, exceptions): | ||
924 | for e in exceptions: | ||
925 | if re.search(e, object): | ||
926 | return True | ||
927 | return False | ||
928 | |||
929 | output_l = output.splitlines() | ||
930 | for l in output_l: | ||
931 | if l.startswith("Sstate summary"): | ||
932 | for idx, item in enumerate(l.split()): | ||
933 | if item == 'Missed': | ||
934 | missing_objects = int(l.split()[idx+1]) | ||
935 | break | ||
936 | else: | ||
937 | self.fail("Did not find missing objects amount in sstate summary: {}".format(l)) | ||
938 | break | ||
939 | else: | ||
940 | self.fail("Did not find 'Sstate summary' line in bitbake output") | ||
941 | |||
942 | failed_urls = [] | ||
943 | failed_urls_extrainfo = [] | ||
944 | for l in output_l: | ||
945 | if "SState: Unsuccessful fetch test for" in l and check_cdn: | ||
946 | missing_object = l.split()[6] | ||
947 | elif "SState: Looked for but didn't find file" in l and not check_cdn: | ||
948 | missing_object = l.split()[8] | ||
949 | else: | ||
950 | missing_object = None | ||
951 | if missing_object: | ||
952 | if not is_exception(missing_object, exceptions): | ||
953 | failed_urls.append(missing_object) | ||
954 | else: | ||
955 | missing_objects -= 1 | ||
956 | |||
957 | if "urlopen failed for" in l and not is_exception(l, exceptions): | ||
958 | failed_urls_extrainfo.append(l) | ||
959 | |||
960 | self.assertEqual(len(failed_urls), missing_objects, "Amount of reported missing objects does not match failed URLs: {}\nFailed URLs:\n{}\nFetcher diagnostics:\n{}".format(missing_objects, "\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) | ||
961 | self.assertEqual(len(failed_urls), 0, "Missing objects in the cache:\n{}\nFetcher diagnostics:\n{}".format("\n".join(failed_urls), "\n".join(failed_urls_extrainfo))) | ||
962 | |||
963 | def run_test(self, machine, targets, exceptions, check_cdn = True, ignore_errors = False): | ||
964 | # sstate is checked for existence of these, but they never get written out to begin with | ||
965 | exceptions += ["{}.*image_qa".format(t) for t in targets.split()] | ||
966 | exceptions += ["{}.*deploy_source_date_epoch".format(t) for t in targets.split()] | ||
967 | exceptions += ["{}.*image_complete".format(t) for t in targets.split()] | ||
968 | exceptions += ["linux-yocto.*shared_workdir"] | ||
969 | # these get influnced by IMAGE_FSTYPES tweaks in yocto-autobuilder-helper's config.json (on x86-64) | ||
970 | # additionally, they depend on noexec (thus, absent stamps) package, install, etc. image tasks, | ||
971 | # which makes tracing other changes difficult | ||
972 | exceptions += ["{}.*create_spdx".format(t) for t in targets.split()] | ||
973 | exceptions += ["{}.*create_runtime_spdx".format(t) for t in targets.split()] | ||
974 | |||
975 | if check_cdn: | ||
976 | self.config_sstate(True) | ||
977 | self.append_config(""" | ||
978 | MACHINE = "{}" | ||
979 | BB_HASHSERVE_UPSTREAM = "hashserv.yocto.io:8687" | ||
980 | SSTATE_MIRRORS ?= "file://.* http://cdn.jsdelivr.net/yocto/sstate/all/PATH;downloadfilename=PATH" | ||
981 | """.format(machine)) | ||
982 | else: | ||
983 | self.append_config(""" | ||
984 | MACHINE = "{}" | ||
985 | """.format(machine)) | ||
986 | result = bitbake("-DD -n {}".format(targets)) | ||
987 | bitbake("-S none {}".format(targets)) | ||
988 | if ignore_errors: | ||
989 | return | ||
990 | self.check_bb_output(result.output, exceptions, check_cdn) | ||
991 | |||
992 | def test_cdn_mirror_qemux86_64(self): | ||
993 | exceptions = [] | ||
994 | self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, ignore_errors = True) | ||
995 | self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions) | ||
996 | |||
997 | def test_cdn_mirror_qemuarm64(self): | ||
998 | exceptions = [] | ||
999 | self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, ignore_errors = True) | ||
1000 | self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions) | ||
1001 | |||
1002 | def test_local_cache_qemux86_64(self): | ||
1003 | exceptions = [] | ||
1004 | self.run_test("qemux86-64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) | ||
1005 | |||
1006 | def test_local_cache_qemuarm64(self): | ||
1007 | exceptions = [] | ||
1008 | self.run_test("qemuarm64", "core-image-minimal core-image-full-cmdline core-image-sato-sdk", exceptions, check_cdn = False) | ||