diff options
Diffstat (limited to 'meta/lib/oeqa/selftest/cases/reproducible.py')
-rw-r--r-- | meta/lib/oeqa/selftest/cases/reproducible.py | 125 |
1 files changed, 116 insertions, 9 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py index d4800022df..be4cdcc429 100644 --- a/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/meta/lib/oeqa/selftest/cases/reproducible.py | |||
@@ -17,6 +17,57 @@ import stat | |||
17 | import os | 17 | import os |
18 | import datetime | 18 | import datetime |
19 | 19 | ||
20 | # For sample packages, see: | ||
21 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/ | ||
22 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/ | ||
23 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/ | ||
24 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/ | ||
25 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/ | ||
26 | # (both packages/ and packages-excluded/) | ||
27 | exclude_packages = [ | ||
28 | 'acpica-src', | ||
29 | 'babeltrace2-ptest', | ||
30 | 'bind', | ||
31 | 'bootchart2-doc', | ||
32 | 'epiphany', | ||
33 | 'gcr', | ||
34 | 'glide', | ||
35 | 'go-dep', | ||
36 | 'go-helloworld', | ||
37 | 'go-runtime', | ||
38 | 'go_', | ||
39 | 'gstreamer1.0-python', | ||
40 | 'hwlatdetect', | ||
41 | 'kernel-devsrc', | ||
42 | 'libcap-ng', | ||
43 | 'libjson', | ||
44 | 'libproxy', | ||
45 | 'lttng-tools-dbg', | ||
46 | 'lttng-tools-ptest', | ||
47 | 'ltp', | ||
48 | 'ovmf-shell-efi', | ||
49 | 'parted-ptest', | ||
50 | 'perf', | ||
51 | 'piglit', | ||
52 | 'pybootchartgui', | ||
53 | 'qemu', | ||
54 | 'quilt-ptest', | ||
55 | 'rsync', | ||
56 | 'ruby', | ||
57 | 'stress-ng', | ||
58 | 'systemd-bootchart', | ||
59 | 'systemtap', | ||
60 | 'valgrind-ptest', | ||
61 | 'webkitgtk', | ||
62 | ] | ||
63 | |||
64 | def is_excluded(package): | ||
65 | package_name = os.path.basename(package) | ||
66 | for i in exclude_packages: | ||
67 | if package_name.startswith(i): | ||
68 | return i | ||
69 | return None | ||
70 | |||
20 | MISSING = 'MISSING' | 71 | MISSING = 'MISSING' |
21 | DIFFERENT = 'DIFFERENT' | 72 | DIFFERENT = 'DIFFERENT' |
22 | SAME = 'SAME' | 73 | SAME = 'SAME' |
@@ -39,14 +90,21 @@ class PackageCompareResults(object): | |||
39 | self.total = [] | 90 | self.total = [] |
40 | self.missing = [] | 91 | self.missing = [] |
41 | self.different = [] | 92 | self.different = [] |
93 | self.different_excluded = [] | ||
42 | self.same = [] | 94 | self.same = [] |
95 | self.active_exclusions = set() | ||
43 | 96 | ||
44 | def add_result(self, r): | 97 | def add_result(self, r): |
45 | self.total.append(r) | 98 | self.total.append(r) |
46 | if r.status == MISSING: | 99 | if r.status == MISSING: |
47 | self.missing.append(r) | 100 | self.missing.append(r) |
48 | elif r.status == DIFFERENT: | 101 | elif r.status == DIFFERENT: |
49 | self.different.append(r) | 102 | exclusion = is_excluded(r.reference) |
103 | if exclusion: | ||
104 | self.different_excluded.append(r) | ||
105 | self.active_exclusions.add(exclusion) | ||
106 | else: | ||
107 | self.different.append(r) | ||
50 | else: | 108 | else: |
51 | self.same.append(r) | 109 | self.same.append(r) |
52 | 110 | ||
@@ -54,10 +112,14 @@ class PackageCompareResults(object): | |||
54 | self.total.sort() | 112 | self.total.sort() |
55 | self.missing.sort() | 113 | self.missing.sort() |
56 | self.different.sort() | 114 | self.different.sort() |
115 | self.different_excluded.sort() | ||
57 | self.same.sort() | 116 | self.same.sort() |
58 | 117 | ||
59 | def __str__(self): | 118 | def __str__(self): |
60 | return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total)) | 119 | return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions()) |
120 | |||
121 | def unused_exclusions(self): | ||
122 | return sorted(set(exclude_packages) - self.active_exclusions) | ||
61 | 123 | ||
62 | def compare_file(reference, test, diffutils_sysroot): | 124 | def compare_file(reference, test, diffutils_sysroot): |
63 | result = CompareResult() | 125 | result = CompareResult() |
@@ -68,7 +130,7 @@ def compare_file(reference, test, diffutils_sysroot): | |||
68 | result.status = MISSING | 130 | result.status = MISSING |
69 | return result | 131 | return result |
70 | 132 | ||
71 | r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True) | 133 | r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False) |
72 | 134 | ||
73 | if r.status: | 135 | if r.status: |
74 | result.status = DIFFERENT | 136 | result.status = DIFFERENT |
@@ -77,9 +139,41 @@ def compare_file(reference, test, diffutils_sysroot): | |||
77 | result.status = SAME | 139 | result.status = SAME |
78 | return result | 140 | return result |
79 | 141 | ||
142 | def run_diffoscope(a_dir, b_dir, html_dir, **kwargs): | ||
143 | return runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir], | ||
144 | **kwargs) | ||
145 | |||
146 | class DiffoscopeTests(OESelftestTestCase): | ||
147 | diffoscope_test_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), "diffoscope") | ||
148 | |||
149 | def test_diffoscope(self): | ||
150 | bitbake("diffoscope-native -c addto_recipe_sysroot") | ||
151 | diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native") | ||
152 | |||
153 | # Check that diffoscope doesn't return an error when the files compare | ||
154 | # the same (a general check that diffoscope is working) | ||
155 | with tempfile.TemporaryDirectory() as tmpdir: | ||
156 | run_diffoscope('A', 'A', tmpdir, | ||
157 | native_sysroot=diffoscope_sysroot, cwd=self.diffoscope_test_files) | ||
158 | |||
159 | # Check that diffoscope generates an index.html file when the files are | ||
160 | # different | ||
161 | with tempfile.TemporaryDirectory() as tmpdir: | ||
162 | r = run_diffoscope('A', 'B', tmpdir, | ||
163 | native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=self.diffoscope_test_files) | ||
164 | |||
165 | self.assertNotEqual(r.status, 0, msg="diffoscope was successful when an error was expected") | ||
166 | self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!") | ||
167 | |||
80 | class ReproducibleTests(OESelftestTestCase): | 168 | class ReproducibleTests(OESelftestTestCase): |
169 | # Test the reproducibility of whatever is built between sstate_targets and targets | ||
170 | |||
81 | package_classes = ['deb', 'ipk'] | 171 | package_classes = ['deb', 'ipk'] |
82 | images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline'] | 172 | |
173 | # targets are the things we want to test the reproducibility of | ||
174 | targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'world'] | ||
175 | # sstate targets are things to pull from sstate to potentially cut build/debugging time | ||
176 | sstate_targets = [] | ||
83 | save_results = False | 177 | save_results = False |
84 | if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: | 178 | if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: |
85 | save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] | 179 | save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] |
@@ -94,7 +188,7 @@ class ReproducibleTests(OESelftestTestCase): | |||
94 | 188 | ||
95 | def setUpLocal(self): | 189 | def setUpLocal(self): |
96 | super().setUpLocal() | 190 | super().setUpLocal() |
97 | needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS'] | 191 | needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS', 'BB_HASHSERVE'] |
98 | bb_vars = get_bb_vars(needed_vars) | 192 | bb_vars = get_bb_vars(needed_vars) |
99 | for v in needed_vars: | 193 | for v in needed_vars: |
100 | setattr(self, v.lower(), bb_vars[v]) | 194 | setattr(self, v.lower(), bb_vars[v]) |
@@ -150,21 +244,29 @@ class ReproducibleTests(OESelftestTestCase): | |||
150 | PACKAGE_CLASSES = "{package_classes}" | 244 | PACKAGE_CLASSES = "{package_classes}" |
151 | INHIBIT_PACKAGE_STRIP = "1" | 245 | INHIBIT_PACKAGE_STRIP = "1" |
152 | TMPDIR = "{tmpdir}" | 246 | TMPDIR = "{tmpdir}" |
247 | LICENSE_FLAGS_WHITELIST = "commercial" | ||
248 | DISTRO_FEATURES_append = ' systemd pam' | ||
153 | ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes), | 249 | ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes), |
154 | tmpdir=tmpdir) | 250 | tmpdir=tmpdir) |
155 | 251 | ||
156 | if not use_sstate: | 252 | if not use_sstate: |
253 | if self.sstate_targets: | ||
254 | self.logger.info("Building prebuild for %s (sstate allowed)..." % (name)) | ||
255 | self.write_config(config) | ||
256 | bitbake(' '.join(self.sstate_targets)) | ||
257 | |||
157 | # This config fragment will disable using shared and the sstate | 258 | # This config fragment will disable using shared and the sstate |
158 | # mirror, forcing a complete build from scratch | 259 | # mirror, forcing a complete build from scratch |
159 | config += textwrap.dedent('''\ | 260 | config += textwrap.dedent('''\ |
160 | SSTATE_DIR = "${TMPDIR}/sstate" | 261 | SSTATE_DIR = "${TMPDIR}/sstate" |
161 | SSTATE_MIRRORS = "" | 262 | SSTATE_MIRRORS = "file://.*/.*-native.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH file://.*/.*-cross.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH" |
162 | ''') | 263 | ''') |
163 | 264 | ||
164 | self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT')) | 265 | self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT')) |
165 | self.write_config(config) | 266 | self.write_config(config) |
166 | d = get_bb_vars(capture_vars) | 267 | d = get_bb_vars(capture_vars) |
167 | bitbake(' '.join(self.images)) | 268 | # targets used to be called images |
269 | bitbake(' '.join(getattr(self, 'images', self.targets))) | ||
168 | return d | 270 | return d |
169 | 271 | ||
170 | def test_reproducible_builds(self): | 272 | def test_reproducible_builds(self): |
@@ -212,6 +314,7 @@ class ReproducibleTests(OESelftestTestCase): | |||
212 | 314 | ||
213 | self.write_package_list(package_class, 'missing', result.missing) | 315 | self.write_package_list(package_class, 'missing', result.missing) |
214 | self.write_package_list(package_class, 'different', result.different) | 316 | self.write_package_list(package_class, 'different', result.different) |
317 | self.write_package_list(package_class, 'different_excluded', result.different_excluded) | ||
215 | self.write_package_list(package_class, 'same', result.same) | 318 | self.write_package_list(package_class, 'same', result.same) |
216 | 319 | ||
217 | if self.save_results: | 320 | if self.save_results: |
@@ -219,8 +322,12 @@ class ReproducibleTests(OESelftestTestCase): | |||
219 | self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) | 322 | self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) |
220 | self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) | 323 | self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) |
221 | 324 | ||
325 | for d in result.different_excluded: | ||
326 | self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)])) | ||
327 | self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)])) | ||
328 | |||
222 | if result.missing or result.different: | 329 | if result.missing or result.different: |
223 | fails.append("The following %s packages are missing or different: %s" % | 330 | fails.append("The following %s packages are missing or different and not in exclusion list: %s" % |
224 | (c, '\n'.join(r.test for r in (result.missing + result.different)))) | 331 | (c, '\n'.join(r.test for r in (result.missing + result.different)))) |
225 | 332 | ||
226 | # Clean up empty directories | 333 | # Clean up empty directories |
@@ -235,7 +342,7 @@ class ReproducibleTests(OESelftestTestCase): | |||
235 | # Copy jquery to improve the diffoscope output usability | 342 | # Copy jquery to improve the diffoscope output usability |
236 | self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js')) | 343 | self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js')) |
237 | 344 | ||
238 | runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', '--html-dir', package_html_dir, 'reproducibleA', 'reproducibleB'], | 345 | run_diffoscope('reproducibleA', 'reproducibleB', package_html_dir, |
239 | native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir) | 346 | native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir) |
240 | 347 | ||
241 | if fails: | 348 | if fails: |