summaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa/selftest/cases/reproducible.py
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/oeqa/selftest/cases/reproducible.py')
-rw-r--r--meta/lib/oeqa/selftest/cases/reproducible.py125
1 files changed, 116 insertions, 9 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
index d4800022df..be4cdcc429 100644
--- a/meta/lib/oeqa/selftest/cases/reproducible.py
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -17,6 +17,57 @@ import stat
17import os 17import os
18import datetime 18import datetime
19 19
20# For sample packages, see:
21# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/
22# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/
23# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/
24# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/
25# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/
26# (both packages/ and packages-excluded/)
27exclude_packages = [
28 'acpica-src',
29 'babeltrace2-ptest',
30 'bind',
31 'bootchart2-doc',
32 'epiphany',
33 'gcr',
34 'glide',
35 'go-dep',
36 'go-helloworld',
37 'go-runtime',
38 'go_',
39 'gstreamer1.0-python',
40 'hwlatdetect',
41 'kernel-devsrc',
42 'libcap-ng',
43 'libjson',
44 'libproxy',
45 'lttng-tools-dbg',
46 'lttng-tools-ptest',
47 'ltp',
48 'ovmf-shell-efi',
49 'parted-ptest',
50 'perf',
51 'piglit',
52 'pybootchartgui',
53 'qemu',
54 'quilt-ptest',
55 'rsync',
56 'ruby',
57 'stress-ng',
58 'systemd-bootchart',
59 'systemtap',
60 'valgrind-ptest',
61 'webkitgtk',
62 ]
63
64def is_excluded(package):
65 package_name = os.path.basename(package)
66 for i in exclude_packages:
67 if package_name.startswith(i):
68 return i
69 return None
70
20MISSING = 'MISSING' 71MISSING = 'MISSING'
21DIFFERENT = 'DIFFERENT' 72DIFFERENT = 'DIFFERENT'
22SAME = 'SAME' 73SAME = 'SAME'
@@ -39,14 +90,21 @@ class PackageCompareResults(object):
39 self.total = [] 90 self.total = []
40 self.missing = [] 91 self.missing = []
41 self.different = [] 92 self.different = []
93 self.different_excluded = []
42 self.same = [] 94 self.same = []
95 self.active_exclusions = set()
43 96
44 def add_result(self, r): 97 def add_result(self, r):
45 self.total.append(r) 98 self.total.append(r)
46 if r.status == MISSING: 99 if r.status == MISSING:
47 self.missing.append(r) 100 self.missing.append(r)
48 elif r.status == DIFFERENT: 101 elif r.status == DIFFERENT:
49 self.different.append(r) 102 exclusion = is_excluded(r.reference)
103 if exclusion:
104 self.different_excluded.append(r)
105 self.active_exclusions.add(exclusion)
106 else:
107 self.different.append(r)
50 else: 108 else:
51 self.same.append(r) 109 self.same.append(r)
52 110
@@ -54,10 +112,14 @@ class PackageCompareResults(object):
54 self.total.sort() 112 self.total.sort()
55 self.missing.sort() 113 self.missing.sort()
56 self.different.sort() 114 self.different.sort()
115 self.different_excluded.sort()
57 self.same.sort() 116 self.same.sort()
58 117
59 def __str__(self): 118 def __str__(self):
60 return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total)) 119 return 'same=%i different=%i different_excluded=%i missing=%i total=%i\nunused_exclusions=%s' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total), self.unused_exclusions())
120
121 def unused_exclusions(self):
122 return sorted(set(exclude_packages) - self.active_exclusions)
61 123
62def compare_file(reference, test, diffutils_sysroot): 124def compare_file(reference, test, diffutils_sysroot):
63 result = CompareResult() 125 result = CompareResult()
@@ -68,7 +130,7 @@ def compare_file(reference, test, diffutils_sysroot):
68 result.status = MISSING 130 result.status = MISSING
69 return result 131 return result
70 132
71 r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True) 133 r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True, sync=False)
72 134
73 if r.status: 135 if r.status:
74 result.status = DIFFERENT 136 result.status = DIFFERENT
@@ -77,9 +139,41 @@ def compare_file(reference, test, diffutils_sysroot):
77 result.status = SAME 139 result.status = SAME
78 return result 140 return result
79 141
142def run_diffoscope(a_dir, b_dir, html_dir, **kwargs):
143 return runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', 'yes', '--html-dir', html_dir, a_dir, b_dir],
144 **kwargs)
145
146class DiffoscopeTests(OESelftestTestCase):
147 diffoscope_test_files = os.path.join(os.path.dirname(os.path.abspath(__file__)), "diffoscope")
148
149 def test_diffoscope(self):
150 bitbake("diffoscope-native -c addto_recipe_sysroot")
151 diffoscope_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffoscope-native")
152
153 # Check that diffoscope doesn't return an error when the files compare
154 # the same (a general check that diffoscope is working)
155 with tempfile.TemporaryDirectory() as tmpdir:
156 run_diffoscope('A', 'A', tmpdir,
157 native_sysroot=diffoscope_sysroot, cwd=self.diffoscope_test_files)
158
159 # Check that diffoscope generates an index.html file when the files are
160 # different
161 with tempfile.TemporaryDirectory() as tmpdir:
162 r = run_diffoscope('A', 'B', tmpdir,
163 native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=self.diffoscope_test_files)
164
165 self.assertNotEqual(r.status, 0, msg="diffoscope was successful when an error was expected")
166 self.assertTrue(os.path.exists(os.path.join(tmpdir, 'index.html')), "HTML index not found!")
167
80class ReproducibleTests(OESelftestTestCase): 168class ReproducibleTests(OESelftestTestCase):
169 # Test the reproducibility of whatever is built between sstate_targets and targets
170
81 package_classes = ['deb', 'ipk'] 171 package_classes = ['deb', 'ipk']
82 images = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline'] 172
173 # targets are the things we want to test the reproducibility of
174 targets = ['core-image-minimal', 'core-image-sato', 'core-image-full-cmdline', 'world']
175 # sstate targets are things to pull from sstate to potentially cut build/debugging time
176 sstate_targets = []
83 save_results = False 177 save_results = False
84 if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ: 178 if 'OEQA_DEBUGGING_SAVED_OUTPUT' in os.environ:
85 save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT'] 179 save_results = os.environ['OEQA_DEBUGGING_SAVED_OUTPUT']
@@ -94,7 +188,7 @@ class ReproducibleTests(OESelftestTestCase):
94 188
95 def setUpLocal(self): 189 def setUpLocal(self):
96 super().setUpLocal() 190 super().setUpLocal()
97 needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS'] 191 needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS', 'BB_HASHSERVE']
98 bb_vars = get_bb_vars(needed_vars) 192 bb_vars = get_bb_vars(needed_vars)
99 for v in needed_vars: 193 for v in needed_vars:
100 setattr(self, v.lower(), bb_vars[v]) 194 setattr(self, v.lower(), bb_vars[v])
@@ -150,21 +244,29 @@ class ReproducibleTests(OESelftestTestCase):
150 PACKAGE_CLASSES = "{package_classes}" 244 PACKAGE_CLASSES = "{package_classes}"
151 INHIBIT_PACKAGE_STRIP = "1" 245 INHIBIT_PACKAGE_STRIP = "1"
152 TMPDIR = "{tmpdir}" 246 TMPDIR = "{tmpdir}"
247 LICENSE_FLAGS_WHITELIST = "commercial"
248 DISTRO_FEATURES_append = ' systemd pam'
153 ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes), 249 ''').format(package_classes=' '.join('package_%s' % c for c in self.package_classes),
154 tmpdir=tmpdir) 250 tmpdir=tmpdir)
155 251
156 if not use_sstate: 252 if not use_sstate:
253 if self.sstate_targets:
254 self.logger.info("Building prebuild for %s (sstate allowed)..." % (name))
255 self.write_config(config)
256 bitbake(' '.join(self.sstate_targets))
257
157 # This config fragment will disable using shared and the sstate 258 # This config fragment will disable using shared and the sstate
158 # mirror, forcing a complete build from scratch 259 # mirror, forcing a complete build from scratch
159 config += textwrap.dedent('''\ 260 config += textwrap.dedent('''\
160 SSTATE_DIR = "${TMPDIR}/sstate" 261 SSTATE_DIR = "${TMPDIR}/sstate"
161 SSTATE_MIRRORS = "" 262 SSTATE_MIRRORS = "file://.*/.*-native.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH file://.*/.*-cross.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH"
162 ''') 263 ''')
163 264
164 self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT')) 265 self.logger.info("Building %s (sstate%s allowed)..." % (name, '' if use_sstate else ' NOT'))
165 self.write_config(config) 266 self.write_config(config)
166 d = get_bb_vars(capture_vars) 267 d = get_bb_vars(capture_vars)
167 bitbake(' '.join(self.images)) 268 # targets used to be called images
269 bitbake(' '.join(getattr(self, 'images', self.targets)))
168 return d 270 return d
169 271
170 def test_reproducible_builds(self): 272 def test_reproducible_builds(self):
@@ -212,6 +314,7 @@ class ReproducibleTests(OESelftestTestCase):
212 314
213 self.write_package_list(package_class, 'missing', result.missing) 315 self.write_package_list(package_class, 'missing', result.missing)
214 self.write_package_list(package_class, 'different', result.different) 316 self.write_package_list(package_class, 'different', result.different)
317 self.write_package_list(package_class, 'different_excluded', result.different_excluded)
215 self.write_package_list(package_class, 'same', result.same) 318 self.write_package_list(package_class, 'same', result.same)
216 319
217 if self.save_results: 320 if self.save_results:
@@ -219,8 +322,12 @@ class ReproducibleTests(OESelftestTestCase):
219 self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) 322 self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)]))
220 self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) 323 self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)]))
221 324
325 for d in result.different_excluded:
326 self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)]))
327 self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)]))
328
222 if result.missing or result.different: 329 if result.missing or result.different:
223 fails.append("The following %s packages are missing or different: %s" % 330 fails.append("The following %s packages are missing or different and not in exclusion list: %s" %
224 (c, '\n'.join(r.test for r in (result.missing + result.different)))) 331 (c, '\n'.join(r.test for r in (result.missing + result.different))))
225 332
226 # Clean up empty directories 333 # Clean up empty directories
@@ -235,7 +342,7 @@ class ReproducibleTests(OESelftestTestCase):
235 # Copy jquery to improve the diffoscope output usability 342 # Copy jquery to improve the diffoscope output usability
236 self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js')) 343 self.copy_file(os.path.join(jquery_sysroot, 'usr/share/javascript/jquery/jquery.min.js'), os.path.join(package_html_dir, 'jquery.js'))
237 344
238 runCmd(['diffoscope', '--no-default-limits', '--exclude-directory-metadata', '--html-dir', package_html_dir, 'reproducibleA', 'reproducibleB'], 345 run_diffoscope('reproducibleA', 'reproducibleB', package_html_dir,
239 native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir) 346 native_sysroot=diffoscope_sysroot, ignore_status=True, cwd=package_dir)
240 347
241 if fails: 348 if fails: