summaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa/selftest
diff options
context:
space:
mode:
authorAlexander Kanavin <alex.kanavin@gmail.com>2020-12-03 14:37:27 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-04-23 10:41:14 +0100
commita225f167d032490209d18e968834ed4cfd9d08a8 (patch)
treeea5c436ffac67a73d03b2e0efae8f334fd4ea8cf /meta/lib/oeqa/selftest
parentd685a71c238823274ee901a4adcedf6d39356d96 (diff)
downloadpoky-a225f167d032490209d18e968834ed4cfd9d08a8.tar.gz
selftest/reproducible: add an exclusion list for items that are not yet reproducible
Hopefully over time this list will be reduced to an empty one. Non-reproducible excluded packages are not given to diffoscope and do not cause a failure, but still saved side-by-side with non-reproducible failing ones to make investigation easier. (From OE-Core rev: 7cd8b42f11e39b473851b6603a5709f95b4dbf74) Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> (cherry picked from commit 406bd0d48d8f90e2c836f7d3e204f21d5f13c833) Signed-off-by: Steve Sakoman <steve@sakoman.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oeqa/selftest')
-rw-r--r--meta/lib/oeqa/selftest/cases/reproducible.py82
1 files changed, 79 insertions, 3 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
index 4b60b2e1b0..cf0375c8a1 100644
--- a/meta/lib/oeqa/selftest/cases/reproducible.py
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -17,6 +17,72 @@ import stat
17import os 17import os
18import datetime 18import datetime
19 19
20# For sample packages, see:
21# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/
22# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/
23# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/
24# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/
25# https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/
26# (both packages/ and packages-excluded/)
27exclude_packages = [
28 'acpica-src',
29 'babeltrace2-ptest',
30 'bootchart2-doc',
31 'cups',
32 'cwautomacros',
33 'dtc',
34 'efivar',
35 'epiphany',
36 'gcr',
37 'git',
38 'glide',
39 'go-dep',
40 'go-helloworld',
41 'go-runtime',
42 'go_',
43 'groff',
44 'gst-devtools',
45 'gstreamer1.0-python',
46 'gtk-doc',
47 'igt-gpu-tools',
48 'kernel-devsrc',
49 'libaprutil',
50 'libcap-ng',
51 'libhandy-1-src',
52 'libid3tag',
53 'libproxy',
54 'libsecret-dev',
55 'libsecret-src',
56 'lttng-tools-dbg',
57 'lttng-tools-ptest',
58 'ltp',
59 'meson',
60 'ovmf-shell-efi',
61 'parted-ptest',
62 'perf',
63 'python3-cython',
64 'qemu',
65 'quilt-ptest',
66 'rsync',
67 'ruby',
68 'spirv-tools-dev',
69 'swig',
70 'syslinux-misc',
71 'systemd-bootchart',
72 'valgrind-ptest',
73 'vim',
74 'watchdog',
75 'xmlto',
76 'xorg-minimal-fonts'
77 ]
78
79def is_excluded(package):
80 package_name = os.path.basename(package)
81 for i in exclude_packages:
82 if package_name.startswith(i):
83 return True
84 return False
85
20MISSING = 'MISSING' 86MISSING = 'MISSING'
21DIFFERENT = 'DIFFERENT' 87DIFFERENT = 'DIFFERENT'
22SAME = 'SAME' 88SAME = 'SAME'
@@ -39,6 +105,7 @@ class PackageCompareResults(object):
39 self.total = [] 105 self.total = []
40 self.missing = [] 106 self.missing = []
41 self.different = [] 107 self.different = []
108 self.different_excluded = []
42 self.same = [] 109 self.same = []
43 110
44 def add_result(self, r): 111 def add_result(self, r):
@@ -46,7 +113,10 @@ class PackageCompareResults(object):
46 if r.status == MISSING: 113 if r.status == MISSING:
47 self.missing.append(r) 114 self.missing.append(r)
48 elif r.status == DIFFERENT: 115 elif r.status == DIFFERENT:
49 self.different.append(r) 116 if is_excluded(r.reference):
117 self.different_excluded.append(r)
118 else:
119 self.different.append(r)
50 else: 120 else:
51 self.same.append(r) 121 self.same.append(r)
52 122
@@ -54,10 +124,11 @@ class PackageCompareResults(object):
54 self.total.sort() 124 self.total.sort()
55 self.missing.sort() 125 self.missing.sort()
56 self.different.sort() 126 self.different.sort()
127 self.different_excluded.sort()
57 self.same.sort() 128 self.same.sort()
58 129
59 def __str__(self): 130 def __str__(self):
60 return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total)) 131 return 'same=%i different=%i different_excluded=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total))
61 132
62def compare_file(reference, test, diffutils_sysroot): 133def compare_file(reference, test, diffutils_sysroot):
63 result = CompareResult() 134 result = CompareResult()
@@ -226,6 +297,7 @@ class ReproducibleTests(OESelftestTestCase):
226 297
227 self.write_package_list(package_class, 'missing', result.missing) 298 self.write_package_list(package_class, 'missing', result.missing)
228 self.write_package_list(package_class, 'different', result.different) 299 self.write_package_list(package_class, 'different', result.different)
300 self.write_package_list(package_class, 'different_excluded', result.different_excluded)
229 self.write_package_list(package_class, 'same', result.same) 301 self.write_package_list(package_class, 'same', result.same)
230 302
231 if self.save_results: 303 if self.save_results:
@@ -233,8 +305,12 @@ class ReproducibleTests(OESelftestTestCase):
233 self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) 305 self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)]))
234 self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) 306 self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)]))
235 307
308 for d in result.different_excluded:
309 self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)]))
310 self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)]))
311
236 if result.missing or result.different: 312 if result.missing or result.different:
237 fails.append("The following %s packages are missing or different: %s" % 313 fails.append("The following %s packages are missing or different and not in exclusion list: %s" %
238 (c, '\n'.join(r.test for r in (result.missing + result.different)))) 314 (c, '\n'.join(r.test for r in (result.missing + result.different))))
239 315
240 # Clean up empty directories 316 # Clean up empty directories