diff options
author | Alexander Kanavin <alex.kanavin@gmail.com> | 2020-12-03 14:37:27 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-04-23 10:41:14 +0100 |
commit | a225f167d032490209d18e968834ed4cfd9d08a8 (patch) | |
tree | ea5c436ffac67a73d03b2e0efae8f334fd4ea8cf /meta/lib/oeqa/selftest | |
parent | d685a71c238823274ee901a4adcedf6d39356d96 (diff) | |
download | poky-a225f167d032490209d18e968834ed4cfd9d08a8.tar.gz |
selftest/reproducible: add an exclusion list for items that are not yet reproducible
Hopefully over time this list will be reduced to an empty one.
Non-reproducible excluded packages are not given to diffoscope and do not cause a
failure, but still saved side-by-side with non-reproducible failing ones to make
investigation easier.
(From OE-Core rev: 7cd8b42f11e39b473851b6603a5709f95b4dbf74)
Signed-off-by: Alexander Kanavin <alex.kanavin@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
(cherry picked from commit 406bd0d48d8f90e2c836f7d3e204f21d5f13c833)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oeqa/selftest')
-rw-r--r-- | meta/lib/oeqa/selftest/cases/reproducible.py | 82 |
1 files changed, 79 insertions, 3 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py index 4b60b2e1b0..cf0375c8a1 100644 --- a/meta/lib/oeqa/selftest/cases/reproducible.py +++ b/meta/lib/oeqa/selftest/cases/reproducible.py | |||
@@ -17,6 +17,72 @@ import stat | |||
17 | import os | 17 | import os |
18 | import datetime | 18 | import datetime |
19 | 19 | ||
20 | # For sample packages, see: | ||
21 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-0t7wr_oo/ | ||
22 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-4s9ejwyp/ | ||
23 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-haiwdlbr/ | ||
24 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201127-hwds3mcl/ | ||
25 | # https://autobuilder.yocto.io/pub/repro-fail/oe-reproducible-20201203-sua0pzvc/ | ||
26 | # (both packages/ and packages-excluded/) | ||
27 | exclude_packages = [ | ||
28 | 'acpica-src', | ||
29 | 'babeltrace2-ptest', | ||
30 | 'bootchart2-doc', | ||
31 | 'cups', | ||
32 | 'cwautomacros', | ||
33 | 'dtc', | ||
34 | 'efivar', | ||
35 | 'epiphany', | ||
36 | 'gcr', | ||
37 | 'git', | ||
38 | 'glide', | ||
39 | 'go-dep', | ||
40 | 'go-helloworld', | ||
41 | 'go-runtime', | ||
42 | 'go_', | ||
43 | 'groff', | ||
44 | 'gst-devtools', | ||
45 | 'gstreamer1.0-python', | ||
46 | 'gtk-doc', | ||
47 | 'igt-gpu-tools', | ||
48 | 'kernel-devsrc', | ||
49 | 'libaprutil', | ||
50 | 'libcap-ng', | ||
51 | 'libhandy-1-src', | ||
52 | 'libid3tag', | ||
53 | 'libproxy', | ||
54 | 'libsecret-dev', | ||
55 | 'libsecret-src', | ||
56 | 'lttng-tools-dbg', | ||
57 | 'lttng-tools-ptest', | ||
58 | 'ltp', | ||
59 | 'meson', | ||
60 | 'ovmf-shell-efi', | ||
61 | 'parted-ptest', | ||
62 | 'perf', | ||
63 | 'python3-cython', | ||
64 | 'qemu', | ||
65 | 'quilt-ptest', | ||
66 | 'rsync', | ||
67 | 'ruby', | ||
68 | 'spirv-tools-dev', | ||
69 | 'swig', | ||
70 | 'syslinux-misc', | ||
71 | 'systemd-bootchart', | ||
72 | 'valgrind-ptest', | ||
73 | 'vim', | ||
74 | 'watchdog', | ||
75 | 'xmlto', | ||
76 | 'xorg-minimal-fonts' | ||
77 | ] | ||
78 | |||
79 | def is_excluded(package): | ||
80 | package_name = os.path.basename(package) | ||
81 | for i in exclude_packages: | ||
82 | if package_name.startswith(i): | ||
83 | return True | ||
84 | return False | ||
85 | |||
20 | MISSING = 'MISSING' | 86 | MISSING = 'MISSING' |
21 | DIFFERENT = 'DIFFERENT' | 87 | DIFFERENT = 'DIFFERENT' |
22 | SAME = 'SAME' | 88 | SAME = 'SAME' |
@@ -39,6 +105,7 @@ class PackageCompareResults(object): | |||
39 | self.total = [] | 105 | self.total = [] |
40 | self.missing = [] | 106 | self.missing = [] |
41 | self.different = [] | 107 | self.different = [] |
108 | self.different_excluded = [] | ||
42 | self.same = [] | 109 | self.same = [] |
43 | 110 | ||
44 | def add_result(self, r): | 111 | def add_result(self, r): |
@@ -46,7 +113,10 @@ class PackageCompareResults(object): | |||
46 | if r.status == MISSING: | 113 | if r.status == MISSING: |
47 | self.missing.append(r) | 114 | self.missing.append(r) |
48 | elif r.status == DIFFERENT: | 115 | elif r.status == DIFFERENT: |
49 | self.different.append(r) | 116 | if is_excluded(r.reference): |
117 | self.different_excluded.append(r) | ||
118 | else: | ||
119 | self.different.append(r) | ||
50 | else: | 120 | else: |
51 | self.same.append(r) | 121 | self.same.append(r) |
52 | 122 | ||
@@ -54,10 +124,11 @@ class PackageCompareResults(object): | |||
54 | self.total.sort() | 124 | self.total.sort() |
55 | self.missing.sort() | 125 | self.missing.sort() |
56 | self.different.sort() | 126 | self.different.sort() |
127 | self.different_excluded.sort() | ||
57 | self.same.sort() | 128 | self.same.sort() |
58 | 129 | ||
59 | def __str__(self): | 130 | def __str__(self): |
60 | return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total)) | 131 | return 'same=%i different=%i different_excluded=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.different_excluded), len(self.missing), len(self.total)) |
61 | 132 | ||
62 | def compare_file(reference, test, diffutils_sysroot): | 133 | def compare_file(reference, test, diffutils_sysroot): |
63 | result = CompareResult() | 134 | result = CompareResult() |
@@ -226,6 +297,7 @@ class ReproducibleTests(OESelftestTestCase): | |||
226 | 297 | ||
227 | self.write_package_list(package_class, 'missing', result.missing) | 298 | self.write_package_list(package_class, 'missing', result.missing) |
228 | self.write_package_list(package_class, 'different', result.different) | 299 | self.write_package_list(package_class, 'different', result.different) |
300 | self.write_package_list(package_class, 'different_excluded', result.different_excluded) | ||
229 | self.write_package_list(package_class, 'same', result.same) | 301 | self.write_package_list(package_class, 'same', result.same) |
230 | 302 | ||
231 | if self.save_results: | 303 | if self.save_results: |
@@ -233,8 +305,12 @@ class ReproducibleTests(OESelftestTestCase): | |||
233 | self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) | 305 | self.copy_file(d.reference, '/'.join([save_dir, 'packages', strip_topdir(d.reference)])) |
234 | self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) | 306 | self.copy_file(d.test, '/'.join([save_dir, 'packages', strip_topdir(d.test)])) |
235 | 307 | ||
308 | for d in result.different_excluded: | ||
309 | self.copy_file(d.reference, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.reference)])) | ||
310 | self.copy_file(d.test, '/'.join([save_dir, 'packages-excluded', strip_topdir(d.test)])) | ||
311 | |||
236 | if result.missing or result.different: | 312 | if result.missing or result.different: |
237 | fails.append("The following %s packages are missing or different: %s" % | 313 | fails.append("The following %s packages are missing or different and not in exclusion list: %s" % |
238 | (c, '\n'.join(r.test for r in (result.missing + result.different)))) | 314 | (c, '\n'.join(r.test for r in (result.missing + result.different)))) |
239 | 315 | ||
240 | # Clean up empty directories | 316 | # Clean up empty directories |