diff options
-rw-r--r-- | meta/lib/oeqa/selftest/cases/reproducible.py | 160 |
1 files changed, 160 insertions, 0 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py new file mode 100644 index 0000000000..6dc83d2847 --- /dev/null +++ b/meta/lib/oeqa/selftest/cases/reproducible.py | |||
@@ -0,0 +1,160 @@ | |||
1 | # | ||
2 | # SPDX-License-Identifier: MIT | ||
3 | # | ||
4 | # Copyright 2019 by Garmin Ltd. or its subsidiaries | ||
5 | |||
6 | from oeqa.selftest.case import OESelftestTestCase | ||
7 | from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars | ||
8 | import functools | ||
9 | import multiprocessing | ||
10 | import textwrap | ||
11 | import unittest | ||
12 | |||
13 | MISSING = 'MISSING' | ||
14 | DIFFERENT = 'DIFFERENT' | ||
15 | SAME = 'SAME' | ||
16 | |||
17 | @functools.total_ordering | ||
18 | class CompareResult(object): | ||
19 | def __init__(self): | ||
20 | self.reference = None | ||
21 | self.test = None | ||
22 | self.status = 'UNKNOWN' | ||
23 | |||
24 | def __eq__(self, other): | ||
25 | return (self.status, self.test) == (other.status, other.test) | ||
26 | |||
27 | def __lt__(self, other): | ||
28 | return (self.status, self.test) < (other.status, other.test) | ||
29 | |||
30 | class PackageCompareResults(object): | ||
31 | def __init__(self): | ||
32 | self.total = [] | ||
33 | self.missing = [] | ||
34 | self.different = [] | ||
35 | self.same = [] | ||
36 | |||
37 | def add_result(self, r): | ||
38 | self.total.append(r) | ||
39 | if r.status == MISSING: | ||
40 | self.missing.append(r) | ||
41 | elif r.status == DIFFERENT: | ||
42 | self.different.append(r) | ||
43 | else: | ||
44 | self.same.append(r) | ||
45 | |||
46 | def sort(self): | ||
47 | self.total.sort() | ||
48 | self.missing.sort() | ||
49 | self.different.sort() | ||
50 | self.same.sort() | ||
51 | |||
52 | def __str__(self): | ||
53 | return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total)) | ||
54 | |||
55 | def compare_file(reference, test, diffutils_sysroot): | ||
56 | result = CompareResult() | ||
57 | result.reference = reference | ||
58 | result.test = test | ||
59 | |||
60 | if not os.path.exists(reference): | ||
61 | result.status = MISSING | ||
62 | return result | ||
63 | |||
64 | r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True) | ||
65 | |||
66 | if r.status: | ||
67 | result.status = DIFFERENT | ||
68 | return result | ||
69 | |||
70 | result.status = SAME | ||
71 | return result | ||
72 | |||
73 | class ReproducibleTests(OESelftestTestCase): | ||
74 | package_classes = ['deb'] | ||
75 | images = ['core-image-minimal'] | ||
76 | |||
77 | def setUpLocal(self): | ||
78 | super().setUpLocal() | ||
79 | needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS'] | ||
80 | bb_vars = get_bb_vars(needed_vars) | ||
81 | for v in needed_vars: | ||
82 | setattr(self, v.lower(), bb_vars[v]) | ||
83 | |||
84 | if not hasattr(self.tc, "extraresults"): | ||
85 | self.tc.extraresults = {} | ||
86 | self.extras = self.tc.extraresults | ||
87 | |||
88 | self.extras.setdefault('reproducible.rawlogs', {})['log'] = '' | ||
89 | |||
90 | def append_to_log(self, msg): | ||
91 | self.extras['reproducible.rawlogs']['log'] += msg | ||
92 | |||
93 | def compare_packages(self, reference_dir, test_dir, diffutils_sysroot): | ||
94 | result = PackageCompareResults() | ||
95 | |||
96 | old_cwd = os.getcwd() | ||
97 | try: | ||
98 | file_result = {} | ||
99 | os.chdir(test_dir) | ||
100 | with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p: | ||
101 | for root, dirs, files in os.walk('.'): | ||
102 | async_result = [] | ||
103 | for f in files: | ||
104 | reference_path = os.path.join(reference_dir, root, f) | ||
105 | test_path = os.path.join(test_dir, root, f) | ||
106 | async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot))) | ||
107 | |||
108 | for a in async_result: | ||
109 | result.add_result(a.get()) | ||
110 | |||
111 | finally: | ||
112 | os.chdir(old_cwd) | ||
113 | |||
114 | result.sort() | ||
115 | return result | ||
116 | |||
117 | @unittest.skip("Reproducible builds do not yet pass") | ||
118 | def test_reproducible_builds(self): | ||
119 | capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes] | ||
120 | |||
121 | common_config = textwrap.dedent('''\ | ||
122 | INHERIT += "reproducible_build" | ||
123 | PACKAGE_CLASSES = "%s" | ||
124 | ''') % (' '.join('package_%s' % c for c in self.package_classes)) | ||
125 | |||
126 | # Do an initial build. It's acceptable for this build to use sstate | ||
127 | self.write_config(common_config) | ||
128 | vars_reference = get_bb_vars(capture_vars) | ||
129 | bitbake(' '.join(self.images)) | ||
130 | |||
131 | # Build native utilities | ||
132 | bitbake("diffutils-native -c addto_recipe_sysroot") | ||
133 | diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native") | ||
134 | |||
135 | # Perform another build. This build should *not* share sstate or pull | ||
136 | # from any mirrors, but sharing a DL_DIR is fine | ||
137 | self.write_config(textwrap.dedent('''\ | ||
138 | TMPDIR = "${TOPDIR}/reproducible/tmp" | ||
139 | SSTATE_DIR = "${TMPDIR}/sstate" | ||
140 | SSTATE_MIRROR = "" | ||
141 | ''') + common_config) | ||
142 | vars_test = get_bb_vars(capture_vars) | ||
143 | bitbake(' '.join(self.images)) | ||
144 | |||
145 | for c in self.package_classes: | ||
146 | package_class = 'package_' + c | ||
147 | |||
148 | deploy_reference = vars_reference['DEPLOY_DIR_' + c.upper()] | ||
149 | deploy_test = vars_test['DEPLOY_DIR_' + c.upper()] | ||
150 | |||
151 | result = self.compare_packages(deploy_reference, deploy_test, diffutils_sysroot) | ||
152 | |||
153 | self.logger.info('Reproducibility summary for %s: %s' % (c, result)) | ||
154 | |||
155 | self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total)) | ||
156 | |||
157 | if result.missing or result.different: | ||
158 | self.fail("The following %s packages are missing or different: %s" % | ||
159 | (c, ' '.join(r.test for r in (result.missing + result.different)))) | ||
160 | |||