summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJoshua Watt <jpewhacker@gmail.com>2019-06-06 15:33:28 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-06-10 14:46:38 +0100
commite5217b6c1024f216226443cb8246db9fc090f113 (patch)
tree4ab347fbbbe36879d310044e3efd9de2008c3cbc
parent17dfe628d175c2af3d3adabd7e1f1f6f1eefec13 (diff)
downloadpoky-e5217b6c1024f216226443cb8246db9fc090f113.tar.gz
oeqa: Add reproducible build selftest
Adds an initial test for reproducible builds to the OE selftest. This initial test builds core-image-minimal using sstate, then does a clean build without sstate in another build directory, and finally does a binary comparison of the resulting package files between the two builds. The test is currently always skipped since it doesn't pass yet, but it can easily be enabled locally (From OE-Core rev: 2e591bdf93ec9e59b900562263dfe8e72b163baa) Signed-off-by: Joshua Watt <JPEWHacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oeqa/selftest/cases/reproducible.py160
1 files changed, 160 insertions, 0 deletions
diff --git a/meta/lib/oeqa/selftest/cases/reproducible.py b/meta/lib/oeqa/selftest/cases/reproducible.py
new file mode 100644
index 0000000000..6dc83d2847
--- /dev/null
+++ b/meta/lib/oeqa/selftest/cases/reproducible.py
@@ -0,0 +1,160 @@
1#
2# SPDX-License-Identifier: MIT
3#
4# Copyright 2019 by Garmin Ltd. or its subsidiaries
5
6from oeqa.selftest.case import OESelftestTestCase
7from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_bb_vars
8import functools
9import multiprocessing
10import textwrap
11import unittest
12
13MISSING = 'MISSING'
14DIFFERENT = 'DIFFERENT'
15SAME = 'SAME'
16
17@functools.total_ordering
18class CompareResult(object):
19 def __init__(self):
20 self.reference = None
21 self.test = None
22 self.status = 'UNKNOWN'
23
24 def __eq__(self, other):
25 return (self.status, self.test) == (other.status, other.test)
26
27 def __lt__(self, other):
28 return (self.status, self.test) < (other.status, other.test)
29
30class PackageCompareResults(object):
31 def __init__(self):
32 self.total = []
33 self.missing = []
34 self.different = []
35 self.same = []
36
37 def add_result(self, r):
38 self.total.append(r)
39 if r.status == MISSING:
40 self.missing.append(r)
41 elif r.status == DIFFERENT:
42 self.different.append(r)
43 else:
44 self.same.append(r)
45
46 def sort(self):
47 self.total.sort()
48 self.missing.sort()
49 self.different.sort()
50 self.same.sort()
51
52 def __str__(self):
53 return 'same=%i different=%i missing=%i total=%i' % (len(self.same), len(self.different), len(self.missing), len(self.total))
54
55def compare_file(reference, test, diffutils_sysroot):
56 result = CompareResult()
57 result.reference = reference
58 result.test = test
59
60 if not os.path.exists(reference):
61 result.status = MISSING
62 return result
63
64 r = runCmd(['cmp', '--quiet', reference, test], native_sysroot=diffutils_sysroot, ignore_status=True)
65
66 if r.status:
67 result.status = DIFFERENT
68 return result
69
70 result.status = SAME
71 return result
72
73class ReproducibleTests(OESelftestTestCase):
74 package_classes = ['deb']
75 images = ['core-image-minimal']
76
77 def setUpLocal(self):
78 super().setUpLocal()
79 needed_vars = ['TOPDIR', 'TARGET_PREFIX', 'BB_NUMBER_THREADS']
80 bb_vars = get_bb_vars(needed_vars)
81 for v in needed_vars:
82 setattr(self, v.lower(), bb_vars[v])
83
84 if not hasattr(self.tc, "extraresults"):
85 self.tc.extraresults = {}
86 self.extras = self.tc.extraresults
87
88 self.extras.setdefault('reproducible.rawlogs', {})['log'] = ''
89
90 def append_to_log(self, msg):
91 self.extras['reproducible.rawlogs']['log'] += msg
92
93 def compare_packages(self, reference_dir, test_dir, diffutils_sysroot):
94 result = PackageCompareResults()
95
96 old_cwd = os.getcwd()
97 try:
98 file_result = {}
99 os.chdir(test_dir)
100 with multiprocessing.Pool(processes=int(self.bb_number_threads or 0)) as p:
101 for root, dirs, files in os.walk('.'):
102 async_result = []
103 for f in files:
104 reference_path = os.path.join(reference_dir, root, f)
105 test_path = os.path.join(test_dir, root, f)
106 async_result.append(p.apply_async(compare_file, (reference_path, test_path, diffutils_sysroot)))
107
108 for a in async_result:
109 result.add_result(a.get())
110
111 finally:
112 os.chdir(old_cwd)
113
114 result.sort()
115 return result
116
117 @unittest.skip("Reproducible builds do not yet pass")
118 def test_reproducible_builds(self):
119 capture_vars = ['DEPLOY_DIR_' + c.upper() for c in self.package_classes]
120
121 common_config = textwrap.dedent('''\
122 INHERIT += "reproducible_build"
123 PACKAGE_CLASSES = "%s"
124 ''') % (' '.join('package_%s' % c for c in self.package_classes))
125
126 # Do an initial build. It's acceptable for this build to use sstate
127 self.write_config(common_config)
128 vars_reference = get_bb_vars(capture_vars)
129 bitbake(' '.join(self.images))
130
131 # Build native utilities
132 bitbake("diffutils-native -c addto_recipe_sysroot")
133 diffutils_sysroot = get_bb_var("RECIPE_SYSROOT_NATIVE", "diffutils-native")
134
135 # Perform another build. This build should *not* share sstate or pull
136 # from any mirrors, but sharing a DL_DIR is fine
137 self.write_config(textwrap.dedent('''\
138 TMPDIR = "${TOPDIR}/reproducible/tmp"
139 SSTATE_DIR = "${TMPDIR}/sstate"
140 SSTATE_MIRROR = ""
141 ''') + common_config)
142 vars_test = get_bb_vars(capture_vars)
143 bitbake(' '.join(self.images))
144
145 for c in self.package_classes:
146 package_class = 'package_' + c
147
148 deploy_reference = vars_reference['DEPLOY_DIR_' + c.upper()]
149 deploy_test = vars_test['DEPLOY_DIR_' + c.upper()]
150
151 result = self.compare_packages(deploy_reference, deploy_test, diffutils_sysroot)
152
153 self.logger.info('Reproducibility summary for %s: %s' % (c, result))
154
155 self.append_to_log('\n'.join("%s: %s" % (r.status, r.test) for r in result.total))
156
157 if result.missing or result.different:
158 self.fail("The following %s packages are missing or different: %s" %
159 (c, ' '.join(r.test for r in (result.missing + result.different))))
160