diff options
Diffstat (limited to 'scripts/lib/compatlayer')
-rw-r--r-- | scripts/lib/compatlayer/__init__.py | 7 | ||||
-rw-r--r-- | scripts/lib/compatlayer/cases/bsp.py | 154 |
2 files changed, 158 insertions, 3 deletions
diff --git a/scripts/lib/compatlayer/__init__.py b/scripts/lib/compatlayer/__init__.py index 6130b8548c..0d6f4e9c3b 100644 --- a/scripts/lib/compatlayer/__init__.py +++ b/scripts/lib/compatlayer/__init__.py | |||
@@ -215,7 +215,7 @@ def check_command(error_msg, cmd): | |||
215 | raise RuntimeError(msg) | 215 | raise RuntimeError(msg) |
216 | return output | 216 | return output |
217 | 217 | ||
218 | def get_signatures(builddir, failsafe=False): | 218 | def get_signatures(builddir, failsafe=False, machine=None): |
219 | import re | 219 | import re |
220 | 220 | ||
221 | # some recipes needs to be excluded like meta-world-pkgdata | 221 | # some recipes needs to be excluded like meta-world-pkgdata |
@@ -226,7 +226,10 @@ def get_signatures(builddir, failsafe=False): | |||
226 | sigs = {} | 226 | sigs = {} |
227 | tune2tasks = {} | 227 | tune2tasks = {} |
228 | 228 | ||
229 | cmd = 'bitbake ' | 229 | cmd = '' |
230 | if machine: | ||
231 | cmd += 'MACHINE=%s ' % machine | ||
232 | cmd += 'bitbake ' | ||
230 | if failsafe: | 233 | if failsafe: |
231 | cmd += '-k ' | 234 | cmd += '-k ' |
232 | cmd += '-S none world' | 235 | cmd += '-S none world' |
diff --git a/scripts/lib/compatlayer/cases/bsp.py b/scripts/lib/compatlayer/cases/bsp.py index 5d9bf93e4a..90256750af 100644 --- a/scripts/lib/compatlayer/cases/bsp.py +++ b/scripts/lib/compatlayer/cases/bsp.py | |||
@@ -3,7 +3,7 @@ | |||
3 | 3 | ||
4 | import unittest | 4 | import unittest |
5 | 5 | ||
6 | from compatlayer import LayerType | 6 | from compatlayer import LayerType, get_signatures, check_command, get_depgraph |
7 | from compatlayer.case import OECompatLayerTestCase | 7 | from compatlayer.case import OECompatLayerTestCase |
8 | 8 | ||
9 | class BSPCompatLayer(OECompatLayerTestCase): | 9 | class BSPCompatLayer(OECompatLayerTestCase): |
@@ -24,3 +24,155 @@ class BSPCompatLayer(OECompatLayerTestCase): | |||
24 | self.assertEqual(self.td['bbvars']['MACHINE'], machine, | 24 | self.assertEqual(self.td['bbvars']['MACHINE'], machine, |
25 | msg="Layer %s modified machine %s -> %s" % \ | 25 | msg="Layer %s modified machine %s -> %s" % \ |
26 | (self.tc.layer['name'], self.td['bbvars']['MACHINE'], machine)) | 26 | (self.tc.layer['name'], self.td['bbvars']['MACHINE'], machine)) |
27 | |||
28 | def test_machine_signatures(self): | ||
29 | ''' | ||
30 | Selecting a machine may only affect the signature of tasks that are specific | ||
31 | to that machine. In other words, when MACHINE=A and MACHINE=B share a recipe | ||
32 | foo and the output of foo, then both machine configurations must build foo | ||
33 | in exactly the same way. Otherwise it is not possible to use both machines | ||
34 | in the same distribution. | ||
35 | |||
36 | This criteria can only be tested by testing different machines in combination, | ||
37 | i.e. one main layer, potentially several additional BSP layers and an explicit | ||
38 | choice of machines: | ||
39 | yocto-compat-layer --additional-layers .../meta-intel --machines intel-corei7-64 imx6slevk -- .../meta-freescale | ||
40 | ''' | ||
41 | |||
42 | if not self.td['machines']: | ||
43 | self.skipTest('No machines set with --machines.') | ||
44 | |||
45 | # Collect signatures for all machines that we are testing | ||
46 | # and merge that into a hash: | ||
47 | # tune -> task -> signature -> list of machines with that combination | ||
48 | # | ||
49 | # It is an error if any tune/task pair has more than one signature, | ||
50 | # because that implies that the machines that caused those different | ||
51 | # signatures do not agree on how to execute the task. | ||
52 | tunes = {} | ||
53 | # Preserve ordering of machines as chosen by the user. | ||
54 | for machine in self.td['machines']: | ||
55 | curr_sigs, tune2tasks = get_signatures(self.td['builddir'], failsafe=True, machine=machine) | ||
56 | # Invert the tune -> [tasks] mapping. | ||
57 | tasks2tune = {} | ||
58 | for tune, tasks in tune2tasks.items(): | ||
59 | for task in tasks: | ||
60 | tasks2tune[task] = tune | ||
61 | for task, sighash in curr_sigs.items(): | ||
62 | tunes.setdefault(tasks2tune[task], {}).setdefault(task, {}).setdefault(sighash, []).append(machine) | ||
63 | |||
64 | msg = [] | ||
65 | pruned = 0 | ||
66 | last_line_key = None | ||
67 | # do_fetch, do_unpack, ..., do_build | ||
68 | taskname_list = [] | ||
69 | if tunes: | ||
70 | # The output below is most useful when we start with tasks that are at | ||
71 | # the bottom of the dependency chain, i.e. those that run first. If | ||
72 | # those tasks differ, the rest also does. | ||
73 | # | ||
74 | # To get an ordering of tasks, we do a topological sort of the entire | ||
75 | # depgraph for the base configuration, then on-the-fly flatten that list by stripping | ||
76 | # out the recipe names and removing duplicates. The base configuration | ||
77 | # is not necessarily representative, but should be close enough. Tasks | ||
78 | # that were not encountered get a default priority. | ||
79 | depgraph = get_depgraph() | ||
80 | depends = depgraph['tdepends'] | ||
81 | WHITE = 1 | ||
82 | GRAY = 2 | ||
83 | BLACK = 3 | ||
84 | color = {} | ||
85 | found = set() | ||
86 | def visit(task): | ||
87 | color[task] = GRAY | ||
88 | for dep in depends.get(task, ()): | ||
89 | if color.setdefault(dep, WHITE) == WHITE: | ||
90 | visit(dep) | ||
91 | color[task] = BLACK | ||
92 | pn, taskname = task.rsplit('.', 1) | ||
93 | if taskname not in found: | ||
94 | taskname_list.append(taskname) | ||
95 | found.add(taskname) | ||
96 | for task in depends.keys(): | ||
97 | if color.setdefault(task, WHITE) == WHITE: | ||
98 | visit(task) | ||
99 | |||
100 | taskname_order = dict([(task, index) for index, task in enumerate(taskname_list) ]) | ||
101 | def task_key(task): | ||
102 | pn, taskname = task.rsplit(':', 1) | ||
103 | return (pn, taskname_order.get(taskname, len(taskname_list)), taskname) | ||
104 | |||
105 | for tune in sorted(tunes.keys()): | ||
106 | tasks = tunes[tune] | ||
107 | # As for test_signatures it would be nicer to sort tasks | ||
108 | # by dependencies here, but that is harder because we have | ||
109 | # to report on tasks from different machines, which might | ||
110 | # have different dependencies. We resort to pruning the | ||
111 | # output by reporting only one task per recipe if the set | ||
112 | # of machines matches. | ||
113 | # | ||
114 | # "bitbake-diffsigs -t -s" is intelligent enough to print | ||
115 | # diffs recursively, so often it does not matter that much | ||
116 | # if we don't pick the underlying difference | ||
117 | # here. However, sometimes recursion fails | ||
118 | # (https://bugzilla.yoctoproject.org/show_bug.cgi?id=6428). | ||
119 | # | ||
120 | # To mitigate that a bit, we use a hard-coded ordering of | ||
121 | # tasks that represents how they normally run and prefer | ||
122 | # to print the ones that run first. | ||
123 | for task in sorted(tasks.keys(), key=task_key): | ||
124 | signatures = tasks[task] | ||
125 | # do_build can be ignored: it is know to have | ||
126 | # different signatures in some cases, for example in | ||
127 | # the allarch ca-certificates due to RDEPENDS=openssl. | ||
128 | # That particular dependency is whitelisted via | ||
129 | # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up | ||
130 | # in the sstate signature hash because filtering it | ||
131 | # out would be hard and running do_build multiple | ||
132 | # times doesn't really matter. | ||
133 | if len(signatures.keys()) > 1 and \ | ||
134 | not task.endswith(':do_build'): | ||
135 | # Error! | ||
136 | # | ||
137 | # Sort signatures by machines, because the hex values don't mean anything. | ||
138 | # => all-arch adwaita-icon-theme:do_build: 1234... (beaglebone, qemux86) != abcdf... (qemux86-64) | ||
139 | # | ||
140 | # Skip the line if it is covered already by the predecessor (same pn, same sets of machines). | ||
141 | pn, taskname = task.rsplit(':', 1) | ||
142 | next_line_key = (pn, sorted(signatures.values())) | ||
143 | if next_line_key != last_line_key: | ||
144 | line = ' %s %s: ' % (tune, task) | ||
145 | line += ' != '.join(['%s (%s)' % (signature, ', '.join([m for m in signatures[signature]])) for | ||
146 | signature in sorted(signatures.keys(), key=lambda s: signatures[s])]) | ||
147 | last_line_key = next_line_key | ||
148 | msg.append(line) | ||
149 | # Randomly pick two mismatched signatures and remember how to invoke | ||
150 | # bitbake-diffsigs for them. | ||
151 | iterator = iter(signatures.items()) | ||
152 | a = next(iterator) | ||
153 | b = next(iterator) | ||
154 | diffsig_machines = '(%s) != (%s)' % (', '.join(a[1]), ', '.join(b[1])) | ||
155 | diffsig_params = '-t %s %s -s %s %s' % (pn, taskname, a[0], b[0]) | ||
156 | else: | ||
157 | pruned += 1 | ||
158 | |||
159 | if msg: | ||
160 | msg.insert(0, 'The machines have conflicting signatures for some shared tasks:') | ||
161 | if pruned > 0: | ||
162 | msg.append('') | ||
163 | msg.append('%d tasks where not listed because some other task of the recipe already differed.' % pruned) | ||
164 | msg.append('It is likely that differences from different recipes also have the same root cause.') | ||
165 | msg.append('') | ||
166 | # Explain how to investigate... | ||
167 | msg.append('To investigate, run bitbake-diffsigs -t recipename taskname -s fromsig tosig.') | ||
168 | cmd = 'bitbake-diffsigs %s' % diffsig_params | ||
169 | msg.append('Example: %s in the last line' % diffsig_machines) | ||
170 | msg.append('Command: %s' % cmd) | ||
171 | # ... and actually do it automatically for that example, but without aborting | ||
172 | # when that fails. | ||
173 | try: | ||
174 | output = check_command('Comparing signatures failed.', cmd).decode('utf-8') | ||
175 | except RuntimeError as ex: | ||
176 | output = str(ex) | ||
177 | msg.extend([' ' + line for line in output.splitlines()]) | ||
178 | self.fail('\n'.join(msg)) | ||