diff options
Diffstat (limited to 'meta/lib/oeqa')
-rw-r--r-- | meta/lib/oeqa/selftest/cases/resulttooltests.py | 243 |
1 files changed, 243 insertions, 0 deletions
diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py index efdfd98af3..f059991c19 100644 --- a/meta/lib/oeqa/selftest/cases/resulttooltests.py +++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py | |||
@@ -98,3 +98,246 @@ class ResultToolTests(OESelftestTestCase): | |||
98 | resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) | 98 | resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) |
99 | self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) | 99 | self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) |
100 | 100 | ||
101 | def test_results_without_metadata_can_be_compared(self): | ||
102 | base_configuration = {"configuration": { | ||
103 | "TEST_TYPE": "oeselftest", | ||
104 | "TESTSERIES": "series1", | ||
105 | "IMAGE_BASENAME": "image", | ||
106 | "IMAGE_PKGTYPE": "ipk", | ||
107 | "DISTRO": "mydistro", | ||
108 | "MACHINE": "qemux86", | ||
109 | "STARTTIME": 1672527600 | ||
110 | }, "result": {}} | ||
111 | target_configuration = {"configuration": { | ||
112 | "TEST_TYPE": "oeselftest", | ||
113 | "TESTSERIES": "series1", | ||
114 | "IMAGE_BASENAME": "image", | ||
115 | "IMAGE_PKGTYPE": "ipk", | ||
116 | "DISTRO": "mydistro", | ||
117 | "MACHINE": "qemux86", | ||
118 | "STARTTIME": 1672527600 | ||
119 | }, "result": {}} | ||
120 | self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
121 | msg="incorrect metadata filtering, tests without metadata should be compared") | ||
122 | |||
123 | def test_target_result_with_missing_metadata_can_not_be_compared(self): | ||
124 | base_configuration = {"configuration": { | ||
125 | "TEST_TYPE": "oeselftest", | ||
126 | "TESTSERIES": "series1", | ||
127 | "IMAGE_BASENAME": "image", | ||
128 | "IMAGE_PKGTYPE": "ipk", | ||
129 | "DISTRO": "mydistro", | ||
130 | "MACHINE": "qemux86", | ||
131 | "OESELFTEST_METADATA": { | ||
132 | "run_all_tests": True, | ||
133 | "run_tests": None, | ||
134 | "skips": None, | ||
135 | "machine": None, | ||
136 | "select_tags": ["toolchain-user", "toolchain-system"], | ||
137 | "exclude_tags": None | ||
138 | }}, "result": {}} | ||
139 | target_configuration = {"configuration": {"TEST_TYPE": "oeselftest", | ||
140 | "TESTSERIES": "series1", | ||
141 | "IMAGE_BASENAME": "image", | ||
142 | "IMAGE_PKGTYPE": "ipk", | ||
143 | "DISTRO": "mydistro", | ||
144 | "MACHINE": "qemux86", | ||
145 | "STARTTIME": 1672527600 | ||
146 | }, "result": {}} | ||
147 | self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
148 | msg="incorrect metadata filtering, tests should not be compared") | ||
149 | |||
150 | def test_results_with_matching_metadata_can_be_compared(self): | ||
151 | base_configuration = {"configuration": { | ||
152 | "TEST_TYPE": "oeselftest", | ||
153 | "TESTSERIES": "series1", | ||
154 | "IMAGE_BASENAME": "image", | ||
155 | "IMAGE_PKGTYPE": "ipk", | ||
156 | "DISTRO": "mydistro", | ||
157 | "MACHINE": "qemux86", | ||
158 | "STARTTIME": 1672527600, | ||
159 | "OESELFTEST_METADATA": {"run_all_tests": True, | ||
160 | "run_tests": None, | ||
161 | "skips": None, | ||
162 | "machine": None, | ||
163 | "select_tags": ["toolchain-user", "toolchain-system"], | ||
164 | "exclude_tags": None} | ||
165 | }, "result": {}} | ||
166 | target_configuration = {"configuration": { | ||
167 | "TEST_TYPE": "oeselftest", | ||
168 | "TESTSERIES": "series1", | ||
169 | "IMAGE_BASENAME": "image", | ||
170 | "IMAGE_PKGTYPE": "ipk", | ||
171 | "DISTRO": "mydistro", | ||
172 | "MACHINE": "qemux86", | ||
173 | "STARTTIME": 1672527600, | ||
174 | "OESELFTEST_METADATA": {"run_all_tests": True, | ||
175 | "run_tests": None, | ||
176 | "skips": None, | ||
177 | "machine": None, | ||
178 | "select_tags": ["toolchain-user", "toolchain-system"], | ||
179 | "exclude_tags": None} | ||
180 | }, "result": {}} | ||
181 | self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
182 | msg="incorrect metadata filtering, tests with matching metadata should be compared") | ||
183 | |||
184 | def test_results_with_mismatching_metadata_can_not_be_compared(self): | ||
185 | base_configuration = {"configuration": { | ||
186 | "TEST_TYPE": "oeselftest", | ||
187 | "TESTSERIES": "series1", | ||
188 | "IMAGE_BASENAME": "image", | ||
189 | "IMAGE_PKGTYPE": "ipk", | ||
190 | "DISTRO": "mydistro", | ||
191 | "MACHINE": "qemux86", | ||
192 | "STARTTIME": 1672527600, | ||
193 | "OESELFTEST_METADATA": {"run_all_tests": True, | ||
194 | "run_tests": None, | ||
195 | "skips": None, | ||
196 | "machine": None, | ||
197 | "select_tags": ["toolchain-user", "toolchain-system"], | ||
198 | "exclude_tags": None} | ||
199 | }, "result": {}} | ||
200 | target_configuration = {"configuration": { | ||
201 | "TEST_TYPE": "oeselftest", | ||
202 | "TESTSERIES": "series1", | ||
203 | "IMAGE_BASENAME": "image", | ||
204 | "IMAGE_PKGTYPE": "ipk", | ||
205 | "DISTRO": "mydistro", | ||
206 | "MACHINE": "qemux86", | ||
207 | "STARTTIME": 1672527600, | ||
208 | "OESELFTEST_METADATA": {"run_all_tests": True, | ||
209 | "run_tests": None, | ||
210 | "skips": None, | ||
211 | "machine": None, | ||
212 | "select_tags": ["machine"], | ||
213 | "exclude_tags": None} | ||
214 | }, "result": {}} | ||
215 | self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
216 | msg="incorrect metadata filtering, tests with mismatching metadata should not be compared") | ||
217 | |||
218 | def test_metadata_matching_is_only_checked_for_relevant_test_type(self): | ||
219 | base_configuration = {"configuration": {"TEST_TYPE": "runtime", | ||
220 | "TESTSERIES": "series1", | ||
221 | "IMAGE_BASENAME": "image", | ||
222 | "IMAGE_PKGTYPE": "ipk", | ||
223 | "DISTRO": "mydistro", | ||
224 | "MACHINE": "qemux86", | ||
225 | "STARTTIME": 1672527600, | ||
226 | "OESELFTEST_METADATA": {"run_all_tests": True, | ||
227 | "run_tests": None, | ||
228 | "skips": None, | ||
229 | "machine": None, | ||
230 | "select_tags": ["toolchain-user", "toolchain-system"], | ||
231 | "exclude_tags": None}}, "result": {}} | ||
232 | target_configuration = {"configuration": {"TEST_TYPE": "runtime", | ||
233 | "TESTSERIES": "series1", | ||
234 | "IMAGE_BASENAME": "image", | ||
235 | "IMAGE_PKGTYPE": "ipk", | ||
236 | "DISTRO": "mydistro", | ||
237 | "MACHINE": "qemux86", | ||
238 | "STARTTIME": 1672527600, | ||
239 | "OESELFTEST_METADATA": {"run_all_tests": True, | ||
240 | "run_tests": None, | ||
241 | "skips": None, | ||
242 | "machine": None, | ||
243 | "select_tags": ["machine"], | ||
244 | "exclude_tags": None}}, "result": {}} | ||
245 | self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
246 | msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['configuration']['TEST_TYPE']) | ||
247 | |||
248 | def test_machine_matches(self): | ||
249 | base_configuration = {"configuration": { | ||
250 | "TEST_TYPE": "runtime", | ||
251 | "MACHINE": "qemux86"}, "result": {}} | ||
252 | target_configuration = {"configuration": { | ||
253 | "TEST_TYPE": "runtime", | ||
254 | "MACHINE": "qemux86" | ||
255 | }, "result": {}} | ||
256 | self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
257 | msg="incorrect machine filtering, identical machine tests should be compared") | ||
258 | |||
259 | def test_machine_mismatches(self): | ||
260 | base_configuration = {"configuration": { | ||
261 | "TEST_TYPE": "runtime", | ||
262 | "MACHINE": "qemux86" | ||
263 | }, "result": {}} | ||
264 | target_configuration = {"configuration": { | ||
265 | "TEST_TYPE": "runtime", | ||
266 | "MACHINE": "qemux86_64" | ||
267 | }, "result": {}} | ||
268 | self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
269 | msg="incorrect machine filtering, mismatching machine tests should not be compared") | ||
270 | |||
271 | def test_can_not_compare_non_ltp_tests(self): | ||
272 | base_configuration = {"configuration": { | ||
273 | "TEST_TYPE": "runtime", | ||
274 | "MACHINE": "qemux86" | ||
275 | }, "result": { | ||
276 | "ltpresult_foo": { | ||
277 | "STATUS": "PASSED" | ||
278 | }}} | ||
279 | target_configuration = {"configuration": { | ||
280 | "TEST_TYPE": "runtime", | ||
281 | "MACHINE": "qemux86_64" | ||
282 | }, "result": { | ||
283 | "bar": { | ||
284 | "STATUS": "PASSED" | ||
285 | }}} | ||
286 | self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
287 | msg="incorrect ltpresult filtering, mismatching ltpresult content should not be compared") | ||
288 | |||
289 | def test_can_compare_ltp_tests(self): | ||
290 | base_configuration = {"configuration": { | ||
291 | "TEST_TYPE": "runtime", | ||
292 | "MACHINE": "qemux86" | ||
293 | }, "result": { | ||
294 | "ltpresult_foo": { | ||
295 | "STATUS": "PASSED" | ||
296 | }}} | ||
297 | target_configuration = {"configuration": { | ||
298 | "TEST_TYPE": "runtime", | ||
299 | "MACHINE": "qemux86" | ||
300 | }, "result": { | ||
301 | "ltpresult_foo": { | ||
302 | "STATUS": "PASSED" | ||
303 | }}} | ||
304 | self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
305 | msg="incorrect ltpresult filtering, matching ltpresult content should be compared") | ||
306 | |||
307 | def test_can_match_non_static_ptest_names(self): | ||
308 | base_configuration = {"configuration": { | ||
309 | "TEST_TYPE": "runtime", | ||
310 | "MACHINE": "qemux86" | ||
311 | }, "result": { | ||
312 | "ptestresult.lttng-tools.foo_-_bar_-_moo": { | ||
313 | "STATUS": "PASSED" | ||
314 | }, | ||
315 | "ptestresult.babeltrace.bar_-_moo_-_foo": { | ||
316 | "STATUS": "PASSED" | ||
317 | }, | ||
318 | "ptestresult.babletrace2.moo_-_foo_-_bar": { | ||
319 | "STATUS": "PASSED" | ||
320 | }, | ||
321 | "ptestresult.curl.test_0000__foo_out_of_bar": { | ||
322 | "STATUS": "PASSED" | ||
323 | } | ||
324 | }} | ||
325 | target_configuration = {"configuration": { | ||
326 | "TEST_TYPE": "runtime", | ||
327 | "MACHINE": "qemux86" | ||
328 | }, "result": { | ||
329 | "ptestresult.lttng-tools.xxx_-_yyy_-_zzz": { | ||
330 | "STATUS": "PASSED" | ||
331 | }, | ||
332 | "ptestresult.babeltrace.yyy_-_zzz_-_xxx": { | ||
333 | "STATUS": "PASSED" | ||
334 | }, | ||
335 | "ptestresult.babletrace2.zzz_-_xxx_-_yyy": { | ||
336 | "STATUS": "PASSED" | ||
337 | }, | ||
338 | "ptestresult.curl.test_0000__xxx_out_of_yyy": { | ||
339 | "STATUS": "PASSED" | ||
340 | } | ||
341 | }} | ||
342 | self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration), | ||
343 | msg="incorrect ptests filtering, tests shoould be compared if prefixes match") | ||