summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexis Lothoré <alexis.lothore@bootlin.com>2023-02-27 20:42:23 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2023-02-28 07:53:54 +0000
commit9b8665bb96568cba19744697aa42c1bbafc3ba04 (patch)
tree8ba33f896d9fd9c0bfce7da874526211fde54854
parentd0f32772bb412837028623808a4df3cd16f8780a (diff)
downloadpoky-9b8665bb96568cba19744697aa42c1bbafc3ba04.tar.gz
oeqa/selftest/resulttool: add test for metadata filtering on regression
Multiple filters have been added to resulttool to ensure that test results are compared only to relevant older tests results Add some unit tests about added filters: - tests should only be compared when machine matches - tests run as oeselftest should only be compared when OESELFTEST_METADATA matches - many ptests should match despite non-static names - runtime tests should match with matching tests regarding ltp tests content (From OE-Core rev: b84302ef56f2516742a496aef43b89d4c3decd37) Signed-off-by: Alexis Lothoré <alexis.lothore@bootlin.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat
-rw-r--r--meta/lib/oeqa/selftest/cases/resulttooltests.py243
1 files changed, 243 insertions, 0 deletions
diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index efdfd98af3..f059991c19 100644
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -98,3 +98,246 @@ class ResultToolTests(OESelftestTestCase):
98 resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map) 98 resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
99 self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results)) 99 self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results))
100 100
101 def test_results_without_metadata_can_be_compared(self):
102 base_configuration = {"configuration": {
103 "TEST_TYPE": "oeselftest",
104 "TESTSERIES": "series1",
105 "IMAGE_BASENAME": "image",
106 "IMAGE_PKGTYPE": "ipk",
107 "DISTRO": "mydistro",
108 "MACHINE": "qemux86",
109 "STARTTIME": 1672527600
110 }, "result": {}}
111 target_configuration = {"configuration": {
112 "TEST_TYPE": "oeselftest",
113 "TESTSERIES": "series1",
114 "IMAGE_BASENAME": "image",
115 "IMAGE_PKGTYPE": "ipk",
116 "DISTRO": "mydistro",
117 "MACHINE": "qemux86",
118 "STARTTIME": 1672527600
119 }, "result": {}}
120 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
121 msg="incorrect metadata filtering, tests without metadata should be compared")
122
123 def test_target_result_with_missing_metadata_can_not_be_compared(self):
124 base_configuration = {"configuration": {
125 "TEST_TYPE": "oeselftest",
126 "TESTSERIES": "series1",
127 "IMAGE_BASENAME": "image",
128 "IMAGE_PKGTYPE": "ipk",
129 "DISTRO": "mydistro",
130 "MACHINE": "qemux86",
131 "OESELFTEST_METADATA": {
132 "run_all_tests": True,
133 "run_tests": None,
134 "skips": None,
135 "machine": None,
136 "select_tags": ["toolchain-user", "toolchain-system"],
137 "exclude_tags": None
138 }}, "result": {}}
139 target_configuration = {"configuration": {"TEST_TYPE": "oeselftest",
140 "TESTSERIES": "series1",
141 "IMAGE_BASENAME": "image",
142 "IMAGE_PKGTYPE": "ipk",
143 "DISTRO": "mydistro",
144 "MACHINE": "qemux86",
145 "STARTTIME": 1672527600
146 }, "result": {}}
147 self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
148 msg="incorrect metadata filtering, tests should not be compared")
149
150 def test_results_with_matching_metadata_can_be_compared(self):
151 base_configuration = {"configuration": {
152 "TEST_TYPE": "oeselftest",
153 "TESTSERIES": "series1",
154 "IMAGE_BASENAME": "image",
155 "IMAGE_PKGTYPE": "ipk",
156 "DISTRO": "mydistro",
157 "MACHINE": "qemux86",
158 "STARTTIME": 1672527600,
159 "OESELFTEST_METADATA": {"run_all_tests": True,
160 "run_tests": None,
161 "skips": None,
162 "machine": None,
163 "select_tags": ["toolchain-user", "toolchain-system"],
164 "exclude_tags": None}
165 }, "result": {}}
166 target_configuration = {"configuration": {
167 "TEST_TYPE": "oeselftest",
168 "TESTSERIES": "series1",
169 "IMAGE_BASENAME": "image",
170 "IMAGE_PKGTYPE": "ipk",
171 "DISTRO": "mydistro",
172 "MACHINE": "qemux86",
173 "STARTTIME": 1672527600,
174 "OESELFTEST_METADATA": {"run_all_tests": True,
175 "run_tests": None,
176 "skips": None,
177 "machine": None,
178 "select_tags": ["toolchain-user", "toolchain-system"],
179 "exclude_tags": None}
180 }, "result": {}}
181 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
182 msg="incorrect metadata filtering, tests with matching metadata should be compared")
183
184 def test_results_with_mismatching_metadata_can_not_be_compared(self):
185 base_configuration = {"configuration": {
186 "TEST_TYPE": "oeselftest",
187 "TESTSERIES": "series1",
188 "IMAGE_BASENAME": "image",
189 "IMAGE_PKGTYPE": "ipk",
190 "DISTRO": "mydistro",
191 "MACHINE": "qemux86",
192 "STARTTIME": 1672527600,
193 "OESELFTEST_METADATA": {"run_all_tests": True,
194 "run_tests": None,
195 "skips": None,
196 "machine": None,
197 "select_tags": ["toolchain-user", "toolchain-system"],
198 "exclude_tags": None}
199 }, "result": {}}
200 target_configuration = {"configuration": {
201 "TEST_TYPE": "oeselftest",
202 "TESTSERIES": "series1",
203 "IMAGE_BASENAME": "image",
204 "IMAGE_PKGTYPE": "ipk",
205 "DISTRO": "mydistro",
206 "MACHINE": "qemux86",
207 "STARTTIME": 1672527600,
208 "OESELFTEST_METADATA": {"run_all_tests": True,
209 "run_tests": None,
210 "skips": None,
211 "machine": None,
212 "select_tags": ["machine"],
213 "exclude_tags": None}
214 }, "result": {}}
215 self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
216 msg="incorrect metadata filtering, tests with mismatching metadata should not be compared")
217
218 def test_metadata_matching_is_only_checked_for_relevant_test_type(self):
219 base_configuration = {"configuration": {"TEST_TYPE": "runtime",
220 "TESTSERIES": "series1",
221 "IMAGE_BASENAME": "image",
222 "IMAGE_PKGTYPE": "ipk",
223 "DISTRO": "mydistro",
224 "MACHINE": "qemux86",
225 "STARTTIME": 1672527600,
226 "OESELFTEST_METADATA": {"run_all_tests": True,
227 "run_tests": None,
228 "skips": None,
229 "machine": None,
230 "select_tags": ["toolchain-user", "toolchain-system"],
231 "exclude_tags": None}}, "result": {}}
232 target_configuration = {"configuration": {"TEST_TYPE": "runtime",
233 "TESTSERIES": "series1",
234 "IMAGE_BASENAME": "image",
235 "IMAGE_PKGTYPE": "ipk",
236 "DISTRO": "mydistro",
237 "MACHINE": "qemux86",
238 "STARTTIME": 1672527600,
239 "OESELFTEST_METADATA": {"run_all_tests": True,
240 "run_tests": None,
241 "skips": None,
242 "machine": None,
243 "select_tags": ["machine"],
244 "exclude_tags": None}}, "result": {}}
245 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
246 msg="incorrect metadata filtering, %s tests should be compared" % base_configuration['configuration']['TEST_TYPE'])
247
248 def test_machine_matches(self):
249 base_configuration = {"configuration": {
250 "TEST_TYPE": "runtime",
251 "MACHINE": "qemux86"}, "result": {}}
252 target_configuration = {"configuration": {
253 "TEST_TYPE": "runtime",
254 "MACHINE": "qemux86"
255 }, "result": {}}
256 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
257 msg="incorrect machine filtering, identical machine tests should be compared")
258
259 def test_machine_mismatches(self):
260 base_configuration = {"configuration": {
261 "TEST_TYPE": "runtime",
262 "MACHINE": "qemux86"
263 }, "result": {}}
264 target_configuration = {"configuration": {
265 "TEST_TYPE": "runtime",
266 "MACHINE": "qemux86_64"
267 }, "result": {}}
268 self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
269 msg="incorrect machine filtering, mismatching machine tests should not be compared")
270
271 def test_can_not_compare_non_ltp_tests(self):
272 base_configuration = {"configuration": {
273 "TEST_TYPE": "runtime",
274 "MACHINE": "qemux86"
275 }, "result": {
276 "ltpresult_foo": {
277 "STATUS": "PASSED"
278 }}}
279 target_configuration = {"configuration": {
280 "TEST_TYPE": "runtime",
281 "MACHINE": "qemux86_64"
282 }, "result": {
283 "bar": {
284 "STATUS": "PASSED"
285 }}}
286 self.assertFalse(regression.can_be_compared(self.logger, base_configuration, target_configuration),
287 msg="incorrect ltpresult filtering, mismatching ltpresult content should not be compared")
288
289 def test_can_compare_ltp_tests(self):
290 base_configuration = {"configuration": {
291 "TEST_TYPE": "runtime",
292 "MACHINE": "qemux86"
293 }, "result": {
294 "ltpresult_foo": {
295 "STATUS": "PASSED"
296 }}}
297 target_configuration = {"configuration": {
298 "TEST_TYPE": "runtime",
299 "MACHINE": "qemux86"
300 }, "result": {
301 "ltpresult_foo": {
302 "STATUS": "PASSED"
303 }}}
304 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
305 msg="incorrect ltpresult filtering, matching ltpresult content should be compared")
306
307 def test_can_match_non_static_ptest_names(self):
308 base_configuration = {"configuration": {
309 "TEST_TYPE": "runtime",
310 "MACHINE": "qemux86"
311 }, "result": {
312 "ptestresult.lttng-tools.foo_-_bar_-_moo": {
313 "STATUS": "PASSED"
314 },
315 "ptestresult.babeltrace.bar_-_moo_-_foo": {
316 "STATUS": "PASSED"
317 },
318 "ptestresult.babletrace2.moo_-_foo_-_bar": {
319 "STATUS": "PASSED"
320 },
321 "ptestresult.curl.test_0000__foo_out_of_bar": {
322 "STATUS": "PASSED"
323 }
324 }}
325 target_configuration = {"configuration": {
326 "TEST_TYPE": "runtime",
327 "MACHINE": "qemux86"
328 }, "result": {
329 "ptestresult.lttng-tools.xxx_-_yyy_-_zzz": {
330 "STATUS": "PASSED"
331 },
332 "ptestresult.babeltrace.yyy_-_zzz_-_xxx": {
333 "STATUS": "PASSED"
334 },
335 "ptestresult.babletrace2.zzz_-_xxx_-_yyy": {
336 "STATUS": "PASSED"
337 },
338 "ptestresult.curl.test_0000__xxx_out_of_yyy": {
339 "STATUS": "PASSED"
340 }
341 }}
342 self.assertTrue(regression.can_be_compared(self.logger, base_configuration, target_configuration),
343 msg="incorrect ptests filtering, tests shoould be compared if prefixes match")
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-05-25 19:51:12 +0000