summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--scripts/lib/resulttool/regression.py16
1 files changed, 14 insertions, 2 deletions
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index 1facbcd85e..f80a9182a9 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -178,6 +178,8 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
178 base_result = base_result.get('result') 178 base_result = base_result.get('result')
179 target_result = target_result.get('result') 179 target_result = target_result.get('result')
180 result = {} 180 result = {}
181 new_tests = 0
182
181 if base_result and target_result: 183 if base_result and target_result:
182 for k in base_result: 184 for k in base_result:
183 base_testcase = base_result[k] 185 base_testcase = base_result[k]
@@ -189,6 +191,13 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
189 result[k] = {'base': base_status, 'target': target_status} 191 result[k] = {'base': base_status, 'target': target_status}
190 else: 192 else:
191 logger.error('Failed to retrieved base test case status: %s' % k) 193 logger.error('Failed to retrieved base test case status: %s' % k)
194
195 # Also count new tests that were not present in base results: it
196 # could be newly added tests, but it could also highlights some tests
197 # renames or fixed faulty ptests
198 for k in target_result:
199 if k not in base_result:
200 new_tests += 1
192 if result: 201 if result:
193 new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values()) 202 new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values())
194 # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...) 203 # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...)
@@ -200,10 +209,13 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
200 if new_pass_count > 0: 209 if new_pass_count > 0:
201 resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n' 210 resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n'
202 else: 211 else:
203 resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)" % (base_name, target_name, new_pass_count) 212 resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)\n" % (base_name, target_name, new_pass_count)
204 result = None 213 result = None
205 else: 214 else:
206 resultstring = "Match: %s\n %s" % (base_name, target_name) 215 resultstring = "Match: %s\n %s\n" % (base_name, target_name)
216
217 if new_tests > 0:
218 resultstring += f' Additionally, {new_tests} new test(s) is/are present\n'
207 return result, resultstring 219 return result, resultstring
208 220
209def get_results(logger, source): 221def get_results(logger, source):