diff options
-rw-r--r-- | scripts/lib/resulttool/regression.py | 16 |
1 files changed, 14 insertions, 2 deletions
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index 1facbcd85e..f80a9182a9 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py | |||
@@ -178,6 +178,8 @@ def compare_result(logger, base_name, target_name, base_result, target_result): | |||
178 | base_result = base_result.get('result') | 178 | base_result = base_result.get('result') |
179 | target_result = target_result.get('result') | 179 | target_result = target_result.get('result') |
180 | result = {} | 180 | result = {} |
181 | new_tests = 0 | ||
182 | |||
181 | if base_result and target_result: | 183 | if base_result and target_result: |
182 | for k in base_result: | 184 | for k in base_result: |
183 | base_testcase = base_result[k] | 185 | base_testcase = base_result[k] |
@@ -189,6 +191,13 @@ def compare_result(logger, base_name, target_name, base_result, target_result): | |||
189 | result[k] = {'base': base_status, 'target': target_status} | 191 | result[k] = {'base': base_status, 'target': target_status} |
190 | else: | 192 | else: |
191 | logger.error('Failed to retrieved base test case status: %s' % k) | 193 | logger.error('Failed to retrieved base test case status: %s' % k) |
194 | |||
195 | # Also count new tests that were not present in base results: it | ||
196 | # could be newly added tests, but it could also highlights some tests | ||
197 | # renames or fixed faulty ptests | ||
198 | for k in target_result: | ||
199 | if k not in base_result: | ||
200 | new_tests += 1 | ||
192 | if result: | 201 | if result: |
193 | new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values()) | 202 | new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values()) |
194 | # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...) | 203 | # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...) |
@@ -200,10 +209,13 @@ def compare_result(logger, base_name, target_name, base_result, target_result): | |||
200 | if new_pass_count > 0: | 209 | if new_pass_count > 0: |
201 | resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n' | 210 | resultstring += f' Additionally, {new_pass_count} previously failing test(s) is/are now passing\n' |
202 | else: | 211 | else: |
203 | resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)" % (base_name, target_name, new_pass_count) | 212 | resultstring = "Improvement: %s\n %s\n (+%d test(s) passing)\n" % (base_name, target_name, new_pass_count) |
204 | result = None | 213 | result = None |
205 | else: | 214 | else: |
206 | resultstring = "Match: %s\n %s" % (base_name, target_name) | 215 | resultstring = "Match: %s\n %s\n" % (base_name, target_name) |
216 | |||
217 | if new_tests > 0: | ||
218 | resultstring += f' Additionally, {new_tests} new test(s) is/are present\n' | ||
207 | return result, resultstring | 219 | return result, resultstring |
208 | 220 | ||
209 | def get_results(logger, source): | 221 | def get_results(logger, source): |