summaryrefslogtreecommitdiffstats
path: root/scripts/lib/resulttool/regression.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-16 18:13:00 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-21 12:34:00 +0000
commit47eb3d00e9d6a66aee1283dab29af8117a006d6d (patch)
tree9d574720d23da3edeab5648f05fb5d2ee95c9cd2 /scripts/lib/resulttool/regression.py
parentbeed7523b667affea71d37d88d2f5c19c935d159 (diff)
downloadpoky-47eb3d00e9d6a66aee1283dab29af8117a006d6d.tar.gz
resulttool: Improvements to allow integration to the autobuilder
This is a combined patch of the various tweaks and improvements I made to resulttool: * Avoid subprocess.run() as its a python 3.6 feature and we have autobuilder workers with 3.5. * Avoid python keywords as variable names * Simplify dict accesses using .get() * Rename resultsutils -> resultutils to match the resultstool -> resulttool rename * Formalised the handling of "file_name" to "TESTSERIES" which the code will now add into the json configuration data if its not present, based on the directory name. * When we don't have failed test cases, print something saying so instead of an empty table * Tweak the table headers in the report to be more readable (reference "Test Series" instead if file_id and ID instead of results_id) * Improve/simplify the max string length handling * Merge the counts and percentage data into one table in the report since printing two reports of the same data confuses the user * Removed the confusing header in the regression report * Show matches, then regressions, then unmatched runs in the regression report, also remove chatting unneeded output * Try harder to "pair" up matching configurations to reduce noise in the regressions report * Abstracted the "mapping" table concept used to pairing in the regression code to general code in resultutils * Created multiple mappings for results analysis, results storage and 'flattening' results data in a merge * Simplify the merge command to take a source and a destination, letting the destination be a directory or a file, removing the need for an output directory parameter * Add the 'IMAGE_PKGTYPE' and 'DISTRO' config options to the regression mappings * Have the store command place the testresults files in a layout from the mapping, making commits into the git repo for results storage more useful for simple comparison purposes * Set the oe-git-archive tag format appropriately for oeqa results storage (and simplify the commit messages closer to their defaults) * Fix oe-git-archive to use the commit/branch data from the results file * Cleaned up the command option help to match other changes * Follow the model of git branch/tag processing used by oe-build-perf-report and use that to read the data using git show to avoid branch change * Add ptest summary to the report command * Update the tests to match the above changes (From OE-Core rev: ff2c029b568f70aa9960dde04ddd207829812ea0) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/resulttool/regression.py')
-rw-r--r--scripts/lib/resulttool/regression.py328
1 files changed, 156 insertions, 172 deletions
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index bee3fb011a..ff77332fa9 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -1,6 +1,7 @@
1# test result tool - regression analysis 1# resulttool - regression analysis
2# 2#
3# Copyright (c) 2019, Intel Corporation. 3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
4# 5#
5# This program is free software; you can redistribute it and/or modify it 6# This program is free software; you can redistribute it and/or modify it
6# under the terms and conditions of the GNU General Public License, 7# under the terms and conditions of the GNU General Public License,
@@ -11,171 +12,170 @@
11# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12# more details. 13# more details.
13# 14#
14from resulttool.resultsutils import load_json_file, get_dict_value, pop_dict_element 15import resulttool.resultutils as resultutils
15import json 16import json
16 17
17class ResultsRegressionSelector(object): 18from oeqa.utils.git import GitRepo
18 19import oeqa.utils.gitarchive as gitarchive
19 def get_results_unique_configurations(self, logger, results): 20
20 unique_configurations_map = {"oeselftest": ['TEST_TYPE', 'HOST_DISTRO', 'MACHINE'], 21def compare_result(logger, base_name, target_name, base_result, target_result):
21 "runtime": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE'], 22 base_result = base_result.get('result')
22 "sdk": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], 23 target_result = target_result.get('result')
23 "sdkext": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE']} 24 result = {}
24 results_unique_configs = {} 25 if base_result and target_result:
25 for k in results: 26 for k in base_result:
26 result = results[k] 27 base_testcase = base_result[k]
27 result_configs = get_dict_value(logger, result, 'configuration') 28 base_status = base_testcase.get('status')
28 result_test_type = get_dict_value(logger, result_configs, 'TEST_TYPE') 29 if base_status:
29 unique_configuration_keys = get_dict_value(logger, unique_configurations_map, result_test_type) 30 target_testcase = target_result.get(k, {})
30 result_unique_config = {} 31 target_status = target_testcase.get('status')
31 for ck in unique_configuration_keys: 32 if base_status != target_status:
32 config_value = get_dict_value(logger, result_configs, ck) 33 result[k] = {'base': base_status, 'target': target_status}
33 if config_value: 34 else:
34 result_unique_config[ck] = config_value 35 logger.error('Failed to retrieved base test case status: %s' % k)
35 results_unique_configs[k] = result_unique_config 36 if result:
36 return results_unique_configs 37 resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
37 38 for k in result:
38 def get_regression_base_target_pair(self, logger, base_results, target_results): 39 resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
39 base_configs = self.get_results_unique_configurations(logger, base_results)
40 logger.debug('Retrieved base configuration: config=%s' % base_configs)
41 target_configs = self.get_results_unique_configurations(logger, target_results)
42 logger.debug('Retrieved target configuration: config=%s' % target_configs)
43 regression_pair = {}
44 for bk in base_configs:
45 base_config = base_configs[bk]
46 for tk in target_configs:
47 target_config = target_configs[tk]
48 if base_config == target_config:
49 if bk in regression_pair:
50 regression_pair[bk].append(tk)
51 else:
52 regression_pair[bk] = [tk]
53 return regression_pair
54
55 def run_regression_with_regression_pairing(self, logger, regression_pair, base_results, target_results):
56 regression = ResultsRegression()
57 for base in regression_pair:
58 for target in regression_pair[base]:
59 print('Getting regression for base=%s target=%s' % (base, target))
60 regression.run(logger, base_results[base], target_results[target])
61
62class ResultsRegression(object):
63
64 def print_regression_result(self, result):
65 if result:
66 print('============================Start Regression============================')
67 print('Only print regression if base status not equal target')
68 print('<test case> : <base status> -> <target status>')
69 print('========================================================================')
70 for k in result:
71 print(k, ':', result[k]['base'], '->', result[k]['target'])
72 print('==============================End Regression==============================')
73
74 def get_regression_result(self, logger, base_result, target_result):
75 base_result = get_dict_value(logger, base_result, 'result')
76 target_result = get_dict_value(logger, target_result, 'result')
77 result = {}
78 if base_result and target_result:
79 logger.debug('Getting regression result')
80 for k in base_result:
81 base_testcase = base_result[k]
82 base_status = get_dict_value(logger, base_testcase, 'status')
83 if base_status:
84 target_testcase = get_dict_value(logger, target_result, k)
85 target_status = get_dict_value(logger, target_testcase, 'status')
86 if base_status != target_status:
87 result[k] = {'base': base_status, 'target': target_status}
88 else:
89 logger.error('Failed to retrieved base test case status: %s' % k)
90 return result
91
92 def run(self, logger, base_result, target_result):
93 if base_result and target_result:
94 result = self.get_regression_result(logger, base_result, target_result)
95 logger.debug('Retrieved regression result =%s' % result)
96 self.print_regression_result(result)
97 else:
98 logger.error('Input data objects must not be empty (base_result=%s, target_result=%s)' %
99 (base_result, target_result))
100
101def get_results_from_directory(logger, source_dir):
102 from resulttool.merge import ResultsMerge
103 from resulttool.resultsutils import get_directory_files
104 result_files = get_directory_files(source_dir, ['.git'], 'testresults.json')
105 base_results = {}
106 for file in result_files:
107 merge = ResultsMerge()
108 results = merge.get_test_results(logger, file, '')
109 base_results = merge.merge_results(base_results, results)
110 return base_results
111
112def remove_testcases_to_optimize_regression_runtime(logger, results):
113 test_case_removal = ['ptestresult.rawlogs', 'ptestresult.sections']
114 for r in test_case_removal:
115 for k in results:
116 result = get_dict_value(logger, results[k], 'result')
117 pop_dict_element(logger, result, r)
118
119def regression_file(args, logger):
120 base_results = load_json_file(args.base_result_file)
121 print('Successfully loaded base test results from: %s' % args.base_result_file)
122 target_results = load_json_file(args.target_result_file)
123 print('Successfully loaded target test results from: %s' % args.target_result_file)
124 remove_testcases_to_optimize_regression_runtime(logger, base_results)
125 remove_testcases_to_optimize_regression_runtime(logger, target_results)
126 if args.base_result_id and args.target_result_id:
127 base_result = get_dict_value(logger, base_results, base_result_id)
128 print('Getting base test result with result_id=%s' % base_result_id)
129 target_result = get_dict_value(logger, target_results, target_result_id)
130 print('Getting target test result with result_id=%s' % target_result_id)
131 regression = ResultsRegression()
132 regression.run(logger, base_result, target_result)
133 else: 40 else:
134 regression = ResultsRegressionSelector() 41 resultstring = "Match: %s\n %s" % (base_name, target_name)
135 regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results) 42 return result, resultstring
136 logger.debug('Retrieved regression pair=%s' % regression_pair) 43
137 regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results) 44def get_results(logger, source):
138 return 0 45 return resultutils.load_resultsdata(source, configmap=resultutils.regression_map)
46
47def regression(args, logger):
48 base_results = get_results(logger, args.base_result)
49 target_results = get_results(logger, args.target_result)
50
51 regression_common(args, logger, base_results, target_results)
52
53def regression_common(args, logger, base_results, target_results):
54 if args.base_result_id:
55 base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
56 if args.target_result_id:
57 target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
58
59 matches = []
60 regressions = []
61 notfound = []
62
63 for a in base_results:
64 if a in target_results:
65 base = list(base_results[a].keys())
66 target = list(target_results[a].keys())
67 # We may have multiple base/targets which are for different configurations. Start by
68 # removing any pairs which match
69 for c in base.copy():
70 for b in target.copy():
71 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
72 if not res:
73 matches.append(resstr)
74 base.remove(c)
75 target.remove(b)
76 break
77 # Should only now see regressions, we may not be able to match multiple pairs directly
78 for c in base:
79 for b in target:
80 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
81 if res:
82 regressions.append(resstr)
83 else:
84 notfound.append("%s not found in target" % a)
85 print("\n".join(matches))
86 print("\n".join(regressions))
87 print("\n".join(notfound))
139 88
140def regression_directory(args, logger):
141 base_results = get_results_from_directory(logger, args.base_result_directory)
142 target_results = get_results_from_directory(logger, args.target_result_directory)
143 remove_testcases_to_optimize_regression_runtime(logger, base_results)
144 remove_testcases_to_optimize_regression_runtime(logger, target_results)
145 regression = ResultsRegressionSelector()
146 regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results)
147 logger.debug('Retrieved regression pair=%s' % regression_pair)
148 regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results)
149 return 0 89 return 0
150 90
151def regression_git(args, logger): 91def regression_git(args, logger):
152 from resulttool.resultsutils import checkout_git_dir
153 base_results = {} 92 base_results = {}
154 target_results = {} 93 target_results = {}
155 if checkout_git_dir(args.source_dir, args.base_git_branch): 94
156 base_results = get_results_from_directory(logger, args.source_dir) 95 tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}"
157 if checkout_git_dir(args.source_dir, args.target_git_branch): 96 repo = GitRepo(args.repo)
158 target_results = get_results_from_directory(logger, args.source_dir) 97
159 if base_results and target_results: 98 revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch)
160 remove_testcases_to_optimize_regression_runtime(logger, base_results) 99
161 remove_testcases_to_optimize_regression_runtime(logger, target_results) 100 if args.branch2:
162 regression = ResultsRegressionSelector() 101 revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2)
163 regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results) 102 if not len(revs2):
164 logger.debug('Retrieved regression pair=%s' % regression_pair) 103 logger.error("No revisions found to compare against")
165 regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results) 104 return 1
105 if not len(revs):
106 logger.error("No revision to report on found")
107 return 1
108 else:
109 if len(revs) < 2:
110 logger.error("Only %d tester revisions found, unable to generate report" % len(revs))
111 return 1
112
113 # Pick revisions
114 if args.commit:
115 if args.commit_number:
116 logger.warning("Ignoring --commit-number as --commit was specified")
117 index1 = gitarchive.rev_find(revs, 'commit', args.commit)
118 elif args.commit_number:
119 index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
120 else:
121 index1 = len(revs) - 1
122
123 if args.branch2:
124 revs2.append(revs[index1])
125 index1 = len(revs2) - 1
126 revs = revs2
127
128 if args.commit2:
129 if args.commit_number2:
130 logger.warning("Ignoring --commit-number2 as --commit2 was specified")
131 index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
132 elif args.commit_number2:
133 index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
134 else:
135 if index1 > 0:
136 index2 = index1 - 1
137 # Find the closest matching commit number for comparision
138 # In future we could check the commit is a common ancestor and
139 # continue back if not but this good enough for now
140 while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
141 index2 = index2 - 1
142 else:
143 logger.error("Unable to determine the other commit, use "
144 "--commit2 or --commit-number2 to specify it")
145 return 1
146
147 logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2]))
148
149 base_results = resultutils.git_get_result(repo, revs[index1][2])
150 target_results = resultutils.git_get_result(repo, revs[index2][2])
151
152 regression_common(args, logger, base_results, target_results)
153
166 return 0 154 return 0
167 155
168def register_commands(subparsers): 156def register_commands(subparsers):
169 """Register subcommands from this plugin""" 157 """Register subcommands from this plugin"""
170 parser_build = subparsers.add_parser('regression-file', help='regression file analysis', 158
159 parser_build = subparsers.add_parser('regression', help='regression file/directory analysis',
160 description='regression analysis comparing the base set of results to the target results',
161 group='analysis')
162 parser_build.set_defaults(func=regression)
163 parser_build.add_argument('base_result',
164 help='base result file/directory for the comparison')
165 parser_build.add_argument('target_result',
166 help='target result file/directory to compare with')
167 parser_build.add_argument('-b', '--base-result-id', default='',
168 help='(optional) filter the base results to this result ID')
169 parser_build.add_argument('-t', '--target-result-id', default='',
170 help='(optional) filter the target results to this result ID')
171
172 parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
171 description='regression analysis comparing base result set to target ' 173 description='regression analysis comparing base result set to target '
172 'result set', 174 'result set',
173 group='analysis') 175 group='analysis')
174 parser_build.set_defaults(func=regression_file) 176 parser_build.set_defaults(func=regression_git)
175 parser_build.add_argument('base_result_file', 177 parser_build.add_argument('repo',
176 help='base result file provide the base result set') 178 help='the git repository containing the data')
177 parser_build.add_argument('target_result_file',
178 help='target result file provide the target result set for comparison with base result')
179 parser_build.add_argument('-b', '--base-result-id', default='', 179 parser_build.add_argument('-b', '--base-result-id', default='',
180 help='(optional) default select regression based on configurations unless base result ' 180 help='(optional) default select regression based on configurations unless base result '
181 'id was provided') 181 'id was provided')
@@ -183,26 +183,10 @@ def register_commands(subparsers):
183 help='(optional) default select regression based on configurations unless target result ' 183 help='(optional) default select regression based on configurations unless target result '
184 'id was provided') 184 'id was provided')
185 185
186 parser_build = subparsers.add_parser('regression-dir', help='regression directory analysis', 186 parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
187 description='regression analysis comparing base result set to target ' 187 parser_build.add_argument('--branch2', help="Branch to find comparision revisions in")
188 'result set', 188 parser_build.add_argument('--commit', help="Revision to search for")
189 group='analysis') 189 parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
190 parser_build.set_defaults(func=regression_directory) 190 parser_build.add_argument('--commit2', help="Revision to compare with")
191 parser_build.add_argument('base_result_directory', 191 parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
192 help='base result directory provide the files for base result set')
193 parser_build.add_argument('target_result_directory',
194 help='target result file provide the files for target result set for comparison with '
195 'base result')
196 192
197 parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
198 description='regression analysis comparing base result set to target '
199 'result set',
200 group='analysis')
201 parser_build.set_defaults(func=regression_git)
202 parser_build.add_argument('source_dir',
203 help='source directory that contain the git repository with test result files')
204 parser_build.add_argument('base_git_branch',
205 help='base git branch that provide the files for base result set')
206 parser_build.add_argument('target_git_branch',
207 help='target git branch that provide the files for target result set for comparison with '
208 'base result')