diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-02-16 18:13:00 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-02-21 12:34:00 +0000 |
commit | 47eb3d00e9d6a66aee1283dab29af8117a006d6d (patch) | |
tree | 9d574720d23da3edeab5648f05fb5d2ee95c9cd2 /scripts/lib/resulttool/regression.py | |
parent | beed7523b667affea71d37d88d2f5c19c935d159 (diff) | |
download | poky-47eb3d00e9d6a66aee1283dab29af8117a006d6d.tar.gz |
resulttool: Improvements to allow integration to the autobuilder
This is a combined patch of the various tweaks and improvements I
made to resulttool:
* Avoid subprocess.run() as its a python 3.6 feature and we
have autobuilder workers with 3.5.
* Avoid python keywords as variable names
* Simplify dict accesses using .get()
* Rename resultsutils -> resultutils to match the resultstool ->
resulttool rename
* Formalised the handling of "file_name" to "TESTSERIES" which the code
will now add into the json configuration data if its not present, based
on the directory name.
* When we don't have failed test cases, print something saying so
instead of an empty table
* Tweak the table headers in the report to be more readable (reference
"Test Series" instead if file_id and ID instead of results_id)
* Improve/simplify the max string length handling
* Merge the counts and percentage data into one table in the report
since printing two reports of the same data confuses the user
* Removed the confusing header in the regression report
* Show matches, then regressions, then unmatched runs in the regression
report, also remove chatting unneeded output
* Try harder to "pair" up matching configurations to reduce noise in
the regressions report
* Abstracted the "mapping" table concept used to pairing in the
regression code to general code in resultutils
* Created multiple mappings for results analysis, results storage and
'flattening' results data in a merge
* Simplify the merge command to take a source and a destination,
letting the destination be a directory or a file, removing the need for
an output directory parameter
* Add the 'IMAGE_PKGTYPE' and 'DISTRO' config options to the regression
mappings
* Have the store command place the testresults files in a layout from
the mapping, making commits into the git repo for results storage more
useful for simple comparison purposes
* Set the oe-git-archive tag format appropriately for oeqa results
storage (and simplify the commit messages closer to their defaults)
* Fix oe-git-archive to use the commit/branch data from the results file
* Cleaned up the command option help to match other changes
* Follow the model of git branch/tag processing used by oe-build-perf-report
and use that to read the data using git show to avoid branch change
* Add ptest summary to the report command
* Update the tests to match the above changes
(From OE-Core rev: ff2c029b568f70aa9960dde04ddd207829812ea0)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/resulttool/regression.py')
-rw-r--r-- | scripts/lib/resulttool/regression.py | 328 |
1 files changed, 156 insertions, 172 deletions
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py index bee3fb011a..ff77332fa9 100644 --- a/scripts/lib/resulttool/regression.py +++ b/scripts/lib/resulttool/regression.py | |||
@@ -1,6 +1,7 @@ | |||
1 | # test result tool - regression analysis | 1 | # resulttool - regression analysis |
2 | # | 2 | # |
3 | # Copyright (c) 2019, Intel Corporation. | 3 | # Copyright (c) 2019, Intel Corporation. |
4 | # Copyright (c) 2019, Linux Foundation | ||
4 | # | 5 | # |
5 | # This program is free software; you can redistribute it and/or modify it | 6 | # This program is free software; you can redistribute it and/or modify it |
6 | # under the terms and conditions of the GNU General Public License, | 7 | # under the terms and conditions of the GNU General Public License, |
@@ -11,171 +12,170 @@ | |||
11 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | 12 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
12 | # more details. | 13 | # more details. |
13 | # | 14 | # |
14 | from resulttool.resultsutils import load_json_file, get_dict_value, pop_dict_element | 15 | import resulttool.resultutils as resultutils |
15 | import json | 16 | import json |
16 | 17 | ||
17 | class ResultsRegressionSelector(object): | 18 | from oeqa.utils.git import GitRepo |
18 | 19 | import oeqa.utils.gitarchive as gitarchive | |
19 | def get_results_unique_configurations(self, logger, results): | 20 | |
20 | unique_configurations_map = {"oeselftest": ['TEST_TYPE', 'HOST_DISTRO', 'MACHINE'], | 21 | def compare_result(logger, base_name, target_name, base_result, target_result): |
21 | "runtime": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE'], | 22 | base_result = base_result.get('result') |
22 | "sdk": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], | 23 | target_result = target_result.get('result') |
23 | "sdkext": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE']} | 24 | result = {} |
24 | results_unique_configs = {} | 25 | if base_result and target_result: |
25 | for k in results: | 26 | for k in base_result: |
26 | result = results[k] | 27 | base_testcase = base_result[k] |
27 | result_configs = get_dict_value(logger, result, 'configuration') | 28 | base_status = base_testcase.get('status') |
28 | result_test_type = get_dict_value(logger, result_configs, 'TEST_TYPE') | 29 | if base_status: |
29 | unique_configuration_keys = get_dict_value(logger, unique_configurations_map, result_test_type) | 30 | target_testcase = target_result.get(k, {}) |
30 | result_unique_config = {} | 31 | target_status = target_testcase.get('status') |
31 | for ck in unique_configuration_keys: | 32 | if base_status != target_status: |
32 | config_value = get_dict_value(logger, result_configs, ck) | 33 | result[k] = {'base': base_status, 'target': target_status} |
33 | if config_value: | 34 | else: |
34 | result_unique_config[ck] = config_value | 35 | logger.error('Failed to retrieved base test case status: %s' % k) |
35 | results_unique_configs[k] = result_unique_config | 36 | if result: |
36 | return results_unique_configs | 37 | resultstring = "Regression: %s\n %s\n" % (base_name, target_name) |
37 | 38 | for k in result: | |
38 | def get_regression_base_target_pair(self, logger, base_results, target_results): | 39 | resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target']) |
39 | base_configs = self.get_results_unique_configurations(logger, base_results) | ||
40 | logger.debug('Retrieved base configuration: config=%s' % base_configs) | ||
41 | target_configs = self.get_results_unique_configurations(logger, target_results) | ||
42 | logger.debug('Retrieved target configuration: config=%s' % target_configs) | ||
43 | regression_pair = {} | ||
44 | for bk in base_configs: | ||
45 | base_config = base_configs[bk] | ||
46 | for tk in target_configs: | ||
47 | target_config = target_configs[tk] | ||
48 | if base_config == target_config: | ||
49 | if bk in regression_pair: | ||
50 | regression_pair[bk].append(tk) | ||
51 | else: | ||
52 | regression_pair[bk] = [tk] | ||
53 | return regression_pair | ||
54 | |||
55 | def run_regression_with_regression_pairing(self, logger, regression_pair, base_results, target_results): | ||
56 | regression = ResultsRegression() | ||
57 | for base in regression_pair: | ||
58 | for target in regression_pair[base]: | ||
59 | print('Getting regression for base=%s target=%s' % (base, target)) | ||
60 | regression.run(logger, base_results[base], target_results[target]) | ||
61 | |||
62 | class ResultsRegression(object): | ||
63 | |||
64 | def print_regression_result(self, result): | ||
65 | if result: | ||
66 | print('============================Start Regression============================') | ||
67 | print('Only print regression if base status not equal target') | ||
68 | print('<test case> : <base status> -> <target status>') | ||
69 | print('========================================================================') | ||
70 | for k in result: | ||
71 | print(k, ':', result[k]['base'], '->', result[k]['target']) | ||
72 | print('==============================End Regression==============================') | ||
73 | |||
74 | def get_regression_result(self, logger, base_result, target_result): | ||
75 | base_result = get_dict_value(logger, base_result, 'result') | ||
76 | target_result = get_dict_value(logger, target_result, 'result') | ||
77 | result = {} | ||
78 | if base_result and target_result: | ||
79 | logger.debug('Getting regression result') | ||
80 | for k in base_result: | ||
81 | base_testcase = base_result[k] | ||
82 | base_status = get_dict_value(logger, base_testcase, 'status') | ||
83 | if base_status: | ||
84 | target_testcase = get_dict_value(logger, target_result, k) | ||
85 | target_status = get_dict_value(logger, target_testcase, 'status') | ||
86 | if base_status != target_status: | ||
87 | result[k] = {'base': base_status, 'target': target_status} | ||
88 | else: | ||
89 | logger.error('Failed to retrieved base test case status: %s' % k) | ||
90 | return result | ||
91 | |||
92 | def run(self, logger, base_result, target_result): | ||
93 | if base_result and target_result: | ||
94 | result = self.get_regression_result(logger, base_result, target_result) | ||
95 | logger.debug('Retrieved regression result =%s' % result) | ||
96 | self.print_regression_result(result) | ||
97 | else: | ||
98 | logger.error('Input data objects must not be empty (base_result=%s, target_result=%s)' % | ||
99 | (base_result, target_result)) | ||
100 | |||
101 | def get_results_from_directory(logger, source_dir): | ||
102 | from resulttool.merge import ResultsMerge | ||
103 | from resulttool.resultsutils import get_directory_files | ||
104 | result_files = get_directory_files(source_dir, ['.git'], 'testresults.json') | ||
105 | base_results = {} | ||
106 | for file in result_files: | ||
107 | merge = ResultsMerge() | ||
108 | results = merge.get_test_results(logger, file, '') | ||
109 | base_results = merge.merge_results(base_results, results) | ||
110 | return base_results | ||
111 | |||
112 | def remove_testcases_to_optimize_regression_runtime(logger, results): | ||
113 | test_case_removal = ['ptestresult.rawlogs', 'ptestresult.sections'] | ||
114 | for r in test_case_removal: | ||
115 | for k in results: | ||
116 | result = get_dict_value(logger, results[k], 'result') | ||
117 | pop_dict_element(logger, result, r) | ||
118 | |||
119 | def regression_file(args, logger): | ||
120 | base_results = load_json_file(args.base_result_file) | ||
121 | print('Successfully loaded base test results from: %s' % args.base_result_file) | ||
122 | target_results = load_json_file(args.target_result_file) | ||
123 | print('Successfully loaded target test results from: %s' % args.target_result_file) | ||
124 | remove_testcases_to_optimize_regression_runtime(logger, base_results) | ||
125 | remove_testcases_to_optimize_regression_runtime(logger, target_results) | ||
126 | if args.base_result_id and args.target_result_id: | ||
127 | base_result = get_dict_value(logger, base_results, base_result_id) | ||
128 | print('Getting base test result with result_id=%s' % base_result_id) | ||
129 | target_result = get_dict_value(logger, target_results, target_result_id) | ||
130 | print('Getting target test result with result_id=%s' % target_result_id) | ||
131 | regression = ResultsRegression() | ||
132 | regression.run(logger, base_result, target_result) | ||
133 | else: | 40 | else: |
134 | regression = ResultsRegressionSelector() | 41 | resultstring = "Match: %s\n %s" % (base_name, target_name) |
135 | regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results) | 42 | return result, resultstring |
136 | logger.debug('Retrieved regression pair=%s' % regression_pair) | 43 | |
137 | regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results) | 44 | def get_results(logger, source): |
138 | return 0 | 45 | return resultutils.load_resultsdata(source, configmap=resultutils.regression_map) |
46 | |||
47 | def regression(args, logger): | ||
48 | base_results = get_results(logger, args.base_result) | ||
49 | target_results = get_results(logger, args.target_result) | ||
50 | |||
51 | regression_common(args, logger, base_results, target_results) | ||
52 | |||
53 | def regression_common(args, logger, base_results, target_results): | ||
54 | if args.base_result_id: | ||
55 | base_results = resultutils.filter_resultsdata(base_results, args.base_result_id) | ||
56 | if args.target_result_id: | ||
57 | target_results = resultutils.filter_resultsdata(target_results, args.target_result_id) | ||
58 | |||
59 | matches = [] | ||
60 | regressions = [] | ||
61 | notfound = [] | ||
62 | |||
63 | for a in base_results: | ||
64 | if a in target_results: | ||
65 | base = list(base_results[a].keys()) | ||
66 | target = list(target_results[a].keys()) | ||
67 | # We may have multiple base/targets which are for different configurations. Start by | ||
68 | # removing any pairs which match | ||
69 | for c in base.copy(): | ||
70 | for b in target.copy(): | ||
71 | res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) | ||
72 | if not res: | ||
73 | matches.append(resstr) | ||
74 | base.remove(c) | ||
75 | target.remove(b) | ||
76 | break | ||
77 | # Should only now see regressions, we may not be able to match multiple pairs directly | ||
78 | for c in base: | ||
79 | for b in target: | ||
80 | res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) | ||
81 | if res: | ||
82 | regressions.append(resstr) | ||
83 | else: | ||
84 | notfound.append("%s not found in target" % a) | ||
85 | print("\n".join(matches)) | ||
86 | print("\n".join(regressions)) | ||
87 | print("\n".join(notfound)) | ||
139 | 88 | ||
140 | def regression_directory(args, logger): | ||
141 | base_results = get_results_from_directory(logger, args.base_result_directory) | ||
142 | target_results = get_results_from_directory(logger, args.target_result_directory) | ||
143 | remove_testcases_to_optimize_regression_runtime(logger, base_results) | ||
144 | remove_testcases_to_optimize_regression_runtime(logger, target_results) | ||
145 | regression = ResultsRegressionSelector() | ||
146 | regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results) | ||
147 | logger.debug('Retrieved regression pair=%s' % regression_pair) | ||
148 | regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results) | ||
149 | return 0 | 89 | return 0 |
150 | 90 | ||
151 | def regression_git(args, logger): | 91 | def regression_git(args, logger): |
152 | from resulttool.resultsutils import checkout_git_dir | ||
153 | base_results = {} | 92 | base_results = {} |
154 | target_results = {} | 93 | target_results = {} |
155 | if checkout_git_dir(args.source_dir, args.base_git_branch): | 94 | |
156 | base_results = get_results_from_directory(logger, args.source_dir) | 95 | tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}" |
157 | if checkout_git_dir(args.source_dir, args.target_git_branch): | 96 | repo = GitRepo(args.repo) |
158 | target_results = get_results_from_directory(logger, args.source_dir) | 97 | |
159 | if base_results and target_results: | 98 | revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch) |
160 | remove_testcases_to_optimize_regression_runtime(logger, base_results) | 99 | |
161 | remove_testcases_to_optimize_regression_runtime(logger, target_results) | 100 | if args.branch2: |
162 | regression = ResultsRegressionSelector() | 101 | revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2) |
163 | regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results) | 102 | if not len(revs2): |
164 | logger.debug('Retrieved regression pair=%s' % regression_pair) | 103 | logger.error("No revisions found to compare against") |
165 | regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results) | 104 | return 1 |
105 | if not len(revs): | ||
106 | logger.error("No revision to report on found") | ||
107 | return 1 | ||
108 | else: | ||
109 | if len(revs) < 2: | ||
110 | logger.error("Only %d tester revisions found, unable to generate report" % len(revs)) | ||
111 | return 1 | ||
112 | |||
113 | # Pick revisions | ||
114 | if args.commit: | ||
115 | if args.commit_number: | ||
116 | logger.warning("Ignoring --commit-number as --commit was specified") | ||
117 | index1 = gitarchive.rev_find(revs, 'commit', args.commit) | ||
118 | elif args.commit_number: | ||
119 | index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number) | ||
120 | else: | ||
121 | index1 = len(revs) - 1 | ||
122 | |||
123 | if args.branch2: | ||
124 | revs2.append(revs[index1]) | ||
125 | index1 = len(revs2) - 1 | ||
126 | revs = revs2 | ||
127 | |||
128 | if args.commit2: | ||
129 | if args.commit_number2: | ||
130 | logger.warning("Ignoring --commit-number2 as --commit2 was specified") | ||
131 | index2 = gitarchive.rev_find(revs, 'commit', args.commit2) | ||
132 | elif args.commit_number2: | ||
133 | index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2) | ||
134 | else: | ||
135 | if index1 > 0: | ||
136 | index2 = index1 - 1 | ||
137 | # Find the closest matching commit number for comparision | ||
138 | # In future we could check the commit is a common ancestor and | ||
139 | # continue back if not but this good enough for now | ||
140 | while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number: | ||
141 | index2 = index2 - 1 | ||
142 | else: | ||
143 | logger.error("Unable to determine the other commit, use " | ||
144 | "--commit2 or --commit-number2 to specify it") | ||
145 | return 1 | ||
146 | |||
147 | logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2])) | ||
148 | |||
149 | base_results = resultutils.git_get_result(repo, revs[index1][2]) | ||
150 | target_results = resultutils.git_get_result(repo, revs[index2][2]) | ||
151 | |||
152 | regression_common(args, logger, base_results, target_results) | ||
153 | |||
166 | return 0 | 154 | return 0 |
167 | 155 | ||
168 | def register_commands(subparsers): | 156 | def register_commands(subparsers): |
169 | """Register subcommands from this plugin""" | 157 | """Register subcommands from this plugin""" |
170 | parser_build = subparsers.add_parser('regression-file', help='regression file analysis', | 158 | |
159 | parser_build = subparsers.add_parser('regression', help='regression file/directory analysis', | ||
160 | description='regression analysis comparing the base set of results to the target results', | ||
161 | group='analysis') | ||
162 | parser_build.set_defaults(func=regression) | ||
163 | parser_build.add_argument('base_result', | ||
164 | help='base result file/directory for the comparison') | ||
165 | parser_build.add_argument('target_result', | ||
166 | help='target result file/directory to compare with') | ||
167 | parser_build.add_argument('-b', '--base-result-id', default='', | ||
168 | help='(optional) filter the base results to this result ID') | ||
169 | parser_build.add_argument('-t', '--target-result-id', default='', | ||
170 | help='(optional) filter the target results to this result ID') | ||
171 | |||
172 | parser_build = subparsers.add_parser('regression-git', help='regression git analysis', | ||
171 | description='regression analysis comparing base result set to target ' | 173 | description='regression analysis comparing base result set to target ' |
172 | 'result set', | 174 | 'result set', |
173 | group='analysis') | 175 | group='analysis') |
174 | parser_build.set_defaults(func=regression_file) | 176 | parser_build.set_defaults(func=regression_git) |
175 | parser_build.add_argument('base_result_file', | 177 | parser_build.add_argument('repo', |
176 | help='base result file provide the base result set') | 178 | help='the git repository containing the data') |
177 | parser_build.add_argument('target_result_file', | ||
178 | help='target result file provide the target result set for comparison with base result') | ||
179 | parser_build.add_argument('-b', '--base-result-id', default='', | 179 | parser_build.add_argument('-b', '--base-result-id', default='', |
180 | help='(optional) default select regression based on configurations unless base result ' | 180 | help='(optional) default select regression based on configurations unless base result ' |
181 | 'id was provided') | 181 | 'id was provided') |
@@ -183,26 +183,10 @@ def register_commands(subparsers): | |||
183 | help='(optional) default select regression based on configurations unless target result ' | 183 | help='(optional) default select regression based on configurations unless target result ' |
184 | 'id was provided') | 184 | 'id was provided') |
185 | 185 | ||
186 | parser_build = subparsers.add_parser('regression-dir', help='regression directory analysis', | 186 | parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in") |
187 | description='regression analysis comparing base result set to target ' | 187 | parser_build.add_argument('--branch2', help="Branch to find comparision revisions in") |
188 | 'result set', | 188 | parser_build.add_argument('--commit', help="Revision to search for") |
189 | group='analysis') | 189 | parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified") |
190 | parser_build.set_defaults(func=regression_directory) | 190 | parser_build.add_argument('--commit2', help="Revision to compare with") |
191 | parser_build.add_argument('base_result_directory', | 191 | parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified") |
192 | help='base result directory provide the files for base result set') | ||
193 | parser_build.add_argument('target_result_directory', | ||
194 | help='target result file provide the files for target result set for comparison with ' | ||
195 | 'base result') | ||
196 | 192 | ||
197 | parser_build = subparsers.add_parser('regression-git', help='regression git analysis', | ||
198 | description='regression analysis comparing base result set to target ' | ||
199 | 'result set', | ||
200 | group='analysis') | ||
201 | parser_build.set_defaults(func=regression_git) | ||
202 | parser_build.add_argument('source_dir', | ||
203 | help='source directory that contain the git repository with test result files') | ||
204 | parser_build.add_argument('base_git_branch', | ||
205 | help='base git branch that provide the files for base result set') | ||
206 | parser_build.add_argument('target_git_branch', | ||
207 | help='target git branch that provide the files for target result set for comparison with ' | ||
208 | 'base result') | ||