diff options
author | Yeoh Ee Peng <ee.peng.yeoh@intel.com> | 2019-02-14 13:50:37 +0800 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-02-21 12:34:00 +0000 |
commit | 1fd5ebdb06224489ad056e261962e23ece36fc87 (patch) | |
tree | 790b33a5498a9f97642ead84ce66dfd354bd8626 /scripts/lib/resulttool/report.py | |
parent | 95bd530b772f97e7329749b403bf9e2dff12ff7f (diff) | |
download | poky-1fd5ebdb06224489ad056e261962e23ece36fc87.tar.gz |
resulttool: enable merge, store, report and regression analysis
OEQA outputs test results into json files and these files were
archived by Autobuilder during QA releases. Example: each oe-selftest
run by Autobuilder for different host distro generate a
testresults.json file.
These scripts were developed as a test result tools to manage
these testresults.json file.
Using the "store" operation, user can store multiple testresults.json
files as well as the pre-configured directories used to hold those files.
Using the "merge" operation, user can merge multiple testresults.json
files to a target file.
Using the "report" operation, user can view the test result summary
for all available testresults.json files inside a ordinary directory
or a git repository.
Using the "regression-file" operation, user can perform regression
analysis on testresults.json files specified. Using the "regression-dir"
and "regression-git" operations, user can perform regression analysis
on directory and git accordingly.
These resulttool operations expect the testresults.json file to use
the json format below.
{
"<testresult_1>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
...
"<testresult_n>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
}
To use these scripts, first source oe environment, then run the
entry point script to look for help.
$ resulttool
To store test result from oeqa automated tests, execute the below
$ resulttool store <source_dir> <git_branch>
To merge multiple testresults.json files, execute the below
$ resulttool merge <base_result_file> <target_result_file>
To report test report, execute the below
$ resulttool report <source_dir>
To perform regression file analysis, execute the below
$ resulttool regression-file <base_result_file> <target_result_file>
To perform regression dir analysis, execute the below
$ resulttool regression-dir <base_result_dir> <target_result_dir>
To perform regression git analysis, execute the below
$ resulttool regression-git <source_dir> <base_branch> <target_branch>
[YOCTO# 13012]
[YOCTO# 12654]
(From OE-Core rev: 78a322d7be402a5b9b5abf26ad35670a8535408a)
Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/resulttool/report.py')
-rw-r--r-- | scripts/lib/resulttool/report.py | 113 |
1 files changed, 113 insertions, 0 deletions
diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py new file mode 100644 index 0000000000..ab5de1f3a7 --- /dev/null +++ b/scripts/lib/resulttool/report.py | |||
@@ -0,0 +1,113 @@ | |||
1 | # test result tool - report text based test results | ||
2 | # | ||
3 | # Copyright (c) 2019, Intel Corporation. | ||
4 | # | ||
5 | # This program is free software; you can redistribute it and/or modify it | ||
6 | # under the terms and conditions of the GNU General Public License, | ||
7 | # version 2, as published by the Free Software Foundation. | ||
8 | # | ||
9 | # This program is distributed in the hope it will be useful, but WITHOUT | ||
10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
11 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
12 | # more details. | ||
13 | # | ||
14 | import os | ||
15 | import glob | ||
16 | import json | ||
17 | from resulttool.resultsutils import checkout_git_dir, load_json_file, get_dict_value, get_directory_files | ||
18 | |||
19 | class ResultsTextReport(object): | ||
20 | |||
21 | def get_aggregated_test_result(self, logger, testresult): | ||
22 | test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} | ||
23 | result_types = {'passed': ['PASSED', 'passed'], | ||
24 | 'failed': ['FAILED', 'failed', 'ERROR', 'error', 'UNKNOWN'], | ||
25 | 'skipped': ['SKIPPED', 'skipped']} | ||
26 | result = get_dict_value(logger, testresult, 'result') | ||
27 | for k in result: | ||
28 | test_status = get_dict_value(logger, result[k], 'status') | ||
29 | for tk in result_types: | ||
30 | if test_status in result_types[tk]: | ||
31 | test_count_report[tk] += 1 | ||
32 | if test_status in result_types['failed']: | ||
33 | test_count_report['failed_testcases'].append(k) | ||
34 | return test_count_report | ||
35 | |||
36 | def get_test_result_percentage(self, test_result_count): | ||
37 | total_tested = test_result_count['passed'] + test_result_count['failed'] + test_result_count['skipped'] | ||
38 | test_percent_report = {'passed': 0, 'failed': 0, 'skipped': 0} | ||
39 | for k in test_percent_report: | ||
40 | test_percent_report[k] = format(test_result_count[k] / total_tested * 100, '.2f') | ||
41 | return test_percent_report | ||
42 | |||
43 | def add_test_configurations(self, test_report, source_dir, file, result_id): | ||
44 | test_report['file_dir'] = self._get_short_file_dir(source_dir, file) | ||
45 | test_report['result_id'] = result_id | ||
46 | test_report['test_file_dir_result_id'] = '%s_%s' % (test_report['file_dir'], test_report['result_id']) | ||
47 | |||
48 | def _get_short_file_dir(self, source_dir, file): | ||
49 | file_dir = os.path.dirname(file) | ||
50 | source_dir = source_dir[:-1] if source_dir[-1] == '/' else source_dir | ||
51 | if file_dir == source_dir: | ||
52 | return 'None' | ||
53 | return file_dir.replace(source_dir, '') | ||
54 | |||
55 | def get_max_string_len(self, test_result_list, key, default_max_len): | ||
56 | max_len = default_max_len | ||
57 | for test_result in test_result_list: | ||
58 | value_len = len(test_result[key]) | ||
59 | if value_len > max_len: | ||
60 | max_len = value_len | ||
61 | return max_len | ||
62 | |||
63 | def print_test_report(self, template_file_name, test_count_reports, test_percent_reports, | ||
64 | max_len_dir, max_len_result_id): | ||
65 | from jinja2 import Environment, FileSystemLoader | ||
66 | script_path = os.path.dirname(os.path.realpath(__file__)) | ||
67 | file_loader = FileSystemLoader(script_path + '/template') | ||
68 | env = Environment(loader=file_loader, trim_blocks=True) | ||
69 | template = env.get_template(template_file_name) | ||
70 | output = template.render(test_count_reports=test_count_reports, | ||
71 | test_percent_reports=test_percent_reports, | ||
72 | max_len_dir=max_len_dir, | ||
73 | max_len_result_id=max_len_result_id) | ||
74 | print('Printing text-based test report:') | ||
75 | print(output) | ||
76 | |||
77 | def view_test_report(self, logger, source_dir, git_branch): | ||
78 | if git_branch: | ||
79 | checkout_git_dir(source_dir, git_branch) | ||
80 | test_count_reports = [] | ||
81 | test_percent_reports = [] | ||
82 | for file in get_directory_files(source_dir, ['.git'], 'testresults.json'): | ||
83 | logger.debug('Computing result for test result file: %s' % file) | ||
84 | testresults = load_json_file(file) | ||
85 | for k in testresults: | ||
86 | test_count_report = self.get_aggregated_test_result(logger, testresults[k]) | ||
87 | test_percent_report = self.get_test_result_percentage(test_count_report) | ||
88 | self.add_test_configurations(test_count_report, source_dir, file, k) | ||
89 | self.add_test_configurations(test_percent_report, source_dir, file, k) | ||
90 | test_count_reports.append(test_count_report) | ||
91 | test_percent_reports.append(test_percent_report) | ||
92 | max_len_dir = self.get_max_string_len(test_count_reports, 'file_dir', len('file_dir')) | ||
93 | max_len_result_id = self.get_max_string_len(test_count_reports, 'result_id', len('result_id')) | ||
94 | self.print_test_report('test_report_full_text.txt', test_count_reports, test_percent_reports, | ||
95 | max_len_dir, max_len_result_id) | ||
96 | |||
97 | def report(args, logger): | ||
98 | report = ResultsTextReport() | ||
99 | report.view_test_report(logger, args.source_dir, args.git_branch) | ||
100 | return 0 | ||
101 | |||
102 | def register_commands(subparsers): | ||
103 | """Register subcommands from this plugin""" | ||
104 | parser_build = subparsers.add_parser('report', help='report test result summary', | ||
105 | description='report text-based test result summary from the source directory', | ||
106 | group='analysis') | ||
107 | parser_build.set_defaults(func=report) | ||
108 | parser_build.add_argument('source_dir', | ||
109 | help='source directory that contain the test result files for reporting') | ||
110 | parser_build.add_argument('-b', '--git-branch', default='', | ||
111 | help='(optional) default assume source directory contains all available files for ' | ||
112 | 'reporting unless a git branch was provided where it will try to checkout ' | ||
113 | 'the provided git branch assuming source directory was a git repository') | ||