diff options
author | Yeoh Ee Peng <ee.peng.yeoh@intel.com> | 2019-02-14 13:50:37 +0800 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-02-21 12:34:00 +0000 |
commit | 1fd5ebdb06224489ad056e261962e23ece36fc87 (patch) | |
tree | 790b33a5498a9f97642ead84ce66dfd354bd8626 /scripts/lib/resulttool/resultsutils.py | |
parent | 95bd530b772f97e7329749b403bf9e2dff12ff7f (diff) | |
download | poky-1fd5ebdb06224489ad056e261962e23ece36fc87.tar.gz |
resulttool: enable merge, store, report and regression analysis
OEQA outputs test results into json files and these files were
archived by Autobuilder during QA releases. Example: each oe-selftest
run by Autobuilder for different host distro generate a
testresults.json file.
These scripts were developed as a test result tools to manage
these testresults.json file.
Using the "store" operation, user can store multiple testresults.json
files as well as the pre-configured directories used to hold those files.
Using the "merge" operation, user can merge multiple testresults.json
files to a target file.
Using the "report" operation, user can view the test result summary
for all available testresults.json files inside a ordinary directory
or a git repository.
Using the "regression-file" operation, user can perform regression
analysis on testresults.json files specified. Using the "regression-dir"
and "regression-git" operations, user can perform regression analysis
on directory and git accordingly.
These resulttool operations expect the testresults.json file to use
the json format below.
{
"<testresult_1>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
...
"<testresult_n>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
}
To use these scripts, first source oe environment, then run the
entry point script to look for help.
$ resulttool
To store test result from oeqa automated tests, execute the below
$ resulttool store <source_dir> <git_branch>
To merge multiple testresults.json files, execute the below
$ resulttool merge <base_result_file> <target_result_file>
To report test report, execute the below
$ resulttool report <source_dir>
To perform regression file analysis, execute the below
$ resulttool regression-file <base_result_file> <target_result_file>
To perform regression dir analysis, execute the below
$ resulttool regression-dir <base_result_dir> <target_result_dir>
To perform regression git analysis, execute the below
$ resulttool regression-git <source_dir> <base_branch> <target_branch>
[YOCTO# 13012]
[YOCTO# 12654]
(From OE-Core rev: 78a322d7be402a5b9b5abf26ad35670a8535408a)
Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/resulttool/resultsutils.py')
-rw-r--r-- | scripts/lib/resulttool/resultsutils.py | 67 |
1 files changed, 67 insertions, 0 deletions
diff --git a/scripts/lib/resulttool/resultsutils.py b/scripts/lib/resulttool/resultsutils.py new file mode 100644 index 0000000000..368786922c --- /dev/null +++ b/scripts/lib/resulttool/resultsutils.py | |||
@@ -0,0 +1,67 @@ | |||
1 | # test result tool - utilities | ||
2 | # | ||
3 | # Copyright (c) 2019, Intel Corporation. | ||
4 | # | ||
5 | # This program is free software; you can redistribute it and/or modify it | ||
6 | # under the terms and conditions of the GNU General Public License, | ||
7 | # version 2, as published by the Free Software Foundation. | ||
8 | # | ||
9 | # This program is distributed in the hope it will be useful, but WITHOUT | ||
10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
11 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
12 | # more details. | ||
13 | # | ||
14 | import os | ||
15 | import json | ||
16 | import scriptpath | ||
17 | scriptpath.add_oe_lib_path() | ||
18 | from oeqa.utils.git import GitRepo, GitError | ||
19 | |||
20 | def load_json_file(file): | ||
21 | with open(file, "r") as f: | ||
22 | return json.load(f) | ||
23 | |||
24 | def dump_json_data(write_dir, file_name, json_data): | ||
25 | file_content = json.dumps(json_data, sort_keys=True, indent=4) | ||
26 | file_path = os.path.join(write_dir, file_name) | ||
27 | with open(file_path, 'w') as the_file: | ||
28 | the_file.write(file_content) | ||
29 | |||
30 | def get_dict_value(logger, dict, key): | ||
31 | try: | ||
32 | return dict[key] | ||
33 | except KeyError: | ||
34 | if logger: | ||
35 | logger.debug('Faced KeyError exception: dict=%s: key=%s' % (dict, key)) | ||
36 | return None | ||
37 | except TypeError: | ||
38 | if logger: | ||
39 | logger.debug('Faced TypeError exception: dict=%s: key=%s' % (dict, key)) | ||
40 | return None | ||
41 | |||
42 | def pop_dict_element(logger, dict, key): | ||
43 | try: | ||
44 | dict.pop(key) | ||
45 | except KeyError: | ||
46 | if logger: | ||
47 | logger.debug('Faced KeyError exception: dict=%s: key=%s' % (dict, key)) | ||
48 | except AttributeError: | ||
49 | if logger: | ||
50 | logger.debug('Faced AttributeError exception: dict=%s: key=%s' % (dict, key)) | ||
51 | |||
52 | def checkout_git_dir(git_dir, git_branch): | ||
53 | try: | ||
54 | repo = GitRepo(git_dir, is_topdir=True) | ||
55 | repo.run_cmd('checkout %s' % git_branch) | ||
56 | return True | ||
57 | except GitError: | ||
58 | return False | ||
59 | |||
60 | def get_directory_files(source_dir, excludes, file): | ||
61 | files_in_dir = [] | ||
62 | for root, dirs, files in os.walk(source_dir, topdown=True): | ||
63 | [dirs.remove(d) for d in list(dirs) if d in excludes] | ||
64 | for name in files: | ||
65 | if name == file: | ||
66 | files_in_dir.append(os.path.join(root, name)) | ||
67 | return files_in_dir \ No newline at end of file | ||