diff options
author | Yeoh Ee Peng <ee.peng.yeoh@intel.com> | 2019-02-14 13:50:37 +0800 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2019-03-26 15:38:22 +0000 |
commit | 661fa1335b6832a1d13b13e8ca4354cdef213358 (patch) | |
tree | 9d465a03e55d4906197db32d0d19c181e9ff42dd /scripts/lib/resulttool/store.py | |
parent | e4c0a8a7cb1b57486952912ba0c4f5b7c582d93d (diff) | |
download | poky-661fa1335b6832a1d13b13e8ca4354cdef213358.tar.gz |
resulttool: enable merge, store, report and regression analysis
OEQA outputs test results into json files and these files were
archived by Autobuilder during QA releases. Example: each oe-selftest
run by Autobuilder for different host distro generate a
testresults.json file.
These scripts were developed as a test result tools to manage
these testresults.json file.
Using the "store" operation, user can store multiple testresults.json
files as well as the pre-configured directories used to hold those files.
Using the "merge" operation, user can merge multiple testresults.json
files to a target file.
Using the "report" operation, user can view the test result summary
for all available testresults.json files inside a ordinary directory
or a git repository.
Using the "regression-file" operation, user can perform regression
analysis on testresults.json files specified. Using the "regression-dir"
and "regression-git" operations, user can perform regression analysis
on directory and git accordingly.
These resulttool operations expect the testresults.json file to use
the json format below.
{
"<testresult_1>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
...
"<testresult_n>": {
"configuration": {
"<config_name_1>": "<config_value_1>",
"<config_name_2>": "<config_value_2>",
...
"<config_name_n>": "<config_value_n>",
},
"result": {
"<testcase_namespace_1>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
"<testcase_namespace_2>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
...
"<testcase_namespace_n>": {
"status": "<PASSED or FAILED or ERROR or SKIPPED>",
"log": "<failure or error logging>"
},
}
},
}
To use these scripts, first source oe environment, then run the
entry point script to look for help.
$ resulttool
To store test result from oeqa automated tests, execute the below
$ resulttool store <source_dir> <git_branch>
To merge multiple testresults.json files, execute the below
$ resulttool merge <base_result_file> <target_result_file>
To report test report, execute the below
$ resulttool report <source_dir>
To perform regression file analysis, execute the below
$ resulttool regression-file <base_result_file> <target_result_file>
To perform regression dir analysis, execute the below
$ resulttool regression-dir <base_result_dir> <target_result_dir>
To perform regression git analysis, execute the below
$ resulttool regression-git <source_dir> <base_branch> <target_branch>
[YOCTO# 13012]
[YOCTO# 12654]
(From OE-Core rev: 39cda3498ec68b91a671ead256231324d74f7d4c)
Signed-off-by: Yeoh Ee Peng <ee.peng.yeoh@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/resulttool/store.py')
-rw-r--r-- | scripts/lib/resulttool/store.py | 110 |
1 files changed, 110 insertions, 0 deletions
diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py new file mode 100644 index 0000000000..2c6fd8492c --- /dev/null +++ b/scripts/lib/resulttool/store.py | |||
@@ -0,0 +1,110 @@ | |||
1 | # test result tool - store test results | ||
2 | # | ||
3 | # Copyright (c) 2019, Intel Corporation. | ||
4 | # | ||
5 | # This program is free software; you can redistribute it and/or modify it | ||
6 | # under the terms and conditions of the GNU General Public License, | ||
7 | # version 2, as published by the Free Software Foundation. | ||
8 | # | ||
9 | # This program is distributed in the hope it will be useful, but WITHOUT | ||
10 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
11 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
12 | # more details. | ||
13 | # | ||
14 | import datetime | ||
15 | import tempfile | ||
16 | import os | ||
17 | import subprocess | ||
18 | import scriptpath | ||
19 | scriptpath.add_bitbake_lib_path() | ||
20 | scriptpath.add_oe_lib_path() | ||
21 | from resulttool.resultsutils import checkout_git_dir | ||
22 | try: | ||
23 | import bb | ||
24 | except ImportError: | ||
25 | pass | ||
26 | |||
27 | class ResultsGitStore(object): | ||
28 | |||
29 | def _get_output_dir(self): | ||
30 | basepath = os.environ['BUILDDIR'] | ||
31 | return basepath + '/testresults_%s/' % datetime.datetime.now().strftime("%Y%m%d%H%M%S") | ||
32 | |||
33 | def _create_temporary_workspace_dir(self): | ||
34 | return tempfile.mkdtemp(prefix='testresults.') | ||
35 | |||
36 | def _remove_temporary_workspace_dir(self, workspace_dir): | ||
37 | return subprocess.run(["rm", "-rf", workspace_dir]) | ||
38 | |||
39 | def _oe_copy_files(self, source_dir, destination_dir): | ||
40 | from oe.path import copytree | ||
41 | copytree(source_dir, destination_dir) | ||
42 | |||
43 | def _copy_files(self, source_dir, destination_dir, copy_ignore=None): | ||
44 | from shutil import copytree | ||
45 | copytree(source_dir, destination_dir, ignore=copy_ignore) | ||
46 | |||
47 | def _store_files_to_git(self, logger, file_dir, git_dir, git_branch, commit_msg_subject, commit_msg_body): | ||
48 | logger.debug('Storing test result into git repository (%s) and branch (%s)' | ||
49 | % (git_dir, git_branch)) | ||
50 | return subprocess.run(["oe-git-archive", | ||
51 | file_dir, | ||
52 | "-g", git_dir, | ||
53 | "-b", git_branch, | ||
54 | "--commit-msg-subject", commit_msg_subject, | ||
55 | "--commit-msg-body", commit_msg_body]) | ||
56 | |||
57 | def store_to_existing(self, logger, source_dir, git_dir, git_branch): | ||
58 | logger.debug('Storing files to existing git repository and branch') | ||
59 | from shutil import ignore_patterns | ||
60 | dest_dir = self._create_temporary_workspace_dir() | ||
61 | dest_top_dir = os.path.join(dest_dir, 'top_dir') | ||
62 | self._copy_files(git_dir, dest_top_dir, copy_ignore=ignore_patterns('.git')) | ||
63 | self._oe_copy_files(source_dir, dest_top_dir) | ||
64 | self._store_files_to_git(logger, dest_top_dir, git_dir, git_branch, | ||
65 | 'Store as existing git and branch', 'Store as existing git repository and branch') | ||
66 | self._remove_temporary_workspace_dir(dest_dir) | ||
67 | return git_dir | ||
68 | |||
69 | def store_to_existing_with_new_branch(self, logger, source_dir, git_dir, git_branch): | ||
70 | logger.debug('Storing files to existing git repository with new branch') | ||
71 | self._store_files_to_git(logger, source_dir, git_dir, git_branch, | ||
72 | 'Store as existing git with new branch', | ||
73 | 'Store as existing git repository with new branch') | ||
74 | return git_dir | ||
75 | |||
76 | def store_to_new(self, logger, source_dir, git_branch): | ||
77 | logger.debug('Storing files to new git repository') | ||
78 | output_dir = self._get_output_dir() | ||
79 | self._store_files_to_git(logger, source_dir, output_dir, git_branch, | ||
80 | 'Store as new', 'Store as new git repository') | ||
81 | return output_dir | ||
82 | |||
83 | def store(self, logger, source_dir, git_dir, git_branch): | ||
84 | if git_dir: | ||
85 | if checkout_git_dir(git_dir, git_branch): | ||
86 | self.store_to_existing(logger, source_dir, git_dir, git_branch) | ||
87 | else: | ||
88 | self.store_to_existing_with_new_branch(logger, source_dir, git_dir, git_branch) | ||
89 | else: | ||
90 | self.store_to_new(logger, source_dir, git_branch) | ||
91 | |||
92 | def store(args, logger): | ||
93 | gitstore = ResultsGitStore() | ||
94 | gitstore.store(logger, args.source_dir, args.git_dir, args.git_branch) | ||
95 | return 0 | ||
96 | |||
97 | def register_commands(subparsers): | ||
98 | """Register subcommands from this plugin""" | ||
99 | parser_build = subparsers.add_parser('store', help='store test result files and directories into git repository', | ||
100 | description='store the testresults.json files and related directories ' | ||
101 | 'from the source directory into the destination git repository ' | ||
102 | 'with the given git branch', | ||
103 | group='setup') | ||
104 | parser_build.set_defaults(func=store) | ||
105 | parser_build.add_argument('source_dir', | ||
106 | help='source directory that contain the test result files and directories to be stored') | ||
107 | parser_build.add_argument('git_branch', help='git branch used for store') | ||
108 | parser_build.add_argument('-d', '--git-dir', default='', | ||
109 | help='(optional) default store to new <top_dir>/<build>/<testresults_datetime> ' | ||
110 | 'directory unless provided with existing git repository as destination') | ||