summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-16 18:13:00 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-21 12:34:00 +0000
commit47eb3d00e9d6a66aee1283dab29af8117a006d6d (patch)
tree9d574720d23da3edeab5648f05fb5d2ee95c9cd2
parentbeed7523b667affea71d37d88d2f5c19c935d159 (diff)
downloadpoky-47eb3d00e9d6a66aee1283dab29af8117a006d6d.tar.gz
resulttool: Improvements to allow integration to the autobuilder
This is a combined patch of the various tweaks and improvements I made to resulttool: * Avoid subprocess.run() as its a python 3.6 feature and we have autobuilder workers with 3.5. * Avoid python keywords as variable names * Simplify dict accesses using .get() * Rename resultsutils -> resultutils to match the resultstool -> resulttool rename * Formalised the handling of "file_name" to "TESTSERIES" which the code will now add into the json configuration data if its not present, based on the directory name. * When we don't have failed test cases, print something saying so instead of an empty table * Tweak the table headers in the report to be more readable (reference "Test Series" instead if file_id and ID instead of results_id) * Improve/simplify the max string length handling * Merge the counts and percentage data into one table in the report since printing two reports of the same data confuses the user * Removed the confusing header in the regression report * Show matches, then regressions, then unmatched runs in the regression report, also remove chatting unneeded output * Try harder to "pair" up matching configurations to reduce noise in the regressions report * Abstracted the "mapping" table concept used to pairing in the regression code to general code in resultutils * Created multiple mappings for results analysis, results storage and 'flattening' results data in a merge * Simplify the merge command to take a source and a destination, letting the destination be a directory or a file, removing the need for an output directory parameter * Add the 'IMAGE_PKGTYPE' and 'DISTRO' config options to the regression mappings * Have the store command place the testresults files in a layout from the mapping, making commits into the git repo for results storage more useful for simple comparison purposes * Set the oe-git-archive tag format appropriately for oeqa results storage (and simplify the commit messages closer to their defaults) * Fix oe-git-archive to use the commit/branch data from the results file * Cleaned up the command option help to match other changes * Follow the model of git branch/tag processing used by oe-build-perf-report and use that to read the data using git show to avoid branch change * Add ptest summary to the report command * Update the tests to match the above changes (From OE-Core rev: ff2c029b568f70aa9960dde04ddd207829812ea0) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat
-rw-r--r--meta/lib/oeqa/selftest/cases/resulttooltests.py106
-rwxr-xr-xscripts/lib/resulttool/manualexecution.py8
-rw-r--r--scripts/lib/resulttool/merge.py69
-rw-r--r--scripts/lib/resulttool/regression.py328
-rw-r--r--scripts/lib/resulttool/report.py157
-rw-r--r--scripts/lib/resulttool/resultsutils.py67
-rw-r--r--scripts/lib/resulttool/resultutils.py127
-rw-r--r--scripts/lib/resulttool/store.py136
-rw-r--r--scripts/lib/resulttool/template/test_report_full_text.txt33
-rwxr-xr-xscripts/resulttool7
10 files changed, 530 insertions, 508 deletions
diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index 7bf1ec60c1..0a089c0b7f 100644
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -4,13 +4,46 @@ basepath = os.path.abspath(os.path.dirname(__file__) + '/../../../../../')
4lib_path = basepath + '/scripts/lib' 4lib_path = basepath + '/scripts/lib'
5sys.path = sys.path + [lib_path] 5sys.path = sys.path + [lib_path]
6from resulttool.report import ResultsTextReport 6from resulttool.report import ResultsTextReport
7from resulttool.regression import ResultsRegressionSelector, ResultsRegression 7from resulttool import regression as regression
8from resulttool.merge import ResultsMerge 8from resulttool import resultutils as resultutils
9from resulttool.store import ResultsGitStore
10from resulttool.resultsutils import checkout_git_dir
11from oeqa.selftest.case import OESelftestTestCase 9from oeqa.selftest.case import OESelftestTestCase
12 10
13class ResultToolTests(OESelftestTestCase): 11class ResultToolTests(OESelftestTestCase):
12 base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "runtime",
13 "TESTSERIES": "series1",
14 "IMAGE_BASENAME": "image",
15 "IMAGE_PKGTYPE": "ipk",
16 "DISTRO": "mydistro",
17 "MACHINE": "qemux86"},
18 'result': {}},
19 'base_result2': {'configuration': {"TEST_TYPE": "runtime",
20 "TESTSERIES": "series1",
21 "IMAGE_BASENAME": "image",
22 "IMAGE_PKGTYPE": "ipk",
23 "DISTRO": "mydistro",
24 "MACHINE": "qemux86-64"},
25 'result': {}}}
26 target_results_data = {'target_result1': {'configuration': {"TEST_TYPE": "runtime",
27 "TESTSERIES": "series1",
28 "IMAGE_BASENAME": "image",
29 "IMAGE_PKGTYPE": "ipk",
30 "DISTRO": "mydistro",
31 "MACHINE": "qemux86"},
32 'result': {}},
33 'target_result2': {'configuration': {"TEST_TYPE": "runtime",
34 "TESTSERIES": "series1",
35 "IMAGE_BASENAME": "image",
36 "IMAGE_PKGTYPE": "ipk",
37 "DISTRO": "mydistro",
38 "MACHINE": "qemux86"},
39 'result': {}},
40 'target_result3': {'configuration': {"TEST_TYPE": "runtime",
41 "TESTSERIES": "series1",
42 "IMAGE_BASENAME": "image",
43 "IMAGE_PKGTYPE": "ipk",
44 "DISTRO": "mydistro",
45 "MACHINE": "qemux86-64"},
46 'result': {}}}
14 47
15 def test_report_can_aggregate_test_result(self): 48 def test_report_can_aggregate_test_result(self):
16 result_data = {'result': {'test1': {'status': 'PASSED'}, 49 result_data = {'result': {'test1': {'status': 'PASSED'},
@@ -25,23 +58,12 @@ class ResultToolTests(OESelftestTestCase):
25 self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped']) 58 self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped'])
26 59
27 def test_regression_can_get_regression_base_target_pair(self): 60 def test_regression_can_get_regression_base_target_pair(self):
28 base_results_data = {'base_result1': {'configuration': {"TEST_TYPE": "oeselftest", 61
29 "HOST": "centos-7"}}, 62 results = {}
30 'base_result2': {'configuration': {"TEST_TYPE": "oeselftest", 63 resultutils.append_resultsdata(results, ResultToolTests.base_results_data)
31 "HOST": "centos-7", 64 resultutils.append_resultsdata(results, ResultToolTests.target_results_data)
32 "MACHINE": "qemux86-64"}}} 65 self.assertTrue('target_result1' in results['runtime/mydistro/qemux86/image'], msg="Pair not correct:%s" % results)
33 target_results_data = {'target_result1': {'configuration': {"TEST_TYPE": "oeselftest", 66 self.assertTrue('target_result3' in results['runtime/mydistro/qemux86-64/image'], msg="Pair not correct:%s" % results)
34 "HOST": "centos-7"}},
35 'target_result2': {'configuration': {"TEST_TYPE": "oeselftest",
36 "HOST": "centos-7",
37 "MACHINE": "qemux86"}},
38 'target_result3': {'configuration': {"TEST_TYPE": "oeselftest",
39 "HOST": "centos-7",
40 "MACHINE": "qemux86-64"}}}
41 regression = ResultsRegressionSelector()
42 pair = regression.get_regression_base_target_pair(self.logger, base_results_data, target_results_data)
43 self.assertTrue('target_result1' in pair['base_result1'], msg="Pair not correct:%s" % pair['base_result1'])
44 self.assertTrue('target_result3' in pair['base_result2'], msg="Pair not correct:%s" % pair['base_result2'])
45 67
46 def test_regrresion_can_get_regression_result(self): 68 def test_regrresion_can_get_regression_result(self):
47 base_result_data = {'result': {'test1': {'status': 'PASSED'}, 69 base_result_data = {'result': {'test1': {'status': 'PASSED'},
@@ -54,8 +76,7 @@ class ResultToolTests(OESelftestTestCase):
54 'test3': {'status': 'PASSED'}, 76 'test3': {'status': 'PASSED'},
55 'test4': {'status': 'ERROR'}, 77 'test4': {'status': 'ERROR'},
56 'test5': {'status': 'SKIPPED'}}} 78 'test5': {'status': 'SKIPPED'}}}
57 regression = ResultsRegression() 79 result, text = regression.compare_result(self.logger, "BaseTestRunName", "TargetTestRunName", base_result_data, target_result_data)
58 result = regression.get_regression_result(self.logger, base_result_data, target_result_data)
59 self.assertTrue(result['test2']['base'] == 'PASSED', 80 self.assertTrue(result['test2']['base'] == 'PASSED',
60 msg="regression not correct:%s" % result['test2']['base']) 81 msg="regression not correct:%s" % result['test2']['base'])
61 self.assertTrue(result['test2']['target'] == 'FAILED', 82 self.assertTrue(result['test2']['target'] == 'FAILED',
@@ -66,39 +87,8 @@ class ResultToolTests(OESelftestTestCase):
66 msg="regression not correct:%s" % result['test3']['target']) 87 msg="regression not correct:%s" % result['test3']['target'])
67 88
68 def test_merge_can_merged_results(self): 89 def test_merge_can_merged_results(self):
69 base_results_data = {'base_result1': {}, 90 results = {}
70 'base_result2': {}} 91 resultutils.append_resultsdata(results, ResultToolTests.base_results_data, configmap=resultutils.flatten_map)
71 target_results_data = {'target_result1': {}, 92 resultutils.append_resultsdata(results, ResultToolTests.target_results_data, configmap=resultutils.flatten_map)
72 'target_result2': {}, 93 self.assertEqual(len(results[''].keys()), 5, msg="Flattened results not correct %s" % str(results))
73 'target_result3': {}}
74
75 merge = ResultsMerge()
76 results = merge.merge_results(base_results_data, target_results_data)
77 self.assertTrue(len(results.keys()) == 5, msg="merge not correct:%s" % len(results.keys()))
78
79 def test_store_can_store_to_new_git_repository(self):
80 basepath = os.path.abspath(os.path.dirname(__file__) + '/../../')
81 source_dir = basepath + '/files/testresults'
82 git_branch = 'qa-cycle-2.7'
83 store = ResultsGitStore()
84 output_dir = store.store_to_new(self.logger, source_dir, git_branch)
85 self.assertTrue(checkout_git_dir(output_dir, git_branch), msg="store to new git repository failed:%s" %
86 output_dir)
87 store._remove_temporary_workspace_dir(output_dir)
88 94
89 def test_store_can_store_to_existing(self):
90 basepath = os.path.abspath(os.path.dirname(__file__) + '/../../')
91 source_dir = basepath + '/files/testresults'
92 git_branch = 'qa-cycle-2.6'
93 store = ResultsGitStore()
94 output_dir = store.store_to_new(self.logger, source_dir, git_branch)
95 self.assertTrue(checkout_git_dir(output_dir, git_branch), msg="store to new git repository failed:%s" %
96 output_dir)
97 git_branch = 'qa-cycle-2.7'
98 output_dir = store.store_to_existing_with_new_branch(self.logger, source_dir, output_dir, git_branch)
99 self.assertTrue(checkout_git_dir(output_dir, git_branch), msg="store to existing git repository failed:%s" %
100 output_dir)
101 output_dir = store.store_to_existing(self.logger, source_dir, output_dir, git_branch)
102 self.assertTrue(checkout_git_dir(output_dir, git_branch), msg="store to existing git repository failed:%s" %
103 output_dir)
104 store._remove_temporary_workspace_dir(output_dir)
diff --git a/scripts/lib/resulttool/manualexecution.py b/scripts/lib/resulttool/manualexecution.py
index 64ec581a9f..ecdc4e7a7a 100755
--- a/scripts/lib/resulttool/manualexecution.py
+++ b/scripts/lib/resulttool/manualexecution.py
@@ -18,7 +18,11 @@ import sys
18import datetime 18import datetime
19import re 19import re
20from oeqa.core.runner import OETestResultJSONHelper 20from oeqa.core.runner import OETestResultJSONHelper
21from resulttool.resultsutils import load_json_file 21
22def load_json_file(file):
23 with open(file, "r") as f:
24 return json.load(f)
25
22 26
23class ManualTestRunner(object): 27class ManualTestRunner(object):
24 def __init__(self): 28 def __init__(self):
@@ -134,4 +138,4 @@ def register_commands(subparsers):
134 description='helper script for results populating during manual test execution. You can find manual test case JSON file in meta/lib/oeqa/manual/', 138 description='helper script for results populating during manual test execution. You can find manual test case JSON file in meta/lib/oeqa/manual/',
135 group='manualexecution') 139 group='manualexecution')
136 parser_build.set_defaults(func=manualexecution) 140 parser_build.set_defaults(func=manualexecution)
137 parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.') \ No newline at end of file 141 parser_build.add_argument('file', help='specify path to manual test case JSON file.Note: Please use \"\" to encapsulate the file path.')
diff --git a/scripts/lib/resulttool/merge.py b/scripts/lib/resulttool/merge.py
index 1d9cfafd41..3e4b7a38ad 100644
--- a/scripts/lib/resulttool/merge.py
+++ b/scripts/lib/resulttool/merge.py
@@ -1,6 +1,7 @@
1# test result tool - merge multiple testresults.json files 1# resulttool - merge multiple testresults.json files into a file or directory
2# 2#
3# Copyright (c) 2019, Intel Corporation. 3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
4# 5#
5# This program is free software; you can redistribute it and/or modify it 6# This program is free software; you can redistribute it and/or modify it
6# under the terms and conditions of the GNU General Public License, 7# under the terms and conditions of the GNU General Public License,
@@ -11,61 +12,31 @@
11# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12# more details. 13# more details.
13# 14#
14from resulttool.resultsutils import load_json_file, get_dict_value, dump_json_data
15import os 15import os
16import json 16import json
17 17import resulttool.resultutils as resultutils
18class ResultsMerge(object):
19
20 def get_test_results(self, logger, file, result_id):
21 results = load_json_file(file)
22 if result_id:
23 result = get_dict_value(logger, results, result_id)
24 if result:
25 return {result_id: result}
26 return result
27 return results
28
29 def merge_results(self, base_results, target_results):
30 for k in target_results:
31 base_results[k] = target_results[k]
32 return base_results
33
34 def _get_write_dir(self):
35 basepath = os.environ['BUILDDIR']
36 return basepath + '/tmp/'
37
38 def dump_merged_results(self, results, output_dir):
39 file_output_dir = output_dir if output_dir else self._get_write_dir()
40 dump_json_data(file_output_dir, 'testresults.json', results)
41 print('Successfully merged results to: %s' % os.path.join(file_output_dir, 'testresults.json'))
42
43 def run(self, logger, base_result_file, target_result_file, target_result_id, output_dir):
44 base_results = self.get_test_results(logger, base_result_file, '')
45 target_results = self.get_test_results(logger, target_result_file, target_result_id)
46 if base_results and target_results:
47 merged_results = self.merge_results(base_results, target_results)
48 self.dump_merged_results(merged_results, output_dir)
49 18
50def merge(args, logger): 19def merge(args, logger):
51 merge = ResultsMerge() 20 if os.path.isdir(args.target_results):
52 merge.run(logger, args.base_result_file, args.target_result_file, args.target_result_id, args.output_dir) 21 results = resultutils.load_resultsdata(args.target_results, configmap=resultutils.store_map)
22 resultutils.append_resultsdata(results, args.base_results, configmap=resultutils.store_map)
23 resultutils.save_resultsdata(results, args.target_results)
24 else:
25 results = resultutils.load_resultsdata(args.base_results, configmap=resultutils.flatten_map)
26 if os.path.exists(args.target_results):
27 resultutils.append_resultsdata(results, args.target_results, configmap=resultutils.flatten_map)
28 resultutils.save_resultsdata(results, os.path.dirname(args.target_results), fn=os.path.basename(args.target_results))
29
53 return 0 30 return 0
54 31
55def register_commands(subparsers): 32def register_commands(subparsers):
56 """Register subcommands from this plugin""" 33 """Register subcommands from this plugin"""
57 parser_build = subparsers.add_parser('merge', help='merge test results', 34 parser_build = subparsers.add_parser('merge', help='merge test result files/directories',
58 description='merge results from multiple files', 35 description='merge the results from multiple files/directories into the target file or directory',
59 group='setup') 36 group='setup')
60 parser_build.set_defaults(func=merge) 37 parser_build.set_defaults(func=merge)
61 parser_build.add_argument('base_result_file', 38 parser_build.add_argument('base_results',
62 help='base result file provide the base result set') 39 help='the results file/directory to import')
63 parser_build.add_argument('target_result_file', 40 parser_build.add_argument('target_results',
64 help='target result file provide the target result set for merging into the ' 41 help='the target file or directory to merge the base_results with')
65 'base result set') 42
66 parser_build.add_argument('-t', '--target-result-id', default='',
67 help='(optional) default merge all result sets available from target to base '
68 'unless specific target result id was provided')
69 parser_build.add_argument('-o', '--output-dir', default='',
70 help='(optional) default write merged results to <poky>/build/tmp/ unless specific '
71 'output directory was provided')
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index bee3fb011a..ff77332fa9 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -1,6 +1,7 @@
1# test result tool - regression analysis 1# resulttool - regression analysis
2# 2#
3# Copyright (c) 2019, Intel Corporation. 3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
4# 5#
5# This program is free software; you can redistribute it and/or modify it 6# This program is free software; you can redistribute it and/or modify it
6# under the terms and conditions of the GNU General Public License, 7# under the terms and conditions of the GNU General Public License,
@@ -11,171 +12,170 @@
11# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12# more details. 13# more details.
13# 14#
14from resulttool.resultsutils import load_json_file, get_dict_value, pop_dict_element 15import resulttool.resultutils as resultutils
15import json 16import json
16 17
17class ResultsRegressionSelector(object): 18from oeqa.utils.git import GitRepo
18 19import oeqa.utils.gitarchive as gitarchive
19 def get_results_unique_configurations(self, logger, results): 20
20 unique_configurations_map = {"oeselftest": ['TEST_TYPE', 'HOST_DISTRO', 'MACHINE'], 21def compare_result(logger, base_name, target_name, base_result, target_result):
21 "runtime": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE'], 22 base_result = base_result.get('result')
22 "sdk": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'], 23 target_result = target_result.get('result')
23 "sdkext": ['TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE']} 24 result = {}
24 results_unique_configs = {} 25 if base_result and target_result:
25 for k in results: 26 for k in base_result:
26 result = results[k] 27 base_testcase = base_result[k]
27 result_configs = get_dict_value(logger, result, 'configuration') 28 base_status = base_testcase.get('status')
28 result_test_type = get_dict_value(logger, result_configs, 'TEST_TYPE') 29 if base_status:
29 unique_configuration_keys = get_dict_value(logger, unique_configurations_map, result_test_type) 30 target_testcase = target_result.get(k, {})
30 result_unique_config = {} 31 target_status = target_testcase.get('status')
31 for ck in unique_configuration_keys: 32 if base_status != target_status:
32 config_value = get_dict_value(logger, result_configs, ck) 33 result[k] = {'base': base_status, 'target': target_status}
33 if config_value: 34 else:
34 result_unique_config[ck] = config_value 35 logger.error('Failed to retrieved base test case status: %s' % k)
35 results_unique_configs[k] = result_unique_config 36 if result:
36 return results_unique_configs 37 resultstring = "Regression: %s\n %s\n" % (base_name, target_name)
37 38 for k in result:
38 def get_regression_base_target_pair(self, logger, base_results, target_results): 39 resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
39 base_configs = self.get_results_unique_configurations(logger, base_results)
40 logger.debug('Retrieved base configuration: config=%s' % base_configs)
41 target_configs = self.get_results_unique_configurations(logger, target_results)
42 logger.debug('Retrieved target configuration: config=%s' % target_configs)
43 regression_pair = {}
44 for bk in base_configs:
45 base_config = base_configs[bk]
46 for tk in target_configs:
47 target_config = target_configs[tk]
48 if base_config == target_config:
49 if bk in regression_pair:
50 regression_pair[bk].append(tk)
51 else:
52 regression_pair[bk] = [tk]
53 return regression_pair
54
55 def run_regression_with_regression_pairing(self, logger, regression_pair, base_results, target_results):
56 regression = ResultsRegression()
57 for base in regression_pair:
58 for target in regression_pair[base]:
59 print('Getting regression for base=%s target=%s' % (base, target))
60 regression.run(logger, base_results[base], target_results[target])
61
62class ResultsRegression(object):
63
64 def print_regression_result(self, result):
65 if result:
66 print('============================Start Regression============================')
67 print('Only print regression if base status not equal target')
68 print('<test case> : <base status> -> <target status>')
69 print('========================================================================')
70 for k in result:
71 print(k, ':', result[k]['base'], '->', result[k]['target'])
72 print('==============================End Regression==============================')
73
74 def get_regression_result(self, logger, base_result, target_result):
75 base_result = get_dict_value(logger, base_result, 'result')
76 target_result = get_dict_value(logger, target_result, 'result')
77 result = {}
78 if base_result and target_result:
79 logger.debug('Getting regression result')
80 for k in base_result:
81 base_testcase = base_result[k]
82 base_status = get_dict_value(logger, base_testcase, 'status')
83 if base_status:
84 target_testcase = get_dict_value(logger, target_result, k)
85 target_status = get_dict_value(logger, target_testcase, 'status')
86 if base_status != target_status:
87 result[k] = {'base': base_status, 'target': target_status}
88 else:
89 logger.error('Failed to retrieved base test case status: %s' % k)
90 return result
91
92 def run(self, logger, base_result, target_result):
93 if base_result and target_result:
94 result = self.get_regression_result(logger, base_result, target_result)
95 logger.debug('Retrieved regression result =%s' % result)
96 self.print_regression_result(result)
97 else:
98 logger.error('Input data objects must not be empty (base_result=%s, target_result=%s)' %
99 (base_result, target_result))
100
101def get_results_from_directory(logger, source_dir):
102 from resulttool.merge import ResultsMerge
103 from resulttool.resultsutils import get_directory_files
104 result_files = get_directory_files(source_dir, ['.git'], 'testresults.json')
105 base_results = {}
106 for file in result_files:
107 merge = ResultsMerge()
108 results = merge.get_test_results(logger, file, '')
109 base_results = merge.merge_results(base_results, results)
110 return base_results
111
112def remove_testcases_to_optimize_regression_runtime(logger, results):
113 test_case_removal = ['ptestresult.rawlogs', 'ptestresult.sections']
114 for r in test_case_removal:
115 for k in results:
116 result = get_dict_value(logger, results[k], 'result')
117 pop_dict_element(logger, result, r)
118
119def regression_file(args, logger):
120 base_results = load_json_file(args.base_result_file)
121 print('Successfully loaded base test results from: %s' % args.base_result_file)
122 target_results = load_json_file(args.target_result_file)
123 print('Successfully loaded target test results from: %s' % args.target_result_file)
124 remove_testcases_to_optimize_regression_runtime(logger, base_results)
125 remove_testcases_to_optimize_regression_runtime(logger, target_results)
126 if args.base_result_id and args.target_result_id:
127 base_result = get_dict_value(logger, base_results, base_result_id)
128 print('Getting base test result with result_id=%s' % base_result_id)
129 target_result = get_dict_value(logger, target_results, target_result_id)
130 print('Getting target test result with result_id=%s' % target_result_id)
131 regression = ResultsRegression()
132 regression.run(logger, base_result, target_result)
133 else: 40 else:
134 regression = ResultsRegressionSelector() 41 resultstring = "Match: %s\n %s" % (base_name, target_name)
135 regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results) 42 return result, resultstring
136 logger.debug('Retrieved regression pair=%s' % regression_pair) 43
137 regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results) 44def get_results(logger, source):
138 return 0 45 return resultutils.load_resultsdata(source, configmap=resultutils.regression_map)
46
47def regression(args, logger):
48 base_results = get_results(logger, args.base_result)
49 target_results = get_results(logger, args.target_result)
50
51 regression_common(args, logger, base_results, target_results)
52
53def regression_common(args, logger, base_results, target_results):
54 if args.base_result_id:
55 base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
56 if args.target_result_id:
57 target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
58
59 matches = []
60 regressions = []
61 notfound = []
62
63 for a in base_results:
64 if a in target_results:
65 base = list(base_results[a].keys())
66 target = list(target_results[a].keys())
67 # We may have multiple base/targets which are for different configurations. Start by
68 # removing any pairs which match
69 for c in base.copy():
70 for b in target.copy():
71 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
72 if not res:
73 matches.append(resstr)
74 base.remove(c)
75 target.remove(b)
76 break
77 # Should only now see regressions, we may not be able to match multiple pairs directly
78 for c in base:
79 for b in target:
80 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
81 if res:
82 regressions.append(resstr)
83 else:
84 notfound.append("%s not found in target" % a)
85 print("\n".join(matches))
86 print("\n".join(regressions))
87 print("\n".join(notfound))
139 88
140def regression_directory(args, logger):
141 base_results = get_results_from_directory(logger, args.base_result_directory)
142 target_results = get_results_from_directory(logger, args.target_result_directory)
143 remove_testcases_to_optimize_regression_runtime(logger, base_results)
144 remove_testcases_to_optimize_regression_runtime(logger, target_results)
145 regression = ResultsRegressionSelector()
146 regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results)
147 logger.debug('Retrieved regression pair=%s' % regression_pair)
148 regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results)
149 return 0 89 return 0
150 90
151def regression_git(args, logger): 91def regression_git(args, logger):
152 from resulttool.resultsutils import checkout_git_dir
153 base_results = {} 92 base_results = {}
154 target_results = {} 93 target_results = {}
155 if checkout_git_dir(args.source_dir, args.base_git_branch): 94
156 base_results = get_results_from_directory(logger, args.source_dir) 95 tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}"
157 if checkout_git_dir(args.source_dir, args.target_git_branch): 96 repo = GitRepo(args.repo)
158 target_results = get_results_from_directory(logger, args.source_dir) 97
159 if base_results and target_results: 98 revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch)
160 remove_testcases_to_optimize_regression_runtime(logger, base_results) 99
161 remove_testcases_to_optimize_regression_runtime(logger, target_results) 100 if args.branch2:
162 regression = ResultsRegressionSelector() 101 revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2)
163 regression_pair = regression.get_regression_base_target_pair(logger, base_results, target_results) 102 if not len(revs2):
164 logger.debug('Retrieved regression pair=%s' % regression_pair) 103 logger.error("No revisions found to compare against")
165 regression.run_regression_with_regression_pairing(logger, regression_pair, base_results, target_results) 104 return 1
105 if not len(revs):
106 logger.error("No revision to report on found")
107 return 1
108 else:
109 if len(revs) < 2:
110 logger.error("Only %d tester revisions found, unable to generate report" % len(revs))
111 return 1
112
113 # Pick revisions
114 if args.commit:
115 if args.commit_number:
116 logger.warning("Ignoring --commit-number as --commit was specified")
117 index1 = gitarchive.rev_find(revs, 'commit', args.commit)
118 elif args.commit_number:
119 index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
120 else:
121 index1 = len(revs) - 1
122
123 if args.branch2:
124 revs2.append(revs[index1])
125 index1 = len(revs2) - 1
126 revs = revs2
127
128 if args.commit2:
129 if args.commit_number2:
130 logger.warning("Ignoring --commit-number2 as --commit2 was specified")
131 index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
132 elif args.commit_number2:
133 index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
134 else:
135 if index1 > 0:
136 index2 = index1 - 1
137 # Find the closest matching commit number for comparision
138 # In future we could check the commit is a common ancestor and
139 # continue back if not but this good enough for now
140 while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
141 index2 = index2 - 1
142 else:
143 logger.error("Unable to determine the other commit, use "
144 "--commit2 or --commit-number2 to specify it")
145 return 1
146
147 logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2]))
148
149 base_results = resultutils.git_get_result(repo, revs[index1][2])
150 target_results = resultutils.git_get_result(repo, revs[index2][2])
151
152 regression_common(args, logger, base_results, target_results)
153
166 return 0 154 return 0
167 155
168def register_commands(subparsers): 156def register_commands(subparsers):
169 """Register subcommands from this plugin""" 157 """Register subcommands from this plugin"""
170 parser_build = subparsers.add_parser('regression-file', help='regression file analysis', 158
159 parser_build = subparsers.add_parser('regression', help='regression file/directory analysis',
160 description='regression analysis comparing the base set of results to the target results',
161 group='analysis')
162 parser_build.set_defaults(func=regression)
163 parser_build.add_argument('base_result',
164 help='base result file/directory for the comparison')
165 parser_build.add_argument('target_result',
166 help='target result file/directory to compare with')
167 parser_build.add_argument('-b', '--base-result-id', default='',
168 help='(optional) filter the base results to this result ID')
169 parser_build.add_argument('-t', '--target-result-id', default='',
170 help='(optional) filter the target results to this result ID')
171
172 parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
171 description='regression analysis comparing base result set to target ' 173 description='regression analysis comparing base result set to target '
172 'result set', 174 'result set',
173 group='analysis') 175 group='analysis')
174 parser_build.set_defaults(func=regression_file) 176 parser_build.set_defaults(func=regression_git)
175 parser_build.add_argument('base_result_file', 177 parser_build.add_argument('repo',
176 help='base result file provide the base result set') 178 help='the git repository containing the data')
177 parser_build.add_argument('target_result_file',
178 help='target result file provide the target result set for comparison with base result')
179 parser_build.add_argument('-b', '--base-result-id', default='', 179 parser_build.add_argument('-b', '--base-result-id', default='',
180 help='(optional) default select regression based on configurations unless base result ' 180 help='(optional) default select regression based on configurations unless base result '
181 'id was provided') 181 'id was provided')
@@ -183,26 +183,10 @@ def register_commands(subparsers):
183 help='(optional) default select regression based on configurations unless target result ' 183 help='(optional) default select regression based on configurations unless target result '
184 'id was provided') 184 'id was provided')
185 185
186 parser_build = subparsers.add_parser('regression-dir', help='regression directory analysis', 186 parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
187 description='regression analysis comparing base result set to target ' 187 parser_build.add_argument('--branch2', help="Branch to find comparision revisions in")
188 'result set', 188 parser_build.add_argument('--commit', help="Revision to search for")
189 group='analysis') 189 parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
190 parser_build.set_defaults(func=regression_directory) 190 parser_build.add_argument('--commit2', help="Revision to compare with")
191 parser_build.add_argument('base_result_directory', 191 parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
192 help='base result directory provide the files for base result set')
193 parser_build.add_argument('target_result_directory',
194 help='target result file provide the files for target result set for comparison with '
195 'base result')
196 192
197 parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
198 description='regression analysis comparing base result set to target '
199 'result set',
200 group='analysis')
201 parser_build.set_defaults(func=regression_git)
202 parser_build.add_argument('source_dir',
203 help='source directory that contain the git repository with test result files')
204 parser_build.add_argument('base_git_branch',
205 help='base git branch that provide the files for base result set')
206 parser_build.add_argument('target_git_branch',
207 help='target git branch that provide the files for target result set for comparison with '
208 'base result')
diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py
index ab5de1f3a7..2f5ea308e2 100644
--- a/scripts/lib/resulttool/report.py
+++ b/scripts/lib/resulttool/report.py
@@ -1,6 +1,7 @@
1# test result tool - report text based test results 1# test result tool - report text based test results
2# 2#
3# Copyright (c) 2019, Intel Corporation. 3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
4# 5#
5# This program is free software; you can redistribute it and/or modify it 6# This program is free software; you can redistribute it and/or modify it
6# under the terms and conditions of the GNU General Public License, 7# under the terms and conditions of the GNU General Public License,
@@ -14,100 +15,120 @@
14import os 15import os
15import glob 16import glob
16import json 17import json
17from resulttool.resultsutils import checkout_git_dir, load_json_file, get_dict_value, get_directory_files 18import resulttool.resultutils as resultutils
19from oeqa.utils.git import GitRepo
20import oeqa.utils.gitarchive as gitarchive
21
18 22
19class ResultsTextReport(object): 23class ResultsTextReport(object):
24 def __init__(self):
25 self.ptests = {}
26 self.result_types = {'passed': ['PASSED', 'passed'],
27 'failed': ['FAILED', 'failed', 'ERROR', 'error', 'UNKNOWN'],
28 'skipped': ['SKIPPED', 'skipped']}
29
30
31 def handle_ptest_result(self, k, status, result):
32 if k == 'ptestresult.sections':
33 return
34 _, suite, test = k.split(".", 2)
35 # Handle 'glib-2.0'
36 if suite not in result['ptestresult.sections']:
37 try:
38 _, suite, suite1, test = k.split(".", 3)
39 if suite + "." + suite1 in result['ptestresult.sections']:
40 suite = suite + "." + suite1
41 except ValueError:
42 pass
43 if suite not in self.ptests:
44 self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
45 for tk in self.result_types:
46 if status in self.result_types[tk]:
47 self.ptests[suite][tk] += 1
48 if suite in result['ptestresult.sections']:
49 if 'duration' in result['ptestresult.sections'][suite]:
50 self.ptests[suite]['duration'] = result['ptestresult.sections'][suite]['duration']
51 if 'timeout' in result['ptestresult.sections'][suite]:
52 self.ptests[suite]['duration'] += " T"
20 53
21 def get_aggregated_test_result(self, logger, testresult): 54 def get_aggregated_test_result(self, logger, testresult):
22 test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} 55 test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
23 result_types = {'passed': ['PASSED', 'passed'], 56 result = testresult.get('result', [])
24 'failed': ['FAILED', 'failed', 'ERROR', 'error', 'UNKNOWN'],
25 'skipped': ['SKIPPED', 'skipped']}
26 result = get_dict_value(logger, testresult, 'result')
27 for k in result: 57 for k in result:
28 test_status = get_dict_value(logger, result[k], 'status') 58 test_status = result[k].get('status', [])
29 for tk in result_types: 59 for tk in self.result_types:
30 if test_status in result_types[tk]: 60 if test_status in self.result_types[tk]:
31 test_count_report[tk] += 1 61 test_count_report[tk] += 1
32 if test_status in result_types['failed']: 62 if test_status in self.result_types['failed']:
33 test_count_report['failed_testcases'].append(k) 63 test_count_report['failed_testcases'].append(k)
64 if k.startswith("ptestresult."):
65 self.handle_ptest_result(k, test_status, result)
34 return test_count_report 66 return test_count_report
35 67
36 def get_test_result_percentage(self, test_result_count): 68 def print_test_report(self, template_file_name, test_count_reports):
37 total_tested = test_result_count['passed'] + test_result_count['failed'] + test_result_count['skipped']
38 test_percent_report = {'passed': 0, 'failed': 0, 'skipped': 0}
39 for k in test_percent_report:
40 test_percent_report[k] = format(test_result_count[k] / total_tested * 100, '.2f')
41 return test_percent_report
42
43 def add_test_configurations(self, test_report, source_dir, file, result_id):
44 test_report['file_dir'] = self._get_short_file_dir(source_dir, file)
45 test_report['result_id'] = result_id
46 test_report['test_file_dir_result_id'] = '%s_%s' % (test_report['file_dir'], test_report['result_id'])
47
48 def _get_short_file_dir(self, source_dir, file):
49 file_dir = os.path.dirname(file)
50 source_dir = source_dir[:-1] if source_dir[-1] == '/' else source_dir
51 if file_dir == source_dir:
52 return 'None'
53 return file_dir.replace(source_dir, '')
54
55 def get_max_string_len(self, test_result_list, key, default_max_len):
56 max_len = default_max_len
57 for test_result in test_result_list:
58 value_len = len(test_result[key])
59 if value_len > max_len:
60 max_len = value_len
61 return max_len
62
63 def print_test_report(self, template_file_name, test_count_reports, test_percent_reports,
64 max_len_dir, max_len_result_id):
65 from jinja2 import Environment, FileSystemLoader 69 from jinja2 import Environment, FileSystemLoader
66 script_path = os.path.dirname(os.path.realpath(__file__)) 70 script_path = os.path.dirname(os.path.realpath(__file__))
67 file_loader = FileSystemLoader(script_path + '/template') 71 file_loader = FileSystemLoader(script_path + '/template')
68 env = Environment(loader=file_loader, trim_blocks=True) 72 env = Environment(loader=file_loader, trim_blocks=True)
69 template = env.get_template(template_file_name) 73 template = env.get_template(template_file_name)
70 output = template.render(test_count_reports=test_count_reports, 74 havefailed = False
71 test_percent_reports=test_percent_reports, 75 haveptest = bool(self.ptests)
72 max_len_dir=max_len_dir, 76 reportvalues = []
73 max_len_result_id=max_len_result_id) 77 cols = ['passed', 'failed', 'skipped']
74 print('Printing text-based test report:') 78 maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 }
79 for line in test_count_reports:
80 total_tested = line['passed'] + line['failed'] + line['skipped']
81 vals = {}
82 vals['result_id'] = line['result_id']
83 vals['testseries'] = line['testseries']
84 vals['sort'] = line['testseries'] + "_" + line['result_id']
85 vals['failed_testcases'] = line['failed_testcases']
86 for k in cols:
87 vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f'))
88 for k in maxlen:
89 if k in vals and len(vals[k]) > maxlen[k]:
90 maxlen[k] = len(vals[k])
91 reportvalues.append(vals)
92 if line['failed_testcases']:
93 havefailed = True
94 for ptest in self.ptests:
95 if len(ptest) > maxlen['ptest']:
96 maxlen['ptest'] = len(ptest)
97 output = template.render(reportvalues=reportvalues,
98 havefailed=havefailed,
99 haveptest=haveptest,
100 ptests=self.ptests,
101 maxlen=maxlen)
75 print(output) 102 print(output)
76 103
77 def view_test_report(self, logger, source_dir, git_branch): 104 def view_test_report(self, logger, source_dir, tag):
78 if git_branch:
79 checkout_git_dir(source_dir, git_branch)
80 test_count_reports = [] 105 test_count_reports = []
81 test_percent_reports = [] 106 if tag:
82 for file in get_directory_files(source_dir, ['.git'], 'testresults.json'): 107 repo = GitRepo(source_dir)
83 logger.debug('Computing result for test result file: %s' % file) 108 testresults = resultutils.git_get_result(repo, [tag])
84 testresults = load_json_file(file) 109 else:
85 for k in testresults: 110 testresults = resultutils.load_resultsdata(source_dir)
86 test_count_report = self.get_aggregated_test_result(logger, testresults[k]) 111 for testsuite in testresults:
87 test_percent_report = self.get_test_result_percentage(test_count_report) 112 for resultid in testresults[testsuite]:
88 self.add_test_configurations(test_count_report, source_dir, file, k) 113 result = testresults[testsuite][resultid]
89 self.add_test_configurations(test_percent_report, source_dir, file, k) 114 test_count_report = self.get_aggregated_test_result(logger, result)
115 test_count_report['testseries'] = result['configuration']['TESTSERIES']
116 test_count_report['result_id'] = resultid
90 test_count_reports.append(test_count_report) 117 test_count_reports.append(test_count_report)
91 test_percent_reports.append(test_percent_report) 118 self.print_test_report('test_report_full_text.txt', test_count_reports)
92 max_len_dir = self.get_max_string_len(test_count_reports, 'file_dir', len('file_dir'))
93 max_len_result_id = self.get_max_string_len(test_count_reports, 'result_id', len('result_id'))
94 self.print_test_report('test_report_full_text.txt', test_count_reports, test_percent_reports,
95 max_len_dir, max_len_result_id)
96 119
97def report(args, logger): 120def report(args, logger):
98 report = ResultsTextReport() 121 report = ResultsTextReport()
99 report.view_test_report(logger, args.source_dir, args.git_branch) 122 report.view_test_report(logger, args.source_dir, args.tag)
100 return 0 123 return 0
101 124
102def register_commands(subparsers): 125def register_commands(subparsers):
103 """Register subcommands from this plugin""" 126 """Register subcommands from this plugin"""
104 parser_build = subparsers.add_parser('report', help='report test result summary', 127 parser_build = subparsers.add_parser('report', help='summarise test results',
105 description='report text-based test result summary from the source directory', 128 description='print a text-based summary of the test results',
106 group='analysis') 129 group='analysis')
107 parser_build.set_defaults(func=report) 130 parser_build.set_defaults(func=report)
108 parser_build.add_argument('source_dir', 131 parser_build.add_argument('source_dir',
109 help='source directory that contain the test result files for reporting') 132 help='source file/directory that contain the test result files to summarise')
110 parser_build.add_argument('-b', '--git-branch', default='', 133 parser_build.add_argument('-t', '--tag', default='',
111 help='(optional) default assume source directory contains all available files for ' 134 help='source_dir is a git repository, report on the tag specified from that repository')
112 'reporting unless a git branch was provided where it will try to checkout '
113 'the provided git branch assuming source directory was a git repository')
diff --git a/scripts/lib/resulttool/resultsutils.py b/scripts/lib/resulttool/resultsutils.py
deleted file mode 100644
index 368786922c..0000000000
--- a/scripts/lib/resulttool/resultsutils.py
+++ /dev/null
@@ -1,67 +0,0 @@
1# test result tool - utilities
2#
3# Copyright (c) 2019, Intel Corporation.
4#
5# This program is free software; you can redistribute it and/or modify it
6# under the terms and conditions of the GNU General Public License,
7# version 2, as published by the Free Software Foundation.
8#
9# This program is distributed in the hope it will be useful, but WITHOUT
10# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12# more details.
13#
14import os
15import json
16import scriptpath
17scriptpath.add_oe_lib_path()
18from oeqa.utils.git import GitRepo, GitError
19
20def load_json_file(file):
21 with open(file, "r") as f:
22 return json.load(f)
23
24def dump_json_data(write_dir, file_name, json_data):
25 file_content = json.dumps(json_data, sort_keys=True, indent=4)
26 file_path = os.path.join(write_dir, file_name)
27 with open(file_path, 'w') as the_file:
28 the_file.write(file_content)
29
30def get_dict_value(logger, dict, key):
31 try:
32 return dict[key]
33 except KeyError:
34 if logger:
35 logger.debug('Faced KeyError exception: dict=%s: key=%s' % (dict, key))
36 return None
37 except TypeError:
38 if logger:
39 logger.debug('Faced TypeError exception: dict=%s: key=%s' % (dict, key))
40 return None
41
42def pop_dict_element(logger, dict, key):
43 try:
44 dict.pop(key)
45 except KeyError:
46 if logger:
47 logger.debug('Faced KeyError exception: dict=%s: key=%s' % (dict, key))
48 except AttributeError:
49 if logger:
50 logger.debug('Faced AttributeError exception: dict=%s: key=%s' % (dict, key))
51
52def checkout_git_dir(git_dir, git_branch):
53 try:
54 repo = GitRepo(git_dir, is_topdir=True)
55 repo.run_cmd('checkout %s' % git_branch)
56 return True
57 except GitError:
58 return False
59
60def get_directory_files(source_dir, excludes, file):
61 files_in_dir = []
62 for root, dirs, files in os.walk(source_dir, topdown=True):
63 [dirs.remove(d) for d in list(dirs) if d in excludes]
64 for name in files:
65 if name == file:
66 files_in_dir.append(os.path.join(root, name))
67 return files_in_dir \ No newline at end of file
diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py
new file mode 100644
index 0000000000..06cceef796
--- /dev/null
+++ b/scripts/lib/resulttool/resultutils.py
@@ -0,0 +1,127 @@
1# resulttool - common library/utility functions
2#
3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
5#
6# This program is free software; you can redistribute it and/or modify it
7# under the terms and conditions of the GNU General Public License,
8# version 2, as published by the Free Software Foundation.
9#
10# This program is distributed in the hope it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15import os
16import json
17import scriptpath
18scriptpath.add_oe_lib_path()
19
20flatten_map = {
21 "oeselftest": [],
22 "runtime": [],
23 "sdk": [],
24 "sdkext": []
25}
26regression_map = {
27 "oeselftest": ['TEST_TYPE', 'MACHINE'],
28 "runtime": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'IMAGE_PKGTYPE', 'DISTRO'],
29 "sdk": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE'],
30 "sdkext": ['TESTSERIES', 'TEST_TYPE', 'IMAGE_BASENAME', 'MACHINE', 'SDKMACHINE']
31}
32store_map = {
33 "oeselftest": ['TEST_TYPE'],
34 "runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'],
35 "sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'],
36 "sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME']
37}
38
39#
40# Load the json file and append the results data into the provided results dict
41#
42def append_resultsdata(results, f, configmap=store_map):
43 if type(f) is str:
44 with open(f, "r") as filedata:
45 data = json.load(filedata)
46 else:
47 data = f
48 for res in data:
49 if "configuration" not in data[res] or "result" not in data[res]:
50 raise ValueError("Test results data without configuration or result section?")
51 if "TESTSERIES" not in data[res]["configuration"]:
52 data[res]["configuration"]["TESTSERIES"] = os.path.basename(os.path.dirname(f))
53 testtype = data[res]["configuration"].get("TEST_TYPE")
54 if testtype not in configmap:
55 raise ValueError("Unknown test type %s" % testtype)
56 configvars = configmap[testtype]
57 testpath = "/".join(data[res]["configuration"].get(i) for i in configmap[testtype])
58 if testpath not in results:
59 results[testpath] = {}
60 if 'ptestresult.rawlogs' in data[res]['result']:
61 del data[res]['result']['ptestresult.rawlogs']
62 if 'ptestresult.sections' in data[res]['result']:
63 for i in data[res]['result']['ptestresult.sections']:
64 del data[res]['result']['ptestresult.sections'][i]['log']
65 results[testpath][res] = data[res]
66
67#
68# Walk a directory and find/load results data
69# or load directly from a file
70#
71def load_resultsdata(source, configmap=store_map):
72 results = {}
73 if os.path.isfile(source):
74 append_resultsdata(results, source, configmap)
75 return results
76 for root, dirs, files in os.walk(source):
77 for name in files:
78 f = os.path.join(root, name)
79 if name == "testresults.json":
80 append_resultsdata(results, f, configmap)
81 return results
82
83def filter_resultsdata(results, resultid):
84 newresults = {}
85 for r in results:
86 for i in results[r]:
87 if i == resultsid:
88 newresults[r] = {}
89 newresults[r][i] = results[r][i]
90 return newresults
91
92def save_resultsdata(results, destdir, fn="testresults.json"):
93 for res in results:
94 if res:
95 dst = destdir + "/" + res + "/" + fn
96 else:
97 dst = destdir + "/" + fn
98 os.makedirs(os.path.dirname(dst), exist_ok=True)
99 with open(dst, 'w') as f:
100 f.write(json.dumps(results[res], sort_keys=True, indent=4))
101
102def git_get_result(repo, tags):
103 git_objs = []
104 for tag in tags:
105 files = repo.run_cmd(['ls-tree', "--name-only", "-r", tag]).splitlines()
106 git_objs.extend([tag + ':' + f for f in files if f.endswith("testresults.json")])
107
108 def parse_json_stream(data):
109 """Parse multiple concatenated JSON objects"""
110 objs = []
111 json_d = ""
112 for line in data.splitlines():
113 if line == '}{':
114 json_d += '}'
115 objs.append(json.loads(json_d))
116 json_d = '{'
117 else:
118 json_d += line
119 objs.append(json.loads(json_d))
120 return objs
121
122 # Optimize by reading all data with one git command
123 results = {}
124 for obj in parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--'])):
125 append_resultsdata(results, obj)
126
127 return results
diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py
index 2c6fd8492c..6744fb3c05 100644
--- a/scripts/lib/resulttool/store.py
+++ b/scripts/lib/resulttool/store.py
@@ -1,6 +1,7 @@
1# test result tool - store test results 1# resulttool - store test results
2# 2#
3# Copyright (c) 2019, Intel Corporation. 3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
4# 5#
5# This program is free software; you can redistribute it and/or modify it 6# This program is free software; you can redistribute it and/or modify it
6# under the terms and conditions of the GNU General Public License, 7# under the terms and conditions of the GNU General Public License,
@@ -11,100 +12,81 @@
11# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12# more details. 13# more details.
13# 14#
14import datetime
15import tempfile 15import tempfile
16import os 16import os
17import subprocess 17import subprocess
18import json
19import shutil
18import scriptpath 20import scriptpath
19scriptpath.add_bitbake_lib_path() 21scriptpath.add_bitbake_lib_path()
20scriptpath.add_oe_lib_path() 22scriptpath.add_oe_lib_path()
21from resulttool.resultsutils import checkout_git_dir 23import resulttool.resultutils as resultutils
22try: 24import oeqa.utils.gitarchive as gitarchive
23 import bb
24except ImportError:
25 pass
26 25
27class ResultsGitStore(object):
28 26
29 def _get_output_dir(self): 27def store(args, logger):
30 basepath = os.environ['BUILDDIR'] 28 tempdir = tempfile.mkdtemp(prefix='testresults.')
31 return basepath + '/testresults_%s/' % datetime.datetime.now().strftime("%Y%m%d%H%M%S") 29 try:
32 30 results = {}
33 def _create_temporary_workspace_dir(self): 31 logger.info('Reading files from %s' % args.source)
34 return tempfile.mkdtemp(prefix='testresults.') 32 for root, dirs, files in os.walk(args.source):
35 33 for name in files:
36 def _remove_temporary_workspace_dir(self, workspace_dir): 34 f = os.path.join(root, name)
37 return subprocess.run(["rm", "-rf", workspace_dir]) 35 if name == "testresults.json":
38 36 resultutils.append_resultsdata(results, f)
39 def _oe_copy_files(self, source_dir, destination_dir): 37 elif args.all:
40 from oe.path import copytree 38 dst = f.replace(args.source, tempdir + "/")
41 copytree(source_dir, destination_dir) 39 os.makedirs(os.path.dirname(dst), exist_ok=True)
40 shutil.copyfile(f, dst)
41 resultutils.save_resultsdata(results, tempdir)
42 42
43 def _copy_files(self, source_dir, destination_dir, copy_ignore=None): 43 if not results and not args.all:
44 from shutil import copytree 44 if args.allow_empty:
45 copytree(source_dir, destination_dir, ignore=copy_ignore) 45 logger.info("No results found to store")
46 return 0
47 logger.error("No results found to store")
48 return 1
46 49
47 def _store_files_to_git(self, logger, file_dir, git_dir, git_branch, commit_msg_subject, commit_msg_body): 50 keywords = {'branch': None, 'commit': None, 'commit_count': None}
48 logger.debug('Storing test result into git repository (%s) and branch (%s)'
49 % (git_dir, git_branch))
50 return subprocess.run(["oe-git-archive",
51 file_dir,
52 "-g", git_dir,
53 "-b", git_branch,
54 "--commit-msg-subject", commit_msg_subject,
55 "--commit-msg-body", commit_msg_body])
56 51
57 def store_to_existing(self, logger, source_dir, git_dir, git_branch): 52 # Find the branch/commit/commit_count and ensure they all match
58 logger.debug('Storing files to existing git repository and branch') 53 for suite in results:
59 from shutil import ignore_patterns 54 for result in results[suite]:
60 dest_dir = self._create_temporary_workspace_dir() 55 config = results[suite][result]['configuration']['LAYERS']['meta']
61 dest_top_dir = os.path.join(dest_dir, 'top_dir') 56 for k in keywords:
62 self._copy_files(git_dir, dest_top_dir, copy_ignore=ignore_patterns('.git')) 57 if keywords[k] is None:
63 self._oe_copy_files(source_dir, dest_top_dir) 58 keywords[k] = config.get(k)
64 self._store_files_to_git(logger, dest_top_dir, git_dir, git_branch, 59 if config.get(k) != keywords[k]:
65 'Store as existing git and branch', 'Store as existing git repository and branch') 60 logger.error("Mismatched source commit/branch/count: %s vs %s" % (config.get(k), keywords[k]))
66 self._remove_temporary_workspace_dir(dest_dir) 61 return 1
67 return git_dir
68 62
69 def store_to_existing_with_new_branch(self, logger, source_dir, git_dir, git_branch): 63 logger.info('Storing test result into git repository %s' % args.git_dir)
70 logger.debug('Storing files to existing git repository with new branch')
71 self._store_files_to_git(logger, source_dir, git_dir, git_branch,
72 'Store as existing git with new branch',
73 'Store as existing git repository with new branch')
74 return git_dir
75 64
76 def store_to_new(self, logger, source_dir, git_branch): 65 gitarchive.gitarchive(tempdir, args.git_dir, False, False,
77 logger.debug('Storing files to new git repository') 66 "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}",
78 output_dir = self._get_output_dir() 67 False, "{branch}/{commit_count}-g{commit}/{tag_number}",
79 self._store_files_to_git(logger, source_dir, output_dir, git_branch, 68 'Test run #{tag_number} of {branch}:{commit}', '',
80 'Store as new', 'Store as new git repository') 69 [], [], False, keywords, logger)
81 return output_dir
82 70
83 def store(self, logger, source_dir, git_dir, git_branch): 71 finally:
84 if git_dir: 72 subprocess.check_call(["rm", "-rf", tempdir])
85 if checkout_git_dir(git_dir, git_branch):
86 self.store_to_existing(logger, source_dir, git_dir, git_branch)
87 else:
88 self.store_to_existing_with_new_branch(logger, source_dir, git_dir, git_branch)
89 else:
90 self.store_to_new(logger, source_dir, git_branch)
91 73
92def store(args, logger):
93 gitstore = ResultsGitStore()
94 gitstore.store(logger, args.source_dir, args.git_dir, args.git_branch)
95 return 0 74 return 0
96 75
97def register_commands(subparsers): 76def register_commands(subparsers):
98 """Register subcommands from this plugin""" 77 """Register subcommands from this plugin"""
99 parser_build = subparsers.add_parser('store', help='store test result files and directories into git repository', 78 parser_build = subparsers.add_parser('store', help='store test results into a git repository',
100 description='store the testresults.json files and related directories ' 79 description='takes a results file or directory of results files and stores '
101 'from the source directory into the destination git repository ' 80 'them into the destination git repository, splitting out the results '
102 'with the given git branch', 81 'files as configured',
103 group='setup') 82 group='setup')
104 parser_build.set_defaults(func=store) 83 parser_build.set_defaults(func=store)
105 parser_build.add_argument('source_dir', 84 parser_build.add_argument('source',
106 help='source directory that contain the test result files and directories to be stored') 85 help='source file or directory that contain the test result files to be stored')
107 parser_build.add_argument('git_branch', help='git branch used for store') 86 parser_build.add_argument('git_dir',
108 parser_build.add_argument('-d', '--git-dir', default='', 87 help='the location of the git repository to store the results in')
109 help='(optional) default store to new <top_dir>/<build>/<testresults_datetime> ' 88 parser_build.add_argument('-a', '--all', action='store_true',
110 'directory unless provided with existing git repository as destination') 89 help='include all files, not just testresults.json files')
90 parser_build.add_argument('-e', '--allow-empty', action='store_true',
91 help='don\'t error if no results to store are found')
92
diff --git a/scripts/lib/resulttool/template/test_report_full_text.txt b/scripts/lib/resulttool/template/test_report_full_text.txt
index bc4874ba4b..5081594cf2 100644
--- a/scripts/lib/resulttool/template/test_report_full_text.txt
+++ b/scripts/lib/resulttool/template/test_report_full_text.txt
@@ -1,35 +1,44 @@
1============================================================================================================== 1==============================================================================================================
2Test Report (Count of passed, failed, skipped group by file_dir, result_id) 2Test Result Status Summary (Counts/Percentages sorted by testseries, ID)
3============================================================================================================== 3==============================================================================================================
4-------------------------------------------------------------------------------------------------------------- 4--------------------------------------------------------------------------------------------------------------
5{{ 'file_dir'.ljust(max_len_dir) }} | {{ 'result_id'.ljust(max_len_result_id) }} | {{ 'passed'.ljust(10) }} | {{ 'failed'.ljust(10) }} | {{ 'skipped'.ljust(10) }} 5{{ 'Test Series'.ljust(maxlen['testseries']) }} | {{ 'ID'.ljust(maxlen['result_id']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }}
6-------------------------------------------------------------------------------------------------------------- 6--------------------------------------------------------------------------------------------------------------
7{% for report in test_count_reports |sort(attribute='test_file_dir_result_id') %} 7{% for report in reportvalues |sort(attribute='sort') %}
8{{ report.file_dir.ljust(max_len_dir) }} | {{ report.result_id.ljust(max_len_result_id) }} | {{ (report.passed|string).ljust(10) }} | {{ (report.failed|string).ljust(10) }} | {{ (report.skipped|string).ljust(10) }} 8{{ report.testseries.ljust(maxlen['testseries']) }} | {{ report.result_id.ljust(maxlen['result_id']) }} | {{ (report.passed|string).ljust(maxlen['passed']) }} | {{ (report.failed|string).ljust(maxlen['failed']) }} | {{ (report.skipped|string).ljust(maxlen['skipped']) }}
9{% endfor %} 9{% endfor %}
10-------------------------------------------------------------------------------------------------------------- 10--------------------------------------------------------------------------------------------------------------
11 11
12{% if haveptest %}
12============================================================================================================== 13==============================================================================================================
13Test Report (Percent of passed, failed, skipped group by file_dir, result_id) 14PTest Result Summary
14============================================================================================================== 15==============================================================================================================
15-------------------------------------------------------------------------------------------------------------- 16--------------------------------------------------------------------------------------------------------------
16{{ 'file_dir'.ljust(max_len_dir) }} | {{ 'result_id'.ljust(max_len_result_id) }} | {{ 'passed_%'.ljust(10) }} | {{ 'failed_%'.ljust(10) }} | {{ 'skipped_%'.ljust(10) }} 17{{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
17-------------------------------------------------------------------------------------------------------------- 18--------------------------------------------------------------------------------------------------------------
18{% for report in test_percent_reports |sort(attribute='test_file_dir_result_id') %} 19{% for ptest in ptests %}
19{{ report.file_dir.ljust(max_len_dir) }} | {{ report.result_id.ljust(max_len_result_id) }} | {{ (report.passed|string).ljust(10) }} | {{ (report.failed|string).ljust(10) }} | {{ (report.skipped|string).ljust(10) }} 20{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[ptest]['duration']|string) }}
20{% endfor %} 21{% endfor %}
21-------------------------------------------------------------------------------------------------------------- 22--------------------------------------------------------------------------------------------------------------
22 23
24{% else %}
25There was no ptest data
26{% endif %}
27
23============================================================================================================== 28==============================================================================================================
24Test Report (Failed test cases group by file_dir, result_id) 29Failed test cases (sorted by testseries, ID)
25============================================================================================================== 30==============================================================================================================
31{% if havefailed %}
26-------------------------------------------------------------------------------------------------------------- 32--------------------------------------------------------------------------------------------------------------
27{% for report in test_count_reports |sort(attribute='test_file_dir_result_id') %} 33{% for report in reportvalues |sort(attribute='sort') %}
28{% if report.failed_testcases %} 34{% if report.failed_testcases %}
29file_dir | result_id : {{ report.file_dir }} | {{ report.result_id }} 35testseries | result_id : {{ report.testseries }} | {{ report.result_id }}
30{% for testcase in report.failed_testcases %} 36{% for testcase in report.failed_testcases %}
31 {{ testcase }} 37 {{ testcase }}
32{% endfor %} 38{% endfor %}
33{% endif %} 39{% endif %}
34{% endfor %} 40{% endfor %}
35-------------------------------------------------------------------------------------------------------------- \ No newline at end of file 41--------------------------------------------------------------------------------------------------------------
42{% else %}
43There were no test failures
44{% endif %}
diff --git a/scripts/resulttool b/scripts/resulttool
index 13430e192a..5a89e1c9be 100755
--- a/scripts/resulttool
+++ b/scripts/resulttool
@@ -1,11 +1,12 @@
1#!/usr/bin/env python3 1#!/usr/bin/env python3
2# 2#
3# test results tool - tool for testresults.json (merge test results, regression analysis) 3# test results tool - tool for manipulating OEQA test result json files
4# (merge results, summarise results, regression analysis, generate manual test results file)
4# 5#
5# To look for help information. 6# To look for help information.
6# $ resulttool 7# $ resulttool
7# 8#
8# To store test result from oeqa automated tests, execute the below 9# To store test results from oeqa automated tests, execute the below
9# $ resulttool store <source_dir> <git_branch> 10# $ resulttool store <source_dir> <git_branch>
10# 11#
11# To merge test results, execute the below 12# To merge test results, execute the below
@@ -58,7 +59,7 @@ def _validate_user_input_arguments(args):
58 return True 59 return True
59 60
60def main(): 61def main():
61 parser = argparse_oe.ArgumentParser(description="OpenEmbedded test results tool.", 62 parser = argparse_oe.ArgumentParser(description="OEQA test result manipulation tool.",
62 epilog="Use %(prog)s <subcommand> --help to get help on a specific command") 63 epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
63 parser.add_argument('-d', '--debug', help='enable debug output', action='store_true') 64 parser.add_argument('-d', '--debug', help='enable debug output', action='store_true')
64 parser.add_argument('-q', '--quiet', help='print only errors', action='store_true') 65 parser.add_argument('-q', '--quiet', help='print only errors', action='store_true')
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-11-20 12:43:43 +0000