summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Mason <jdmason@kudzu.us>2019-06-02 14:29:12 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-06-04 09:09:42 +0100
commitcb153e8da8897080e128d7e29d6a1f48e1190fdf (patch)
treeb749b8dfe889f16fadc0a46c66c4e5dd8c086795
parentc113a83dd5ef6dbf8c4bbaf409628dcfd73af283 (diff)
downloadpoky-cb153e8da8897080e128d7e29d6a1f48e1190fdf.tar.gz
resulttool: modify to be multi-machine
Currently, the code will sum all of the different machine results into a single report of the tests results. This can lead to confusion as to which machine may be experiencing issues. Modify the code to store the results in a per machine basis and report them accordingly. (From OE-Core rev: 16d4031ea5df8a4ddfdb937d35464c09e1abd10e) Signed-off-by: Jon Mason <jdmason@kudzu.us> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oeqa/selftest/cases/resulttooltests.py2
-rw-r--r--scripts/lib/resulttool/report.py104
-rw-r--r--scripts/lib/resulttool/template/test_report_full_text.txt44
3 files changed, 82 insertions, 68 deletions
diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index 10eb9c12d9..dac5c46801 100644
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -56,7 +56,7 @@ class ResultToolTests(OESelftestTestCase):
56 'test4': {'status': 'ERROR'}, 56 'test4': {'status': 'ERROR'},
57 'test5': {'status': 'SKIPPED'}}} 57 'test5': {'status': 'SKIPPED'}}}
58 report = ResultsTextReport() 58 report = ResultsTextReport()
59 result_report = report.get_aggregated_test_result(None, result_data) 59 result_report = report.get_aggregated_test_result(None, result_data, 'DummyMachine')
60 self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed']) 60 self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed'])
61 self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed']) 61 self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed'])
62 self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped']) 62 self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped'])
diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py
index cb6b1cf948..a48c59f632 100644
--- a/scripts/lib/resulttool/report.py
+++ b/scripts/lib/resulttool/report.py
@@ -24,16 +24,19 @@ class ResultsTextReport(object):
24 'skipped': ['SKIPPED', 'skipped']} 24 'skipped': ['SKIPPED', 'skipped']}
25 25
26 26
27 def handle_ptest_result(self, k, status, result): 27 def handle_ptest_result(self, k, status, result, machine):
28 if machine not in self.ptests:
29 self.ptests[machine] = {}
30
28 if k == 'ptestresult.sections': 31 if k == 'ptestresult.sections':
29 # Ensure tests without any test results still show up on the report 32 # Ensure tests without any test results still show up on the report
30 for suite in result['ptestresult.sections']: 33 for suite in result['ptestresult.sections']:
31 if suite not in self.ptests: 34 if suite not in self.ptests[machine]:
32 self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} 35 self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
33 if 'duration' in result['ptestresult.sections'][suite]: 36 if 'duration' in result['ptestresult.sections'][suite]:
34 self.ptests[suite]['duration'] = result['ptestresult.sections'][suite]['duration'] 37 self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration']
35 if 'timeout' in result['ptestresult.sections'][suite]: 38 if 'timeout' in result['ptestresult.sections'][suite]:
36 self.ptests[suite]['duration'] += " T" 39 self.ptests[machine][suite]['duration'] += " T"
37 return 40 return
38 try: 41 try:
39 _, suite, test = k.split(".", 2) 42 _, suite, test = k.split(".", 2)
@@ -47,22 +50,25 @@ class ResultsTextReport(object):
47 suite = suite + "." + suite1 50 suite = suite + "." + suite1
48 except ValueError: 51 except ValueError:
49 pass 52 pass
50 if suite not in self.ptests: 53 if suite not in self.ptests[machine]:
51 self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} 54 self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
52 for tk in self.result_types: 55 for tk in self.result_types:
53 if status in self.result_types[tk]: 56 if status in self.result_types[tk]:
54 self.ptests[suite][tk] += 1 57 self.ptests[machine][suite][tk] += 1
58
59 def handle_ltptest_result(self, k, status, result, machine):
60 if machine not in self.ltptests:
61 self.ltptests[machine] = {}
55 62
56 def handle_ltptest_result(self, k, status, result):
57 if k == 'ltpresult.sections': 63 if k == 'ltpresult.sections':
58 # Ensure tests without any test results still show up on the report 64 # Ensure tests without any test results still show up on the report
59 for suite in result['ltpresult.sections']: 65 for suite in result['ltpresult.sections']:
60 if suite not in self.ltptests: 66 if suite not in self.ltptests[machine]:
61 self.ltptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} 67 self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
62 if 'duration' in result['ltpresult.sections'][suite]: 68 if 'duration' in result['ltpresult.sections'][suite]:
63 self.ltptests[suite]['duration'] = result['ltpresult.sections'][suite]['duration'] 69 self.ltptests[machine][suite]['duration'] = result['ltpresult.sections'][suite]['duration']
64 if 'timeout' in result['ltpresult.sections'][suite]: 70 if 'timeout' in result['ltpresult.sections'][suite]:
65 self.ltptests[suite]['duration'] += " T" 71 self.ltptests[machine][suite]['duration'] += " T"
66 return 72 return
67 try: 73 try:
68 _, suite, test = k.split(".", 2) 74 _, suite, test = k.split(".", 2)
@@ -77,20 +83,23 @@ class ResultsTextReport(object):
77 suite = suite + "." + suite1 83 suite = suite + "." + suite1
78 except ValueError: 84 except ValueError:
79 pass 85 pass
80 if suite not in self.ltptests: 86 if suite not in self.ltptests[machine]:
81 self.ltptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} 87 self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
82 for tk in self.result_types: 88 for tk in self.result_types:
83 if status in self.result_types[tk]: 89 if status in self.result_types[tk]:
84 self.ltptests[suite][tk] += 1 90 self.ltptests[machine][suite][tk] += 1
91
92 def handle_ltpposixtest_result(self, k, status, result, machine):
93 if machine not in self.ltpposixtests:
94 self.ltpposixtests[machine] = {}
85 95
86 def handle_ltpposixtest_result(self, k, status, result):
87 if k == 'ltpposixresult.sections': 96 if k == 'ltpposixresult.sections':
88 # Ensure tests without any test results still show up on the report 97 # Ensure tests without any test results still show up on the report
89 for suite in result['ltpposixresult.sections']: 98 for suite in result['ltpposixresult.sections']:
90 if suite not in self.ltpposixtests: 99 if suite not in self.ltpposixtests[machine]:
91 self.ltpposixtests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} 100 self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
92 if 'duration' in result['ltpposixresult.sections'][suite]: 101 if 'duration' in result['ltpposixresult.sections'][suite]:
93 self.ltpposixtests[suite]['duration'] = result['ltpposixresult.sections'][suite]['duration'] 102 self.ltpposixtests[machine][suite]['duration'] = result['ltpposixresult.sections'][suite]['duration']
94 return 103 return
95 try: 104 try:
96 _, suite, test = k.split(".", 2) 105 _, suite, test = k.split(".", 2)
@@ -104,19 +113,13 @@ class ResultsTextReport(object):
104 suite = suite + "." + suite1 113 suite = suite + "." + suite1
105 except ValueError: 114 except ValueError:
106 pass 115 pass
107 if suite not in self.ltpposixtests: 116 if suite not in self.ltpposixtests[machine]:
108 self.ltpposixtests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []} 117 self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
109 for tk in self.result_types: 118 for tk in self.result_types:
110 if status in self.result_types[tk]: 119 if status in self.result_types[tk]:
111 self.ltpposixtests[suite][tk] += 1 120 self.ltpposixtests[machine][suite][tk] += 1
112 121
113 def get_aggregated_test_result(self, logger, testresult): 122 def get_aggregated_test_result(self, logger, testresult, machine):
114 test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
115 def get_aggregated_test_result(self, logger, testresult):
116 test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
117 def get_aggregated_test_result(self, logger, testresult):
118 test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
119 def get_aggregated_test_result(self, logger, testresult):
120 test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []} 123 test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
121 result = testresult.get('result', []) 124 result = testresult.get('result', [])
122 for k in result: 125 for k in result:
@@ -127,11 +130,11 @@ class ResultsTextReport(object):
127 if test_status in self.result_types['failed']: 130 if test_status in self.result_types['failed']:
128 test_count_report['failed_testcases'].append(k) 131 test_count_report['failed_testcases'].append(k)
129 if k.startswith("ptestresult."): 132 if k.startswith("ptestresult."):
130 self.handle_ptest_result(k, test_status, result) 133 self.handle_ptest_result(k, test_status, result, machine)
131 if k.startswith("ltpresult."): 134 if k.startswith("ltpresult."):
132 self.handle_ltptest_result(k, test_status, result) 135 self.handle_ltptest_result(k, test_status, result, machine)
133 if k.startswith("ltpposixresult."): 136 if k.startswith("ltpposixresult."):
134 self.handle_ltpposixtest_result(k, test_status, result) 137 self.handle_ltpposixtest_result(k, test_status, result, machine)
135 return test_count_report 138 return test_count_report
136 139
137 def print_test_report(self, template_file_name, test_count_reports): 140 def print_test_report(self, template_file_name, test_count_reports):
@@ -141,10 +144,8 @@ class ResultsTextReport(object):
141 env = Environment(loader=file_loader, trim_blocks=True) 144 env = Environment(loader=file_loader, trim_blocks=True)
142 template = env.get_template(template_file_name) 145 template = env.get_template(template_file_name)
143 havefailed = False 146 havefailed = False
144 haveptest = bool(self.ptests)
145 haveltp = bool(self.ltptests)
146 haveltpposix = bool(self.ltpposixtests)
147 reportvalues = [] 147 reportvalues = []
148 machines = []
148 cols = ['passed', 'failed', 'skipped'] 149 cols = ['passed', 'failed', 'skipped']
149 maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 ,'ltptest': 0, 'ltpposixtest': 0} 150 maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 ,'ltptest': 0, 'ltpposixtest': 0}
150 for line in test_count_reports: 151 for line in test_count_reports:
@@ -162,21 +163,24 @@ class ResultsTextReport(object):
162 reportvalues.append(vals) 163 reportvalues.append(vals)
163 if line['failed_testcases']: 164 if line['failed_testcases']:
164 havefailed = True 165 havefailed = True
165 for ptest in self.ptests: 166 if line['machine'] not in machines:
166 if len(ptest) > maxlen['ptest']: 167 machines.append(line['machine'])
167 maxlen['ptest'] = len(ptest) 168 for (machine, report) in self.ptests.items():
168 for ltptest in self.ltptests: 169 for ptest in self.ptests[machine]:
169 if len(ltptest) > maxlen['ltptest']: 170 if len(ptest) > maxlen['ptest']:
170 maxlen['ltptest'] = len(ltptest) 171 maxlen['ptest'] = len(ptest)
171 for ltpposixtest in self.ltpposixtests: 172 for (machine, report) in self.ltptests.items():
172 if len(ltpposixtest) > maxlen['ltpposixtest']: 173 for ltptest in self.ltptests[machine]:
173 maxlen['ltpposixtest'] = len(ltpposixtest) 174 if len(ltptest) > maxlen['ltptest']:
175 maxlen['ltptest'] = len(ltptest)
176 for (machine, report) in self.ltpposixtests.items():
177 for ltpposixtest in self.ltpposixtests[machine]:
178 if len(ltpposixtest) > maxlen['ltpposixtest']:
179 maxlen['ltpposixtest'] = len(ltpposixtest)
174 output = template.render(reportvalues=reportvalues, 180 output = template.render(reportvalues=reportvalues,
175 havefailed=havefailed, 181 havefailed=havefailed,
176 haveptest=haveptest, 182 machines=machines,
177 ptests=self.ptests, 183 ptests=self.ptests,
178 haveltp=haveltp,
179 haveltpposix=haveltpposix,
180 ltptests=self.ltptests, 184 ltptests=self.ltptests,
181 ltpposixtests=self.ltpposixtests, 185 ltpposixtests=self.ltpposixtests,
182 maxlen=maxlen) 186 maxlen=maxlen)
@@ -200,7 +204,9 @@ class ResultsTextReport(object):
200 for testsuite in testresults: 204 for testsuite in testresults:
201 for resultid in testresults[testsuite]: 205 for resultid in testresults[testsuite]:
202 result = testresults[testsuite][resultid] 206 result = testresults[testsuite][resultid]
203 test_count_report = self.get_aggregated_test_result(logger, result) 207 machine = result['configuration']['MACHINE']
208 test_count_report = self.get_aggregated_test_result(logger, result, machine)
209 test_count_report['machine'] = machine
204 test_count_report['testseries'] = result['configuration']['TESTSERIES'] 210 test_count_report['testseries'] = result['configuration']['TESTSERIES']
205 test_count_report['result_id'] = resultid 211 test_count_report['result_id'] = resultid
206 test_count_reports.append(test_count_report) 212 test_count_reports.append(test_count_report)
diff --git a/scripts/lib/resulttool/template/test_report_full_text.txt b/scripts/lib/resulttool/template/test_report_full_text.txt
index d2725b8d01..6a5a081c6d 100644
--- a/scripts/lib/resulttool/template/test_report_full_text.txt
+++ b/scripts/lib/resulttool/template/test_report_full_text.txt
@@ -9,54 +9,62 @@ Test Result Status Summary (Counts/Percentages sorted by testseries, ID)
9{% endfor %} 9{% endfor %}
10-------------------------------------------------------------------------------------------------------------- 10--------------------------------------------------------------------------------------------------------------
11 11
12{% if haveptest %} 12
13{% for machine in machines %}
14{% if ptests[machine] %}
13============================================================================================================== 15==============================================================================================================
14PTest Result Summary 16{{ machine }} PTest Result Summary
15============================================================================================================== 17==============================================================================================================
16-------------------------------------------------------------------------------------------------------------- 18--------------------------------------------------------------------------------------------------------------
17{{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} 19{{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
18-------------------------------------------------------------------------------------------------------------- 20--------------------------------------------------------------------------------------------------------------
19{% for ptest in ptests |sort %} 21{% for ptest in ptests[machine] |sort %}
20{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[ptest]['duration']|string) }} 22{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[machine][ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[machine][ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[machine][ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[machine][ptest]['duration']|string) }}
21{% endfor %} 23{% endfor %}
22-------------------------------------------------------------------------------------------------------------- 24--------------------------------------------------------------------------------------------------------------
23
24{% else %} 25{% else %}
25There was no ptest data 26There was no ptest data for {{ machine }}
26{% endif %} 27{% endif %}
27 28
28{% if haveltp %} 29{% endfor %}
30
31
32{% for machine in machines %}
33{% if ltptests[machine] %}
29============================================================================================================== 34==============================================================================================================
30Ltp Test Result Summary 35{{ machine }} Ltp Test Result Summary
31============================================================================================================== 36==============================================================================================================
32-------------------------------------------------------------------------------------------------------------- 37--------------------------------------------------------------------------------------------------------------
33{{ 'Recipe'.ljust(maxlen['ltptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} 38{{ 'Recipe'.ljust(maxlen['ltptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
34-------------------------------------------------------------------------------------------------------------- 39--------------------------------------------------------------------------------------------------------------
35{% for ltptest in ltptests |sort %} 40{% for ltptest in ltptests[machine] |sort %}
36{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[ltptest]['duration']|string) }} 41{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[machine][ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[machine][ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[machine][ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[machine][ltptest]['duration']|string) }}
37{% endfor %} 42{% endfor %}
38-------------------------------------------------------------------------------------------------------------- 43--------------------------------------------------------------------------------------------------------------
39
40{% else %} 44{% else %}
41There was no LTP Test data 45There was no LTP Test data for {{ machine }}
42{% endif %} 46{% endif %}
43 47
44{% if haveltpposix %} 48{% endfor %}
49
50
51{% for machine in machines %}
52{% if ltpposixtests[machine] %}
45============================================================================================================== 53==============================================================================================================
46Ltp Posix Result Summary 54{{ machine }} Ltp Posix Result Summary
47============================================================================================================== 55==============================================================================================================
48-------------------------------------------------------------------------------------------------------------- 56--------------------------------------------------------------------------------------------------------------
49{{ 'Recipe'.ljust(maxlen['ltpposixtest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }} 57{{ 'Recipe'.ljust(maxlen['ltpposixtest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
50-------------------------------------------------------------------------------------------------------------- 58--------------------------------------------------------------------------------------------------------------
51{% for ltpposixtest in ltpposixtests |sort %} 59{% for ltpposixtest in ltpposixtests[machine] |sort %}
52{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[ltpposixtest]['duration']|string) }} 60{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[machine][ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[machine][ltpposixtest]['duration']|string) }}
53{% endfor %} 61{% endfor %}
54-------------------------------------------------------------------------------------------------------------- 62--------------------------------------------------------------------------------------------------------------
55
56{% else %} 63{% else %}
57There was no LTP Posix Test data 64There was no LTP Posix Test data for {{ machine }}
58{% endif %} 65{% endif %}
59 66
67{% endfor %}
60 68
61 69
62============================================================================================================== 70==============================================================================================================