diff options
Diffstat (limited to 'scripts/lib/build_perf/report.py')
-rw-r--r-- | scripts/lib/build_perf/report.py | 342 |
1 files changed, 342 insertions, 0 deletions
diff --git a/scripts/lib/build_perf/report.py b/scripts/lib/build_perf/report.py new file mode 100644 index 0000000000..eb00ccca2d --- /dev/null +++ b/scripts/lib/build_perf/report.py | |||
@@ -0,0 +1,342 @@ | |||
1 | # | ||
2 | # Copyright (c) 2017, Intel Corporation. | ||
3 | # | ||
4 | # This program is free software; you can redistribute it and/or modify it | ||
5 | # under the terms and conditions of the GNU General Public License, | ||
6 | # version 2, as published by the Free Software Foundation. | ||
7 | # | ||
8 | # This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | # more details. | ||
12 | # | ||
13 | """Handling of build perf test reports""" | ||
14 | from collections import OrderedDict, Mapping | ||
15 | from datetime import datetime, timezone | ||
16 | from numbers import Number | ||
17 | from statistics import mean, stdev, variance | ||
18 | |||
19 | |||
20 | def isofmt_to_timestamp(string): | ||
21 | """Convert timestamp string in ISO 8601 format into unix timestamp""" | ||
22 | if '.' in string: | ||
23 | dt = datetime.strptime(string, '%Y-%m-%dT%H:%M:%S.%f') | ||
24 | else: | ||
25 | dt = datetime.strptime(string, '%Y-%m-%dT%H:%M:%S') | ||
26 | return dt.replace(tzinfo=timezone.utc).timestamp() | ||
27 | |||
28 | |||
29 | def metadata_xml_to_json(elem): | ||
30 | """Convert metadata xml into JSON format""" | ||
31 | assert elem.tag == 'metadata', "Invalid metadata file format" | ||
32 | |||
33 | def _xml_to_json(elem): | ||
34 | """Convert xml element to JSON object""" | ||
35 | out = OrderedDict() | ||
36 | for child in elem.getchildren(): | ||
37 | key = child.attrib.get('name', child.tag) | ||
38 | if len(child): | ||
39 | out[key] = _xml_to_json(child) | ||
40 | else: | ||
41 | out[key] = child.text | ||
42 | return out | ||
43 | return _xml_to_json(elem) | ||
44 | |||
45 | |||
46 | def results_xml_to_json(elem): | ||
47 | """Convert results xml into JSON format""" | ||
48 | rusage_fields = ('ru_utime', 'ru_stime', 'ru_maxrss', 'ru_minflt', | ||
49 | 'ru_majflt', 'ru_inblock', 'ru_oublock', 'ru_nvcsw', | ||
50 | 'ru_nivcsw') | ||
51 | iostat_fields = ('rchar', 'wchar', 'syscr', 'syscw', 'read_bytes', | ||
52 | 'write_bytes', 'cancelled_write_bytes') | ||
53 | |||
54 | def _read_measurement(elem): | ||
55 | """Convert measurement to JSON""" | ||
56 | data = OrderedDict() | ||
57 | data['type'] = elem.tag | ||
58 | data['name'] = elem.attrib['name'] | ||
59 | data['legend'] = elem.attrib['legend'] | ||
60 | values = OrderedDict() | ||
61 | |||
62 | # SYSRES measurement | ||
63 | if elem.tag == 'sysres': | ||
64 | for subel in elem: | ||
65 | if subel.tag == 'time': | ||
66 | values['start_time'] = isofmt_to_timestamp(subel.attrib['timestamp']) | ||
67 | values['elapsed_time'] = float(subel.text) | ||
68 | elif subel.tag == 'rusage': | ||
69 | rusage = OrderedDict() | ||
70 | for field in rusage_fields: | ||
71 | if 'time' in field: | ||
72 | rusage[field] = float(subel.attrib[field]) | ||
73 | else: | ||
74 | rusage[field] = int(subel.attrib[field]) | ||
75 | values['rusage'] = rusage | ||
76 | elif subel.tag == 'iostat': | ||
77 | values['iostat'] = OrderedDict([(f, int(subel.attrib[f])) | ||
78 | for f in iostat_fields]) | ||
79 | elif subel.tag == 'buildstats_file': | ||
80 | values['buildstats_file'] = subel.text | ||
81 | else: | ||
82 | raise TypeError("Unknown sysres value element '{}'".format(subel.tag)) | ||
83 | # DISKUSAGE measurement | ||
84 | elif elem.tag == 'diskusage': | ||
85 | values['size'] = int(elem.find('size').text) | ||
86 | else: | ||
87 | raise Exception("Unknown measurement tag '{}'".format(elem.tag)) | ||
88 | data['values'] = values | ||
89 | return data | ||
90 | |||
91 | def _read_testcase(elem): | ||
92 | """Convert testcase into JSON""" | ||
93 | assert elem.tag == 'testcase', "Expecting 'testcase' element instead of {}".format(elem.tag) | ||
94 | |||
95 | data = OrderedDict() | ||
96 | data['name'] = elem.attrib['name'] | ||
97 | data['description'] = elem.attrib['description'] | ||
98 | data['status'] = 'SUCCESS' | ||
99 | data['start_time'] = isofmt_to_timestamp(elem.attrib['timestamp']) | ||
100 | data['elapsed_time'] = float(elem.attrib['time']) | ||
101 | measurements = OrderedDict() | ||
102 | |||
103 | for subel in elem.getchildren(): | ||
104 | if subel.tag == 'error' or subel.tag == 'failure': | ||
105 | data['status'] = subel.tag.upper() | ||
106 | data['message'] = subel.attrib['message'] | ||
107 | data['err_type'] = subel.attrib['type'] | ||
108 | data['err_output'] = subel.text | ||
109 | elif subel.tag == 'skipped': | ||
110 | data['status'] = 'SKIPPED' | ||
111 | data['message'] = subel.text | ||
112 | else: | ||
113 | measurements[subel.attrib['name']] = _read_measurement(subel) | ||
114 | data['measurements'] = measurements | ||
115 | return data | ||
116 | |||
117 | def _read_testsuite(elem): | ||
118 | """Convert suite to JSON""" | ||
119 | assert elem.tag == 'testsuite', \ | ||
120 | "Expecting 'testsuite' element instead of {}".format(elem.tag) | ||
121 | |||
122 | data = OrderedDict() | ||
123 | if 'hostname' in elem.attrib: | ||
124 | data['tester_host'] = elem.attrib['hostname'] | ||
125 | data['start_time'] = isofmt_to_timestamp(elem.attrib['timestamp']) | ||
126 | data['elapsed_time'] = float(elem.attrib['time']) | ||
127 | tests = OrderedDict() | ||
128 | |||
129 | for case in elem.getchildren(): | ||
130 | tests[case.attrib['name']] = _read_testcase(case) | ||
131 | data['tests'] = tests | ||
132 | return data | ||
133 | |||
134 | # Main function | ||
135 | assert elem.tag == 'testsuites', "Invalid test report format" | ||
136 | assert len(elem) == 1, "Too many testsuites" | ||
137 | |||
138 | return _read_testsuite(elem.getchildren()[0]) | ||
139 | |||
140 | |||
141 | def aggregate_metadata(metadata): | ||
142 | """Aggregate metadata into one, basically a sanity check""" | ||
143 | mutable_keys = ('pretty_name', 'version_id') | ||
144 | |||
145 | def aggregate_obj(aggregate, obj, assert_str=True): | ||
146 | """Aggregate objects together""" | ||
147 | assert type(aggregate) is type(obj), \ | ||
148 | "Type mismatch: {} != {}".format(type(aggregate), type(obj)) | ||
149 | if isinstance(obj, Mapping): | ||
150 | assert set(aggregate.keys()) == set(obj.keys()) | ||
151 | for key, val in obj.items(): | ||
152 | aggregate_obj(aggregate[key], val, key not in mutable_keys) | ||
153 | elif isinstance(obj, list): | ||
154 | assert len(aggregate) == len(obj) | ||
155 | for i, val in enumerate(obj): | ||
156 | aggregate_obj(aggregate[i], val) | ||
157 | elif not isinstance(obj, str) or (isinstance(obj, str) and assert_str): | ||
158 | assert aggregate == obj, "Data mismatch {} != {}".format(aggregate, obj) | ||
159 | |||
160 | if not metadata: | ||
161 | return {} | ||
162 | |||
163 | # Do the aggregation | ||
164 | aggregate = metadata[0].copy() | ||
165 | for testrun in metadata[1:]: | ||
166 | aggregate_obj(aggregate, testrun) | ||
167 | aggregate['testrun_count'] = len(metadata) | ||
168 | return aggregate | ||
169 | |||
170 | |||
171 | def aggregate_data(data): | ||
172 | """Aggregate multiple test results JSON structures into one""" | ||
173 | |||
174 | mutable_keys = ('status', 'message', 'err_type', 'err_output') | ||
175 | |||
176 | class SampleList(list): | ||
177 | """Container for numerical samples""" | ||
178 | pass | ||
179 | |||
180 | def new_aggregate_obj(obj): | ||
181 | """Create new object for aggregate""" | ||
182 | if isinstance(obj, Number): | ||
183 | new_obj = SampleList() | ||
184 | new_obj.append(obj) | ||
185 | elif isinstance(obj, str): | ||
186 | new_obj = obj | ||
187 | else: | ||
188 | # Lists and and dicts are kept as is | ||
189 | new_obj = obj.__class__() | ||
190 | aggregate_obj(new_obj, obj) | ||
191 | return new_obj | ||
192 | |||
193 | def aggregate_obj(aggregate, obj, assert_str=True): | ||
194 | """Recursive "aggregation" of JSON objects""" | ||
195 | if isinstance(obj, Number): | ||
196 | assert isinstance(aggregate, SampleList) | ||
197 | aggregate.append(obj) | ||
198 | return | ||
199 | |||
200 | assert type(aggregate) == type(obj), \ | ||
201 | "Type mismatch: {} != {}".format(type(aggregate), type(obj)) | ||
202 | if isinstance(obj, Mapping): | ||
203 | for key, val in obj.items(): | ||
204 | if not key in aggregate: | ||
205 | aggregate[key] = new_aggregate_obj(val) | ||
206 | else: | ||
207 | aggregate_obj(aggregate[key], val, key not in mutable_keys) | ||
208 | elif isinstance(obj, list): | ||
209 | for i, val in enumerate(obj): | ||
210 | if i >= len(aggregate): | ||
211 | aggregate[key] = new_aggregate_obj(val) | ||
212 | else: | ||
213 | aggregate_obj(aggregate[i], val) | ||
214 | elif isinstance(obj, str): | ||
215 | # Sanity check for data | ||
216 | if assert_str: | ||
217 | assert aggregate == obj, "Data mismatch {} != {}".format(aggregate, obj) | ||
218 | else: | ||
219 | raise Exception("BUG: unable to aggregate '{}' ({})".format(type(obj), str(obj))) | ||
220 | |||
221 | if not data: | ||
222 | return {} | ||
223 | |||
224 | # Do the aggregation | ||
225 | aggregate = data[0].__class__() | ||
226 | for testrun in data: | ||
227 | aggregate_obj(aggregate, testrun) | ||
228 | return aggregate | ||
229 | |||
230 | |||
231 | class MeasurementVal(float): | ||
232 | """Base class representing measurement values""" | ||
233 | gv_data_type = 'number' | ||
234 | |||
235 | def gv_value(self): | ||
236 | """Value formatting for visualization""" | ||
237 | if self != self: | ||
238 | return "null" | ||
239 | else: | ||
240 | return self | ||
241 | |||
242 | |||
243 | class TimeVal(MeasurementVal): | ||
244 | """Class representing time values""" | ||
245 | quantity = 'time' | ||
246 | gv_title = 'elapsed time' | ||
247 | gv_data_type = 'timeofday' | ||
248 | |||
249 | def hms(self): | ||
250 | """Split time into hours, minutes and seconeds""" | ||
251 | hhh = int(abs(self) / 3600) | ||
252 | mmm = int((abs(self) % 3600) / 60) | ||
253 | sss = abs(self) % 60 | ||
254 | return hhh, mmm, sss | ||
255 | |||
256 | def __str__(self): | ||
257 | if self != self: | ||
258 | return "nan" | ||
259 | hh, mm, ss = self.hms() | ||
260 | sign = '-' if self < 0 else '' | ||
261 | if hh > 0: | ||
262 | return '{}{:d}:{:02d}:{:02.0f}'.format(sign, hh, mm, ss) | ||
263 | elif mm > 0: | ||
264 | return '{}{:d}:{:04.1f}'.format(sign, mm, ss) | ||
265 | elif ss > 1: | ||
266 | return '{}{:.1f} s'.format(sign, ss) | ||
267 | else: | ||
268 | return '{}{:.2f} s'.format(sign, ss) | ||
269 | |||
270 | def gv_value(self): | ||
271 | """Value formatting for visualization""" | ||
272 | if self != self: | ||
273 | return "null" | ||
274 | hh, mm, ss = self.hms() | ||
275 | return [hh, mm, int(ss), int(ss*1000) % 1000] | ||
276 | |||
277 | |||
278 | class SizeVal(MeasurementVal): | ||
279 | """Class representing time values""" | ||
280 | quantity = 'size' | ||
281 | gv_title = 'size in MiB' | ||
282 | gv_data_type = 'number' | ||
283 | |||
284 | def __str__(self): | ||
285 | if self != self: | ||
286 | return "nan" | ||
287 | if abs(self) < 1024: | ||
288 | return '{:.1f} kiB'.format(self) | ||
289 | elif abs(self) < 1048576: | ||
290 | return '{:.2f} MiB'.format(self / 1024) | ||
291 | else: | ||
292 | return '{:.2f} GiB'.format(self / 1048576) | ||
293 | |||
294 | def gv_value(self): | ||
295 | """Value formatting for visualization""" | ||
296 | if self != self: | ||
297 | return "null" | ||
298 | return self / 1024 | ||
299 | |||
300 | def measurement_stats(meas, prefix=''): | ||
301 | """Get statistics of a measurement""" | ||
302 | if not meas: | ||
303 | return {prefix + 'sample_cnt': 0, | ||
304 | prefix + 'mean': MeasurementVal('nan'), | ||
305 | prefix + 'stdev': MeasurementVal('nan'), | ||
306 | prefix + 'variance': MeasurementVal('nan'), | ||
307 | prefix + 'min': MeasurementVal('nan'), | ||
308 | prefix + 'max': MeasurementVal('nan'), | ||
309 | prefix + 'minus': MeasurementVal('nan'), | ||
310 | prefix + 'plus': MeasurementVal('nan')} | ||
311 | |||
312 | stats = {'name': meas['name']} | ||
313 | if meas['type'] == 'sysres': | ||
314 | val_cls = TimeVal | ||
315 | values = meas['values']['elapsed_time'] | ||
316 | elif meas['type'] == 'diskusage': | ||
317 | val_cls = SizeVal | ||
318 | values = meas['values']['size'] | ||
319 | else: | ||
320 | raise Exception("Unknown measurement type '{}'".format(meas['type'])) | ||
321 | stats['val_cls'] = val_cls | ||
322 | stats['quantity'] = val_cls.quantity | ||
323 | stats[prefix + 'sample_cnt'] = len(values) | ||
324 | |||
325 | mean_val = val_cls(mean(values)) | ||
326 | min_val = val_cls(min(values)) | ||
327 | max_val = val_cls(max(values)) | ||
328 | |||
329 | stats[prefix + 'mean'] = mean_val | ||
330 | if len(values) > 1: | ||
331 | stats[prefix + 'stdev'] = val_cls(stdev(values)) | ||
332 | stats[prefix + 'variance'] = val_cls(variance(values)) | ||
333 | else: | ||
334 | stats[prefix + 'stdev'] = float('nan') | ||
335 | stats[prefix + 'variance'] = float('nan') | ||
336 | stats[prefix + 'min'] = min_val | ||
337 | stats[prefix + 'max'] = max_val | ||
338 | stats[prefix + 'minus'] = val_cls(mean_val - min_val) | ||
339 | stats[prefix + 'plus'] = val_cls(max_val - mean_val) | ||
340 | |||
341 | return stats | ||
342 | |||