diff options
author | Markus Lehtonen <markus.lehtonen@linux.intel.com> | 2017-09-15 16:04:37 +0300 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2017-09-18 11:07:30 +0100 |
commit | b5fb3dd904cd22212c720f3c79d71952ecbaa9c2 (patch) | |
tree | 69620698ccaa22f97cfc7b3eb8be3e3118086c19 /scripts/lib/buildstats.py | |
parent | 873707489fafaa5e1bd43b03b068df74d5956b00 (diff) | |
download | poky-b5fb3dd904cd22212c720f3c79d71952ecbaa9c2.tar.gz |
scripts/buildstats-diff: move code to lib/buildstats.py
Move over code from buildstats-diff to new scripts/lib/buildstats.py
module in order to share code related to buildstats processing. Also,
refactor the code, introducing new classes to make the code readable,
maintainable and easier to debug.
[YOCTO #11381]
(From OE-Core rev: 8a2cd9afc95919737d8e75234e78bbc52e1494a1)
Signed-off-by: Markus Lehtonen <markus.lehtonen@linux.intel.com>
Signed-off-by: Ross Burton <ross.burton@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/buildstats.py')
-rw-r--r-- | scripts/lib/buildstats.py | 301 |
1 files changed, 301 insertions, 0 deletions
diff --git a/scripts/lib/buildstats.py b/scripts/lib/buildstats.py new file mode 100644 index 0000000000..9eb60b1c69 --- /dev/null +++ b/scripts/lib/buildstats.py | |||
@@ -0,0 +1,301 @@ | |||
1 | # | ||
2 | # Copyright (c) 2017, Intel Corporation. | ||
3 | # | ||
4 | # This program is free software; you can redistribute it and/or modify it | ||
5 | # under the terms and conditions of the GNU General Public License, | ||
6 | # version 2, as published by the Free Software Foundation. | ||
7 | # | ||
8 | # This program is distributed in the hope it will be useful, but WITHOUT | ||
9 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||
10 | # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
11 | # more details. | ||
12 | # | ||
13 | """Functionality for analyzing buildstats""" | ||
14 | import json | ||
15 | import logging | ||
16 | import os | ||
17 | import re | ||
18 | from collections import namedtuple | ||
19 | from statistics import mean | ||
20 | |||
21 | |||
22 | log = logging.getLogger() | ||
23 | |||
24 | |||
25 | taskdiff_fields = ('pkg', 'pkg_op', 'task', 'task_op', 'value1', 'value2', | ||
26 | 'absdiff', 'reldiff') | ||
27 | TaskDiff = namedtuple('TaskDiff', ' '.join(taskdiff_fields)) | ||
28 | |||
29 | |||
30 | class BSError(Exception): | ||
31 | """Error handling of buildstats""" | ||
32 | pass | ||
33 | |||
34 | |||
35 | class BSTask(dict): | ||
36 | def __init__(self, *args, **kwargs): | ||
37 | self['start_time'] = None | ||
38 | self['elapsed_time'] = None | ||
39 | self['status'] = None | ||
40 | self['iostat'] = {} | ||
41 | self['rusage'] = {} | ||
42 | self['child_rusage'] = {} | ||
43 | super(BSTask, self).__init__(*args, **kwargs) | ||
44 | |||
45 | @property | ||
46 | def cputime(self): | ||
47 | """Sum of user and system time taken by the task""" | ||
48 | rusage = self['rusage']['ru_stime'] + self['rusage']['ru_utime'] | ||
49 | if self['child_rusage']: | ||
50 | # Child rusage may have been optimized out | ||
51 | return rusage + self['child_rusage']['ru_stime'] + self['child_rusage']['ru_utime'] | ||
52 | else: | ||
53 | return rusage | ||
54 | |||
55 | @property | ||
56 | def walltime(self): | ||
57 | """Elapsed wall clock time""" | ||
58 | return self['elapsed_time'] | ||
59 | |||
60 | @property | ||
61 | def read_bytes(self): | ||
62 | """Bytes read from the block layer""" | ||
63 | return self['iostat']['read_bytes'] | ||
64 | |||
65 | @property | ||
66 | def write_bytes(self): | ||
67 | """Bytes written to the block layer""" | ||
68 | return self['iostat']['write_bytes'] | ||
69 | |||
70 | @property | ||
71 | def read_ops(self): | ||
72 | """Number of read operations on the block layer""" | ||
73 | if self['child_rusage']: | ||
74 | # Child rusage may have been optimized out | ||
75 | return self['rusage']['ru_inblock'] + self['child_rusage']['ru_inblock'] | ||
76 | else: | ||
77 | return self['rusage']['ru_inblock'] | ||
78 | |||
79 | @property | ||
80 | def write_ops(self): | ||
81 | """Number of write operations on the block layer""" | ||
82 | if self['child_rusage']: | ||
83 | # Child rusage may have been optimized out | ||
84 | return self['rusage']['ru_oublock'] + self['child_rusage']['ru_oublock'] | ||
85 | else: | ||
86 | return self['rusage']['ru_oublock'] | ||
87 | |||
88 | @classmethod | ||
89 | def from_file(cls, buildstat_file): | ||
90 | """Read buildstat text file""" | ||
91 | bs_task = cls() | ||
92 | log.debug("Reading task buildstats from %s", buildstat_file) | ||
93 | end_time = None | ||
94 | with open(buildstat_file) as fobj: | ||
95 | for line in fobj.readlines(): | ||
96 | key, val = line.split(':', 1) | ||
97 | val = val.strip() | ||
98 | if key == 'Started': | ||
99 | start_time = float(val) | ||
100 | bs_task['start_time'] = start_time | ||
101 | elif key == 'Ended': | ||
102 | end_time = float(val) | ||
103 | elif key.startswith('IO '): | ||
104 | split = key.split() | ||
105 | bs_task['iostat'][split[1]] = int(val) | ||
106 | elif key.find('rusage') >= 0: | ||
107 | split = key.split() | ||
108 | ru_key = split[-1] | ||
109 | if ru_key in ('ru_stime', 'ru_utime'): | ||
110 | val = float(val) | ||
111 | else: | ||
112 | val = int(val) | ||
113 | ru_type = 'rusage' if split[0] == 'rusage' else \ | ||
114 | 'child_rusage' | ||
115 | bs_task[ru_type][ru_key] = val | ||
116 | elif key == 'Status': | ||
117 | bs_task['status'] = val | ||
118 | if end_time is not None and start_time is not None: | ||
119 | bs_task['elapsed_time'] = end_time - start_time | ||
120 | else: | ||
121 | raise BSError("{} looks like a invalid buildstats file".format(buildstat_file)) | ||
122 | return bs_task | ||
123 | |||
124 | |||
125 | class BSTaskAggregate(object): | ||
126 | """Class representing multiple runs of the same task""" | ||
127 | properties = ('cputime', 'walltime', 'read_bytes', 'write_bytes', | ||
128 | 'read_ops', 'write_ops') | ||
129 | |||
130 | def __init__(self, tasks=None): | ||
131 | self._tasks = tasks or [] | ||
132 | self._properties = {} | ||
133 | |||
134 | def __getattr__(self, name): | ||
135 | if name in self.properties: | ||
136 | if name not in self._properties: | ||
137 | # Calculate properties on demand only. We only provide mean | ||
138 | # value, so far | ||
139 | self._properties[name] = mean([getattr(t, name) for t in self._tasks]) | ||
140 | return self._properties[name] | ||
141 | else: | ||
142 | raise AttributeError("'BSTaskAggregate' has no attribute '{}'".format(name)) | ||
143 | |||
144 | def append(self, task): | ||
145 | """Append new task""" | ||
146 | # Reset pre-calculated properties | ||
147 | assert isinstance(task, BSTask), "Type is '{}' instead of 'BSTask'".format(type(task)) | ||
148 | self._properties = {} | ||
149 | self._tasks.append(task) | ||
150 | |||
151 | |||
152 | class BSRecipe(object): | ||
153 | """Class representing buildstats of one recipe""" | ||
154 | def __init__(self, name, epoch, version, revision): | ||
155 | self.name = name | ||
156 | self.epoch = epoch | ||
157 | self.version = version | ||
158 | self.revision = revision | ||
159 | if epoch is None: | ||
160 | self.nevr = "{}-{}-{}".format(name, version, revision) | ||
161 | else: | ||
162 | self.nevr = "{}-{}_{}-{}".format(name, epoch, version, revision) | ||
163 | self.tasks = {} | ||
164 | |||
165 | def aggregate(self, bsrecipe): | ||
166 | """Aggregate data of another recipe buildstats""" | ||
167 | if self.nevr != bsrecipe.nevr: | ||
168 | raise ValueError("Refusing to aggregate buildstats, recipe version " | ||
169 | "differs: {} vs. {}".format(self.nevr, bsrecipe.nevr)) | ||
170 | if set(self.tasks.keys()) != set(bsrecipe.tasks.keys()): | ||
171 | raise ValueError("Refusing to aggregate buildstats, set of tasks " | ||
172 | "in {} differ".format(self.name)) | ||
173 | |||
174 | for taskname, taskdata in bsrecipe.tasks.items(): | ||
175 | if not isinstance(self.tasks[taskname], BSTaskAggregate): | ||
176 | self.tasks[taskname] = BSTaskAggregate([self.tasks[taskname]]) | ||
177 | self.tasks[taskname].append(taskdata) | ||
178 | |||
179 | |||
180 | class BuildStats(dict): | ||
181 | """Class representing buildstats of one build""" | ||
182 | |||
183 | @classmethod | ||
184 | def from_json(cls, bs_json): | ||
185 | """Create new BuildStats object from JSON object""" | ||
186 | buildstats = cls() | ||
187 | for recipe in bs_json: | ||
188 | if recipe['name'] in buildstats: | ||
189 | raise BSError("Cannot handle multiple versions of the same " | ||
190 | "package ({})".format(recipe['name'])) | ||
191 | bsrecipe = BSRecipe(recipe['name'], recipe['epoch'], | ||
192 | recipe['version'], recipe['revision']) | ||
193 | for task, data in recipe['tasks'].items(): | ||
194 | bsrecipe.tasks[task] = BSTask(data) | ||
195 | |||
196 | buildstats[recipe['name']] = bsrecipe | ||
197 | |||
198 | return buildstats | ||
199 | |||
200 | @staticmethod | ||
201 | def from_file_json(path): | ||
202 | """Load buildstats from a JSON file""" | ||
203 | with open(path) as fobj: | ||
204 | bs_json = json.load(fobj) | ||
205 | return BuildStats.from_json(bs_json) | ||
206 | |||
207 | |||
208 | @staticmethod | ||
209 | def split_nevr(nevr): | ||
210 | """Split name and version information from recipe "nevr" string""" | ||
211 | n_e_v, revision = nevr.rsplit('-', 1) | ||
212 | match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[0-9]\S*)$', | ||
213 | n_e_v) | ||
214 | if not match: | ||
215 | # If we're not able to parse a version starting with a number, just | ||
216 | # take the part after last dash | ||
217 | match = re.match(r'^(?P<name>\S+)-((?P<epoch>[0-9]{1,5})_)?(?P<version>[^-]+)$', | ||
218 | n_e_v) | ||
219 | name = match.group('name') | ||
220 | version = match.group('version') | ||
221 | epoch = match.group('epoch') | ||
222 | return name, epoch, version, revision | ||
223 | |||
224 | @classmethod | ||
225 | def from_dir(cls, path): | ||
226 | """Load buildstats from a buildstats directory""" | ||
227 | if not os.path.isfile(os.path.join(path, 'build_stats')): | ||
228 | raise BSError("{} does not look like a buildstats directory".format(path)) | ||
229 | |||
230 | log.debug("Reading buildstats directory %s", path) | ||
231 | |||
232 | buildstats = cls() | ||
233 | subdirs = os.listdir(path) | ||
234 | for dirname in subdirs: | ||
235 | recipe_dir = os.path.join(path, dirname) | ||
236 | if not os.path.isdir(recipe_dir): | ||
237 | continue | ||
238 | name, epoch, version, revision = cls.split_nevr(dirname) | ||
239 | bsrecipe = BSRecipe(name, epoch, version, revision) | ||
240 | for task in os.listdir(recipe_dir): | ||
241 | bsrecipe.tasks[task] = BSTask.from_file( | ||
242 | os.path.join(recipe_dir, task)) | ||
243 | if name in buildstats: | ||
244 | raise BSError("Cannot handle multiple versions of the same " | ||
245 | "package ({})".format(name)) | ||
246 | buildstats[name] = bsrecipe | ||
247 | |||
248 | return buildstats | ||
249 | |||
250 | def aggregate(self, buildstats): | ||
251 | """Aggregate other buildstats into this""" | ||
252 | if set(self.keys()) != set(buildstats.keys()): | ||
253 | raise ValueError("Refusing to aggregate buildstats, set of " | ||
254 | "recipes is different") | ||
255 | for pkg, data in buildstats.items(): | ||
256 | self[pkg].aggregate(data) | ||
257 | |||
258 | |||
259 | def diff_buildstats(bs1, bs2, stat_attr, min_val=None, min_absdiff=None): | ||
260 | """Compare the tasks of two buildstats""" | ||
261 | tasks_diff = [] | ||
262 | pkgs = set(bs1.keys()).union(set(bs2.keys())) | ||
263 | for pkg in pkgs: | ||
264 | tasks1 = bs1[pkg].tasks if pkg in bs1 else {} | ||
265 | tasks2 = bs2[pkg].tasks if pkg in bs2 else {} | ||
266 | if not tasks1: | ||
267 | pkg_op = '+' | ||
268 | elif not tasks2: | ||
269 | pkg_op = '-' | ||
270 | else: | ||
271 | pkg_op = ' ' | ||
272 | |||
273 | for task in set(tasks1.keys()).union(set(tasks2.keys())): | ||
274 | task_op = ' ' | ||
275 | if task in tasks1: | ||
276 | val1 = getattr(bs1[pkg].tasks[task], stat_attr) | ||
277 | else: | ||
278 | task_op = '+' | ||
279 | val1 = 0 | ||
280 | if task in tasks2: | ||
281 | val2 = getattr(bs2[pkg].tasks[task], stat_attr) | ||
282 | else: | ||
283 | val2 = 0 | ||
284 | task_op = '-' | ||
285 | |||
286 | if val1 == 0: | ||
287 | reldiff = float('inf') | ||
288 | else: | ||
289 | reldiff = 100 * (val2 - val1) / val1 | ||
290 | |||
291 | if min_val and max(val1, val2) < min_val: | ||
292 | log.debug("Filtering out %s:%s (%s)", pkg, task, | ||
293 | max(val1, val2)) | ||
294 | continue | ||
295 | if min_absdiff and abs(val2 - val1) < min_absdiff: | ||
296 | log.debug("Filtering out %s:%s (difference of %s)", pkg, task, | ||
297 | val2-val1) | ||
298 | continue | ||
299 | tasks_diff.append(TaskDiff(pkg, pkg_op, task, task_op, val1, val2, | ||
300 | val2-val1, reldiff)) | ||
301 | return tasks_diff | ||