diff options
author | Tudor Florea <tudor.florea@enea.com> | 2014-10-16 03:05:19 +0200 |
---|---|---|
committer | Tudor Florea <tudor.florea@enea.com> | 2014-10-16 03:05:19 +0200 |
commit | c527fd1f14c27855a37f2e8ac5346ce8d940ced2 (patch) | |
tree | bb002c1fdf011c41dbd2f0927bed23ecb5f83c97 /meta/classes/buildstats.bbclass | |
download | poky-c527fd1f14c27855a37f2e8ac5346ce8d940ced2.tar.gz |
initial commit for Enea Linux 4.0-140929daisy-140929
Migrated from the internal git server on the daisy-enea-point-release branch
Signed-off-by: Tudor Florea <tudor.florea@enea.com>
Diffstat (limited to 'meta/classes/buildstats.bbclass')
-rw-r--r-- | meta/classes/buildstats.bbclass | 289 |
1 files changed, 289 insertions, 0 deletions
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass new file mode 100644 index 0000000000..89ae72c679 --- /dev/null +++ b/meta/classes/buildstats.bbclass | |||
@@ -0,0 +1,289 @@ | |||
1 | BUILDSTATS_BASE = "${TMPDIR}/buildstats/" | ||
2 | BNFILE = "${BUILDSTATS_BASE}/.buildname" | ||
3 | DEVFILE = "${BUILDSTATS_BASE}/.device" | ||
4 | |||
5 | ################################################################################ | ||
6 | # Build statistics gathering. | ||
7 | # | ||
8 | # The CPU and Time gathering/tracking functions and bbevent inspiration | ||
9 | # were written by Christopher Larson and can be seen here: | ||
10 | # http://kergoth.pastey.net/142813 | ||
11 | # | ||
12 | ################################################################################ | ||
13 | |||
14 | def get_process_cputime(pid): | ||
15 | with open("/proc/%d/stat" % pid, "r") as f: | ||
16 | fields = f.readline().rstrip().split() | ||
17 | # 13: utime, 14: stime, 15: cutime, 16: cstime | ||
18 | return sum(int(field) for field in fields[13:16]) | ||
19 | |||
20 | def get_cputime(): | ||
21 | with open("/proc/stat", "r") as f: | ||
22 | fields = f.readline().rstrip().split()[1:] | ||
23 | return sum(int(field) for field in fields) | ||
24 | |||
25 | def set_bn(e): | ||
26 | bn = e.getPkgs()[0] + "-" + e.data.getVar('MACHINE', True) | ||
27 | try: | ||
28 | os.remove(e.data.getVar('BNFILE', True)) | ||
29 | except: | ||
30 | pass | ||
31 | with open(e.data.getVar('BNFILE', True), "w") as f: | ||
32 | f.write(os.path.join(bn, e.data.getVar('BUILDNAME', True))) | ||
33 | |||
34 | def get_bn(e): | ||
35 | with open(e.data.getVar('BNFILE', True)) as f: | ||
36 | bn = f.readline() | ||
37 | return bn | ||
38 | |||
39 | def set_device(e): | ||
40 | tmpdir = e.data.getVar('TMPDIR', True) | ||
41 | try: | ||
42 | os.remove(e.data.getVar('DEVFILE', True)) | ||
43 | except: | ||
44 | pass | ||
45 | ############################################################################ | ||
46 | # We look for the volume TMPDIR lives on. To do all disks would make little | ||
47 | # sense and not give us any particularly useful data. In theory we could do | ||
48 | # something like stick DL_DIR on a different partition and this would | ||
49 | # throw stats gathering off. The same goes with SSTATE_DIR. However, let's | ||
50 | # get the basics in here and work on the cornercases later. | ||
51 | # A note. /proc/diskstats does not contain info on encryptfs, tmpfs, etc. | ||
52 | # If we end up hitting one of these fs, we'll just skip diskstats collection. | ||
53 | ############################################################################ | ||
54 | device=os.stat(tmpdir) | ||
55 | majordev=os.major(device.st_dev) | ||
56 | minordev=os.minor(device.st_dev) | ||
57 | ############################################################################ | ||
58 | # Bug 1700: | ||
59 | # Because tmpfs/encryptfs/ramfs etc inserts no entry in /proc/diskstats | ||
60 | # we set rdev to NoLogicalDevice and search for it later. If we find NLD | ||
61 | # we do not collect diskstats as the method to collect meaningful statistics | ||
62 | # for these fs types requires a bit more research. | ||
63 | ############################################################################ | ||
64 | rdev="NoLogicalDevice" | ||
65 | try: | ||
66 | with open("/proc/diskstats", "r") as f: | ||
67 | for line in f: | ||
68 | if majordev == int(line.split()[0]) and minordev == int(line.split()[1]): | ||
69 | rdev=line.split()[2] | ||
70 | except: | ||
71 | pass | ||
72 | file = open(e.data.getVar('DEVFILE', True), "w") | ||
73 | file.write(rdev) | ||
74 | file.close() | ||
75 | |||
76 | def get_device(e): | ||
77 | file = open(e.data.getVar('DEVFILE', True)) | ||
78 | device = file.readline() | ||
79 | file.close() | ||
80 | return device | ||
81 | |||
82 | def get_diskstats(dev): | ||
83 | import itertools | ||
84 | ############################################################################ | ||
85 | # For info on what these are, see kernel doc file iostats.txt | ||
86 | ############################################################################ | ||
87 | DSTAT_KEYS = ['ReadsComp', 'ReadsMerged', 'SectRead', 'TimeReads', 'WritesComp', 'SectWrite', 'TimeWrite', 'IOinProgress', 'TimeIO', 'WTimeIO'] | ||
88 | try: | ||
89 | with open("/proc/diskstats", "r") as f: | ||
90 | for x in f: | ||
91 | if dev in x: | ||
92 | diskstats_val = x.rstrip().split()[4:] | ||
93 | except IOError as e: | ||
94 | return | ||
95 | diskstats = dict(itertools.izip(DSTAT_KEYS, diskstats_val)) | ||
96 | return diskstats | ||
97 | |||
98 | def set_diskdata(var, dev, data): | ||
99 | data.setVar(var, get_diskstats(dev)) | ||
100 | |||
101 | def get_diskdata(var, dev, data): | ||
102 | olddiskdata = data.getVar(var, False) | ||
103 | diskdata = {} | ||
104 | if olddiskdata is None: | ||
105 | return | ||
106 | newdiskdata = get_diskstats(dev) | ||
107 | for key in olddiskdata.iterkeys(): | ||
108 | diskdata["Start"+key] = str(int(olddiskdata[key])) | ||
109 | diskdata["End"+key] = str(int(newdiskdata[key])) | ||
110 | return diskdata | ||
111 | |||
112 | def set_timedata(var, data, server_time=None): | ||
113 | import time | ||
114 | if server_time: | ||
115 | time = server_time | ||
116 | else: | ||
117 | time = time.time() | ||
118 | cputime = get_cputime() | ||
119 | proctime = get_process_cputime(os.getpid()) | ||
120 | data.setVar(var, (time, cputime, proctime)) | ||
121 | |||
122 | def get_timedata(var, data, server_time=None): | ||
123 | import time | ||
124 | timedata = data.getVar(var, False) | ||
125 | if timedata is None: | ||
126 | return | ||
127 | oldtime, oldcpu, oldproc = timedata | ||
128 | procdiff = get_process_cputime(os.getpid()) - oldproc | ||
129 | cpudiff = get_cputime() - oldcpu | ||
130 | if server_time: | ||
131 | end_time = server_time | ||
132 | else: | ||
133 | end_time = time.time() | ||
134 | timediff = end_time - oldtime | ||
135 | if cpudiff > 0: | ||
136 | cpuperc = float(procdiff) * 100 / cpudiff | ||
137 | else: | ||
138 | cpuperc = None | ||
139 | return timediff, cpuperc | ||
140 | |||
141 | def write_task_data(status, logfile, dev, e): | ||
142 | bn = get_bn(e) | ||
143 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
144 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
145 | file = open(os.path.join(logfile), "a") | ||
146 | timedata = get_timedata("__timedata_task", e.data, e.time) | ||
147 | if timedata: | ||
148 | elapsedtime, cpu = timedata | ||
149 | file.write(bb.data.expand("${PF}: %s: Elapsed time: %0.2f seconds \n" % | ||
150 | (e.task, elapsedtime), e.data)) | ||
151 | if cpu: | ||
152 | file.write("CPU usage: %0.1f%% \n" % cpu) | ||
153 | ############################################################################ | ||
154 | # Here we gather up disk data. In an effort to avoid lying with stats | ||
155 | # I do a bare minimum of analysis of collected data. | ||
156 | # The simple fact is, doing disk io collection on a per process basis | ||
157 | # without effecting build time would be difficult. | ||
158 | # For the best information, running things with BB_TOTAL_THREADS = "1" | ||
159 | # would return accurate per task results. | ||
160 | ############################################################################ | ||
161 | if dev != "NoLogicalDevice": | ||
162 | diskdata = get_diskdata("__diskdata_task", dev, e.data) | ||
163 | if diskdata: | ||
164 | for key in sorted(diskdata.iterkeys()): | ||
165 | file.write(key + ": " + diskdata[key] + "\n") | ||
166 | if status is "passed": | ||
167 | file.write("Status: PASSED \n") | ||
168 | else: | ||
169 | file.write("Status: FAILED \n") | ||
170 | file.write("Ended: %0.2f \n" % e.time) | ||
171 | file.close() | ||
172 | |||
173 | python run_buildstats () { | ||
174 | import bb.build | ||
175 | import bb.event | ||
176 | import bb.data | ||
177 | import time, subprocess, platform | ||
178 | |||
179 | if isinstance(e, bb.event.BuildStarted): | ||
180 | ######################################################################## | ||
181 | # at first pass make the buildstats heriarchy and then | ||
182 | # set the buildname | ||
183 | ######################################################################## | ||
184 | try: | ||
185 | bb.utils.mkdirhier(e.data.getVar('BUILDSTATS_BASE', True)) | ||
186 | except: | ||
187 | pass | ||
188 | set_bn(e) | ||
189 | bn = get_bn(e) | ||
190 | set_device(e) | ||
191 | device = get_device(e) | ||
192 | |||
193 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
194 | try: | ||
195 | bb.utils.mkdirhier(bsdir) | ||
196 | except: | ||
197 | pass | ||
198 | if device != "NoLogicalDevice": | ||
199 | set_diskdata("__diskdata_build", device, e.data) | ||
200 | set_timedata("__timedata_build", e.data) | ||
201 | build_time = os.path.join(bsdir, "build_stats") | ||
202 | # write start of build into build_time | ||
203 | file = open(build_time,"a") | ||
204 | host_info = platform.uname() | ||
205 | file.write("Host Info: ") | ||
206 | for x in host_info: | ||
207 | if x: | ||
208 | file.write(x + " ") | ||
209 | file.write("\n") | ||
210 | file.write("Build Started: %0.2f \n" % time.time()) | ||
211 | file.close() | ||
212 | |||
213 | elif isinstance(e, bb.event.BuildCompleted): | ||
214 | bn = get_bn(e) | ||
215 | device = get_device(e) | ||
216 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
217 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
218 | build_time = os.path.join(bsdir, "build_stats") | ||
219 | file = open(build_time, "a") | ||
220 | ######################################################################## | ||
221 | # Write build statistics for the build | ||
222 | ######################################################################## | ||
223 | timedata = get_timedata("__timedata_build", e.data) | ||
224 | if timedata: | ||
225 | time, cpu = timedata | ||
226 | # write end of build and cpu used into build_time | ||
227 | file.write("Elapsed time: %0.2f seconds \n" % (time)) | ||
228 | if cpu: | ||
229 | file.write("CPU usage: %0.1f%% \n" % cpu) | ||
230 | if device != "NoLogicalDevice": | ||
231 | diskio = get_diskdata("__diskdata_build", device, e.data) | ||
232 | if diskio: | ||
233 | for key in sorted(diskio.iterkeys()): | ||
234 | file.write(key + ": " + diskio[key] + "\n") | ||
235 | file.close() | ||
236 | |||
237 | if isinstance(e, bb.build.TaskStarted): | ||
238 | bn = get_bn(e) | ||
239 | device = get_device(e) | ||
240 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
241 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
242 | if device != "NoLogicalDevice": | ||
243 | set_diskdata("__diskdata_task", device, e.data) | ||
244 | set_timedata("__timedata_task", e.data, e.time) | ||
245 | try: | ||
246 | bb.utils.mkdirhier(taskdir) | ||
247 | except: | ||
248 | pass | ||
249 | # write into the task event file the name and start time | ||
250 | file = open(os.path.join(taskdir, e.task), "a") | ||
251 | file.write("Event: %s \n" % bb.event.getName(e)) | ||
252 | file.write("Started: %0.2f \n" % e.time) | ||
253 | file.close() | ||
254 | |||
255 | elif isinstance(e, bb.build.TaskSucceeded): | ||
256 | bn = get_bn(e) | ||
257 | device = get_device(e) | ||
258 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
259 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
260 | write_task_data("passed", os.path.join(taskdir, e.task), device, e) | ||
261 | if e.task == "do_rootfs": | ||
262 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
263 | bs=os.path.join(bsdir, "build_stats") | ||
264 | file = open(bs,"a") | ||
265 | rootfs = e.data.getVar('IMAGE_ROOTFS', True) | ||
266 | rootfs_size = subprocess.Popen(["du", "-sh", rootfs], stdout=subprocess.PIPE).stdout.read() | ||
267 | file.write("Uncompressed Rootfs size: %s" % rootfs_size) | ||
268 | file.close() | ||
269 | |||
270 | elif isinstance(e, bb.build.TaskFailed): | ||
271 | bn = get_bn(e) | ||
272 | device = get_device(e) | ||
273 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
274 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
275 | write_task_data("failed", os.path.join(taskdir, e.task), device, e) | ||
276 | ######################################################################## | ||
277 | # Lets make things easier and tell people where the build failed in | ||
278 | # build_status. We do this here because BuildCompleted triggers no | ||
279 | # matter what the status of the build actually is | ||
280 | ######################################################################## | ||
281 | build_status = os.path.join(bsdir, "build_stats") | ||
282 | file = open(build_status,"a") | ||
283 | file.write(e.data.expand("Failed at: ${PF} at task: %s \n" % e.task)) | ||
284 | file.close() | ||
285 | } | ||
286 | |||
287 | addhandler run_buildstats | ||
288 | run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed" | ||
289 | |||