diff options
Diffstat (limited to 'meta/classes/buildstats.bbclass')
-rw-r--r-- | meta/classes/buildstats.bbclass | 282 |
1 files changed, 282 insertions, 0 deletions
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass new file mode 100644 index 0000000000..72fff1167f --- /dev/null +++ b/meta/classes/buildstats.bbclass | |||
@@ -0,0 +1,282 @@ | |||
1 | BUILDSTATS_BASE = "${TMPDIR}/buildstats/" | ||
2 | BNFILE = "${BUILDSTATS_BASE}/.buildname" | ||
3 | DEVFILE = "${BUILDSTATS_BASE}/.device" | ||
4 | |||
5 | ################################################################################ | ||
6 | # Build statistics gathering. | ||
7 | # | ||
8 | # The CPU and Time gathering/tracking functions and bbevent inspiration | ||
9 | # were written by Christopher Larson and can be seen here: | ||
10 | # http://kergoth.pastey.net/142813 | ||
11 | # | ||
12 | ################################################################################ | ||
13 | |||
14 | def get_process_cputime(pid): | ||
15 | with open("/proc/%d/stat" % pid, "r") as f: | ||
16 | fields = f.readline().rstrip().split() | ||
17 | # 13: utime, 14: stime, 15: cutime, 16: cstime | ||
18 | return sum(int(field) for field in fields[13:16]) | ||
19 | |||
20 | def get_cputime(): | ||
21 | with open("/proc/stat", "r") as f: | ||
22 | fields = f.readline().rstrip().split()[1:] | ||
23 | return sum(int(field) for field in fields) | ||
24 | |||
25 | def set_bn(e): | ||
26 | bn = e.getPkgs()[0] + "-" + e.data.getVar('MACHINE', True) | ||
27 | try: | ||
28 | os.remove(e.data.getVar('BNFILE', True)) | ||
29 | except: | ||
30 | pass | ||
31 | with open(e.data.getVar('BNFILE', True), "w") as f: | ||
32 | f.write(os.path.join(bn, e.data.getVar('BUILDNAME', True))) | ||
33 | |||
34 | def get_bn(e): | ||
35 | with open(e.data.getVar('BNFILE', True)) as f: | ||
36 | bn = f.readline() | ||
37 | return bn | ||
38 | |||
39 | def set_device(e): | ||
40 | tmpdir = e.data.getVar('TMPDIR', True) | ||
41 | try: | ||
42 | os.remove(e.data.getVar('DEVFILE', True)) | ||
43 | except: | ||
44 | pass | ||
45 | ############################################################################ | ||
46 | # We look for the volume TMPDIR lives on. To do all disks would make little | ||
47 | # sense and not give us any particularly useful data. In theory we could do | ||
48 | # something like stick DL_DIR on a different partition and this would | ||
49 | # throw stats gathering off. The same goes with SSTATE_DIR. However, let's | ||
50 | # get the basics in here and work on the cornercases later. | ||
51 | # A note. /proc/diskstats does not contain info on encryptfs, tmpfs, etc. | ||
52 | # If we end up hitting one of these fs, we'll just skip diskstats collection. | ||
53 | ############################################################################ | ||
54 | device=os.stat(tmpdir) | ||
55 | majordev=os.major(device.st_dev) | ||
56 | minordev=os.minor(device.st_dev) | ||
57 | ############################################################################ | ||
58 | # Bug 1700: | ||
59 | # Because tmpfs/encryptfs/ramfs etc inserts no entry in /proc/diskstats | ||
60 | # we set rdev to NoLogicalDevice and search for it later. If we find NLD | ||
61 | # we do not collect diskstats as the method to collect meaningful statistics | ||
62 | # for these fs types requires a bit more research. | ||
63 | ############################################################################ | ||
64 | rdev="NoLogicalDevice" | ||
65 | try: | ||
66 | with open("/proc/diskstats", "r") as f: | ||
67 | for line in f: | ||
68 | if majordev == int(line.split()[0]) and minordev == int(line.split()[1]): | ||
69 | rdev=line.split()[2] | ||
70 | except: | ||
71 | pass | ||
72 | file = open(e.data.getVar('DEVFILE', True), "w") | ||
73 | file.write(rdev) | ||
74 | file.close() | ||
75 | |||
76 | def get_device(e): | ||
77 | file = open(e.data.getVar('DEVFILE', True)) | ||
78 | device = file.readline() | ||
79 | file.close() | ||
80 | return device | ||
81 | |||
82 | def get_diskstats(dev): | ||
83 | import itertools | ||
84 | ############################################################################ | ||
85 | # For info on what these are, see kernel doc file iostats.txt | ||
86 | ############################################################################ | ||
87 | DSTAT_KEYS = ['ReadsComp', 'ReadsMerged', 'SectRead', 'TimeReads', 'WritesComp', 'SectWrite', 'TimeWrite', 'IOinProgress', 'TimeIO', 'WTimeIO'] | ||
88 | try: | ||
89 | with open("/proc/diskstats", "r") as f: | ||
90 | for x in f: | ||
91 | if dev in x: | ||
92 | diskstats_val = x.rstrip().split()[4:] | ||
93 | except IOError as e: | ||
94 | return | ||
95 | diskstats = dict(itertools.izip(DSTAT_KEYS, diskstats_val)) | ||
96 | return diskstats | ||
97 | |||
98 | def set_diskdata(var, dev, data): | ||
99 | data.setVar(var, get_diskstats(dev)) | ||
100 | |||
101 | def get_diskdata(var, dev, data): | ||
102 | olddiskdata = data.getVar(var, False) | ||
103 | diskdata = {} | ||
104 | if olddiskdata is None: | ||
105 | return | ||
106 | newdiskdata = get_diskstats(dev) | ||
107 | for key in olddiskdata.iterkeys(): | ||
108 | diskdata["Start"+key] = str(int(olddiskdata[key])) | ||
109 | diskdata["End"+key] = str(int(newdiskdata[key])) | ||
110 | return diskdata | ||
111 | |||
112 | def set_timedata(var, data): | ||
113 | import time | ||
114 | time = time.time() | ||
115 | cputime = get_cputime() | ||
116 | proctime = get_process_cputime(os.getpid()) | ||
117 | data.setVar(var, (time, cputime, proctime)) | ||
118 | |||
119 | def get_timedata(var, data): | ||
120 | import time | ||
121 | timedata = data.getVar(var, False) | ||
122 | if timedata is None: | ||
123 | return | ||
124 | oldtime, oldcpu, oldproc = timedata | ||
125 | procdiff = get_process_cputime(os.getpid()) - oldproc | ||
126 | cpudiff = get_cputime() - oldcpu | ||
127 | timediff = time.time() - oldtime | ||
128 | if cpudiff > 0: | ||
129 | cpuperc = float(procdiff) * 100 / cpudiff | ||
130 | else: | ||
131 | cpuperc = None | ||
132 | return timediff, cpuperc | ||
133 | |||
134 | def write_task_data(status, logfile, dev, e): | ||
135 | bn = get_bn(e) | ||
136 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
137 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
138 | file = open(os.path.join(logfile), "a") | ||
139 | timedata = get_timedata("__timedata_task", e.data) | ||
140 | if timedata: | ||
141 | elapsedtime, cpu = timedata | ||
142 | file.write(bb.data.expand("${PF}: %s: Elapsed time: %0.2f seconds \n" % | ||
143 | (e.task, elapsedtime), e.data)) | ||
144 | if cpu: | ||
145 | file.write("CPU usage: %0.1f%% \n" % cpu) | ||
146 | ############################################################################ | ||
147 | # Here we gather up disk data. In an effort to avoid lying with stats | ||
148 | # I do a bare minimum of analysis of collected data. | ||
149 | # The simple fact is, doing disk io collection on a per process basis | ||
150 | # without effecting build time would be difficult. | ||
151 | # For the best information, running things with BB_TOTAL_THREADS = "1" | ||
152 | # would return accurate per task results. | ||
153 | ############################################################################ | ||
154 | if dev != "NoLogicalDevice": | ||
155 | diskdata = get_diskdata("__diskdata_task", dev, e.data) | ||
156 | if diskdata: | ||
157 | for key in sorted(diskdata.iterkeys()): | ||
158 | file.write(key + ": " + diskdata[key] + "\n") | ||
159 | if status is "passed": | ||
160 | file.write("Status: PASSED \n") | ||
161 | else: | ||
162 | file.write("Status: FAILED \n") | ||
163 | file.write("Ended: %0.2f \n" % time.time()) | ||
164 | file.close() | ||
165 | |||
166 | python run_buildstats () { | ||
167 | import bb.build | ||
168 | import bb.event | ||
169 | import bb.data | ||
170 | import time, subprocess, platform | ||
171 | |||
172 | if isinstance(e, bb.event.BuildStarted): | ||
173 | ######################################################################## | ||
174 | # at first pass make the buildstats heriarchy and then | ||
175 | # set the buildname | ||
176 | ######################################################################## | ||
177 | try: | ||
178 | bb.utils.mkdirhier(e.data.getVar('BUILDSTATS_BASE', True)) | ||
179 | except: | ||
180 | pass | ||
181 | set_bn(e) | ||
182 | bn = get_bn(e) | ||
183 | set_device(e) | ||
184 | device = get_device(e) | ||
185 | |||
186 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
187 | try: | ||
188 | bb.utils.mkdirhier(bsdir) | ||
189 | except: | ||
190 | pass | ||
191 | if device != "NoLogicalDevice": | ||
192 | set_diskdata("__diskdata_build", device, e.data) | ||
193 | set_timedata("__timedata_build", e.data) | ||
194 | build_time = os.path.join(bsdir, "build_stats") | ||
195 | # write start of build into build_time | ||
196 | file = open(build_time,"a") | ||
197 | host_info = platform.uname() | ||
198 | file.write("Host Info: ") | ||
199 | for x in host_info: | ||
200 | if x: | ||
201 | file.write(x + " ") | ||
202 | file.write("\n") | ||
203 | file.write("Build Started: %0.2f \n" % time.time()) | ||
204 | file.close() | ||
205 | |||
206 | elif isinstance(e, bb.event.BuildCompleted): | ||
207 | bn = get_bn(e) | ||
208 | device = get_device(e) | ||
209 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
210 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
211 | build_time = os.path.join(bsdir, "build_stats") | ||
212 | file = open(build_time, "a") | ||
213 | ######################################################################## | ||
214 | # Write build statistics for the build | ||
215 | ######################################################################## | ||
216 | timedata = get_timedata("__timedata_build", e.data) | ||
217 | if timedata: | ||
218 | time, cpu = timedata | ||
219 | # write end of build and cpu used into build_time | ||
220 | file.write("Elapsed time: %0.2f seconds \n" % (time)) | ||
221 | if cpu: | ||
222 | file.write("CPU usage: %0.1f%% \n" % cpu) | ||
223 | if device != "NoLogicalDevice": | ||
224 | diskio = get_diskdata("__diskdata_build", device, e.data) | ||
225 | if diskio: | ||
226 | for key in sorted(diskio.iterkeys()): | ||
227 | file.write(key + ": " + diskio[key] + "\n") | ||
228 | file.close() | ||
229 | |||
230 | if isinstance(e, bb.build.TaskStarted): | ||
231 | bn = get_bn(e) | ||
232 | device = get_device(e) | ||
233 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
234 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
235 | if device != "NoLogicalDevice": | ||
236 | set_diskdata("__diskdata_task", device, e.data) | ||
237 | set_timedata("__timedata_task", e.data) | ||
238 | try: | ||
239 | bb.utils.mkdirhier(taskdir) | ||
240 | except: | ||
241 | pass | ||
242 | # write into the task event file the name and start time | ||
243 | file = open(os.path.join(taskdir, e.task), "a") | ||
244 | file.write("Event: %s \n" % bb.event.getName(e)) | ||
245 | file.write("Started: %0.2f \n" % time.time()) | ||
246 | file.close() | ||
247 | |||
248 | elif isinstance(e, bb.build.TaskSucceeded): | ||
249 | bn = get_bn(e) | ||
250 | device = get_device(e) | ||
251 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
252 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
253 | write_task_data("passed", os.path.join(taskdir, e.task), device, e) | ||
254 | if e.task == "do_rootfs": | ||
255 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
256 | bs=os.path.join(bsdir, "build_stats") | ||
257 | file = open(bs,"a") | ||
258 | rootfs = e.data.getVar('IMAGE_ROOTFS', True) | ||
259 | rootfs_size = subprocess.Popen(["du", "-sh", rootfs], stdout=subprocess.PIPE).stdout.read() | ||
260 | file.write("Uncompressed Rootfs size: %s" % rootfs_size) | ||
261 | file.close() | ||
262 | |||
263 | elif isinstance(e, bb.build.TaskFailed): | ||
264 | bn = get_bn(e) | ||
265 | device = get_device(e) | ||
266 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | ||
267 | taskdir = os.path.join(bsdir, e.data.expand("${PF}")) | ||
268 | write_task_data("failed", os.path.join(taskdir, e.task), device, e) | ||
269 | ######################################################################## | ||
270 | # Lets make things easier and tell people where the build failed in | ||
271 | # build_status. We do this here because BuildCompleted triggers no | ||
272 | # matter what the status of the build actually is | ||
273 | ######################################################################## | ||
274 | build_status = os.path.join(bsdir, "build_stats") | ||
275 | file = open(build_status,"a") | ||
276 | file.write(e.data.expand("Failed at: ${PF} at task: %s \n" % e.task)) | ||
277 | file.close() | ||
278 | } | ||
279 | |||
280 | addhandler run_buildstats | ||
281 | run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed" | ||
282 | |||