summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBeth Flanagan <elizabeth.flanagan@intel.com>2011-08-03 12:31:08 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2011-08-05 17:15:59 +0100
commitb525fc13063a6d681a14c871580efaab458dd370 (patch)
treea1d3f303afd07053032f65ca2e2d62a9e6df6ebb
parent17e8d8558dbabe2540860ecf1232214dfdf90ff0 (diff)
downloadpoky-b525fc13063a6d681a14c871580efaab458dd370.tar.gz
buildstats.bbclass: Adding disk io measurement
This commit adds disk-io statistics functionality on a build and per-task basis. It pulls measurements for whatever partition TMPDIR exists on. This data could be off if SSTATE_DIR and DL_DIR exist on a different partition/volume. Notes on what this pulls: ReadsComp: Total number of reads complete ReadsMerged: Total number of adjacent reads merged SectRead: Total number of sectors read TimeReads: Total number of m/s spent reading WritesComp: Total number of writes completed SectWrite: Total number of sectors written TimeWrite: Total number of m/s spent writing IOinProgress: Total amount of IO in progress at the time of we look at /proc/diskstats. TimeIO: Total number of m/s spent doing IO WTimeIO: Weighted time doing I/O. From iostats.txt: "This field is incremented at each I/O start, I/O completion, I/O merge, or read of these stats by the number of I/Os in progress (field 9) times the number of milliseconds spent doing I/O since the last update of this field. This can provide an easy measure of both I/O completion time and the backlog that may be accumulating." (From OE-Core rev: eeebcebf0d695358e72f3aed753f66cddd5e0e61) Signed-off-by: Beth Flanagan <elizabeth.flanagan@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/buildstats.bbclass220
1 files changed, 142 insertions, 78 deletions
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass
index 8077035367..939e3af603 100644
--- a/meta/classes/buildstats.bbclass
+++ b/meta/classes/buildstats.bbclass
@@ -1,5 +1,6 @@
1BUILDSTATS_BASE = ${TMPDIR}/buildstats/ 1BUILDSTATS_BASE = ${TMPDIR}/buildstats/
2BNFILE = ${BUILDSTATS_BASE}/.buildname 2BNFILE = ${BUILDSTATS_BASE}/.buildname
3DEVFILE = ${BUILDSTATS_BASE}/.device
3 4
4################################################################################ 5################################################################################
5# Build statistics gathering. 6# Build statistics gathering.
@@ -19,9 +20,79 @@ def get_cputime():
19 fields = open("/proc/stat", "r").readline().rstrip().split()[1:] 20 fields = open("/proc/stat", "r").readline().rstrip().split()[1:]
20 return sum(int(field) for field in fields) 21 return sum(int(field) for field in fields)
21 22
23def set_bn(e):
24 bn = e.getPkgs()[0] + "-" + bb.data.getVar('MACHINE',e.data, True)
25 try:
26 os.remove(bb.data.getVar('BNFILE', e.data, True))
27 except:
28 pass
29 file = open(bb.data.getVar('BNFILE', e.data, True), "w")
30 file.write(os.path.join(bn, bb.data.getVar('BUILDNAME', e.data, True)))
31 file.close()
32
33def get_bn(e):
34 file = open(bb.data.getVar('BNFILE', e.data, True))
35 bn = file.readline()
36 file.close()
37 return bn
38
39def set_device(e):
40 tmpdir = bb.data.getVar('TMPDIR', e.data, True)
41 try:
42 os.remove(bb.data.getVar('DEVFILE', e.data, True))
43 except:
44 pass
45 ############################################################################
46 # We look for the volume TMPDIR lives on. To do all disks would make little
47 # sense and not give us any particularly useful data. In theory we could do
48 # something like stick DL_DIR on a different partition and this would
49 # throw stats gathering off. The same goes with SSTATE_DIR. However, let's
50 # get the basics in here and work on the cornercases later.
51 ############################################################################
52 device=os.stat(tmpdir)
53 majordev=os.major(device.st_dev)
54 minordev=os.minor(device.st_dev)
55 for line in open("/proc/diskstats", "r"):
56 if majordev == int(line.split()[0]) and minordev == int(line.split()[1]):
57 rdev=line.split()[2]
58 file = open(bb.data.getVar('DEVFILE', e.data, True), "w")
59 file.write(rdev)
60 file.close()
61
62def get_device(e):
63 file = open(bb.data.getVar('DEVFILE', e.data, True))
64 device = file.readline()
65 file.close()
66 return device
67
68def get_diskstats(dev):
69 import itertools
70 ############################################################################
71 # For info on what these are, see kernel doc file iostats.txt
72 ############################################################################
73 DSTAT_KEYS = ['ReadsComp', 'ReadsMerged', 'SectRead', 'TimeReads', 'WritesComp', 'SectWrite', 'TimeWrite', 'IOinProgress', 'TimeIO', 'WTimeIO']
74 for x in open("/proc/diskstats", "r"):
75 if dev in x:
76 diskstats_val = x.rstrip().split()[4:]
77 diskstats = dict(itertools.izip(DSTAT_KEYS, diskstats_val))
78 return diskstats
79
80def set_diskdata(var, dev, data):
81 data.setVar(var, get_diskstats(dev))
82
83def get_diskdata(var, dev, data):
84 olddiskdata = data.getVar(var, False)
85 diskdata = {}
86 if olddiskdata is None:
87 return
88 newdiskdata = get_diskstats(dev)
89 for key in olddiskdata.iterkeys():
90 diskdata["Start"+key] = str(int(olddiskdata[key]))
91 diskdata["End"+key] = str(int(newdiskdata[key]))
92 return diskdata
93
22def set_timedata(var, data): 94def set_timedata(var, data):
23 import time 95 import time
24
25 time = time.time() 96 time = time.time()
26 cputime = get_cputime() 97 cputime = get_cputime()
27 proctime = get_process_cputime(os.getpid()) 98 proctime = get_process_cputime(os.getpid())
@@ -42,26 +113,35 @@ def get_timedata(var, data):
42 cpuperc = None 113 cpuperc = None
43 return timediff, cpuperc 114 return timediff, cpuperc
44 115
45############################################## 116def write_task_data(status, logfile, dev, e):
46# We need to set the buildname to a file since 117 bn = get_bn(e)
47# BUILDNAME changes throughout a build 118 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn)
48############################################## 119 taskdir = os.path.join(bsdir, bb.data.expand("${PF}", e.data))
49 120 file = open(os.path.join(logfile), "a")
50def set_bn(e): 121 timedata = get_timedata("__timedata_task", e.data)
51 bn = e.getPkgs()[0] + "-" + bb.data.getVar('MACHINE',e.data, True) 122 if timedata:
52 try: 123 elapsedtime, cpu = timedata
53 os.remove(bb.data.getVar('BNFILE',e.data, True)) 124 file.write(bb.data.expand("${PF}: %s: Elapsed time: %0.2f seconds \n" %
54 except: 125 (e.task, elapsedtime), e.data))
55 pass 126 if cpu:
56 file = open(bb.data.getVar('BNFILE',e.data, True), "w") 127 file.write("CPU usage: %0.1f%% \n" % cpu)
57 file.write(os.path.join(bn, bb.data.getVar('BUILDNAME', e.data, True))) 128 ############################################################################
58 file.close() 129 # Here we gather up disk data. In an effort to avoid lying with stats
59 130 # I do a bare minimum of analysis of collected data.
60def get_bn(e): 131 # The simple fact is, doing disk io collection on a per process basis
61 file = open(bb.data.getVar('BNFILE',e.data, True)) 132 # without effecting build time would be difficult.
62 bn = file.readline() 133 # For the best information, running things with BB_TOTAL_THREADS = "1"
134 # would return accurate per task results.
135 ############################################################################
136 diskdata = get_diskdata("__diskdata_task", dev, e.data)
137 if diskdata:
138 for key in sorted(diskdata.iterkeys()):
139 file.write(key + ": " + diskdata[key] + "\n")
140 if status is "passed":
141 file.write("Status: PASSED")
142 else:
143 file.write("Status: FAILED")
63 file.close() 144 file.close()
64 return bn
65 145
66python run_buildstats () { 146python run_buildstats () {
67 import bb.build 147 import bb.build
@@ -69,22 +149,27 @@ python run_buildstats () {
69 import bb.data 149 import bb.data
70 import time, subprocess, platform 150 import time, subprocess, platform
71 151
152
72 if isinstance(e, bb.event.BuildStarted): 153 if isinstance(e, bb.event.BuildStarted):
73 ############################################## 154 ########################################################################
74 # at first pass make the buildstats heriarchy and then 155 # at first pass make the buildstats heriarchy and then
75 # set the buildname 156 # set the buildname
76 ############################################## 157 ########################################################################
77 try: 158 try:
78 bb.mkdirhier(bb.data.getVar('BUILDSTATS_BASE', e.data, True)) 159 bb.mkdirhier(bb.data.getVar('BUILDSTATS_BASE', e.data, True))
79 except: 160 except:
80 pass 161 pass
81 set_bn(e) 162 set_bn(e)
82 bn = get_bn(e) 163 bn = get_bn(e)
164 set_device(e)
165 device = get_device(e)
166
83 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn) 167 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn)
84 try: 168 try:
85 bb.mkdirhier(bsdir) 169 bb.mkdirhier(bsdir)
86 except: 170 except:
87 pass 171 pass
172 set_diskdata("__diskdata_build", device, e.data)
88 set_timedata("__timedata_build", e.data) 173 set_timedata("__timedata_build", e.data)
89 build_time = os.path.join(bsdir, "build_stats") 174 build_time = os.path.join(bsdir, "build_stats")
90 # write start of build into build_time 175 # write start of build into build_time
@@ -99,27 +184,36 @@ python run_buildstats () {
99 file.close() 184 file.close()
100 185
101 elif isinstance(e, bb.event.BuildCompleted): 186 elif isinstance(e, bb.event.BuildCompleted):
102 bn=get_bn(e) 187 bn = get_bn(e)
103 timedata = get_timedata("__timedata_build", e.data) 188 dev = get_device(e)
104 if not timedata:
105 return
106 time, cpu = timedata
107 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn) 189 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn)
190 taskdir = os.path.join(bsdir, bb.data.expand("${PF}", e.data))
108 build_time = os.path.join(bsdir, "build_stats") 191 build_time = os.path.join(bsdir, "build_stats")
109 # write end of build and cpu used into build_time
110 file = open(build_time, "a") 192 file = open(build_time, "a")
111 file.write("Elapsed time: %0.2f seconds \n" % (time)) 193 ########################################################################
112 if cpu: 194 # Write build statistics for the build
113 file.write("CPU usage: %0.1f%% \n" % cpu) 195 ########################################################################
196 timedata = get_timedata("__timedata_build", e.data)
197 if timedata:
198 time, cpu = timedata
199 # write end of build and cpu used into build_time
200 file = open(build_time, "a")
201 file.write("Elapsed time: %0.2f seconds \n" % (time))
202 if cpu:
203 file.write("CPU usage: %0.1f%% \n" % cpu)
204 diskio = get_diskdata("__diskdata_build", dev, e.data)
205 if diskio:
206 for key in sorted(diskio.iterkeys()):
207 file.write(key + ": " + diskio[key] + "\n")
114 file.close() 208 file.close()
115 209
116
117 if isinstance(e, bb.build.TaskStarted): 210 if isinstance(e, bb.build.TaskStarted):
118 bn=get_bn(e) 211 bn = get_bn(e)
119 set_timedata("__timedata_task", e.data) 212 device = get_device(e)
120
121 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn) 213 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn)
122 taskdir = os.path.join(bsdir, bb.data.expand("${PF}", e.data)) 214 taskdir = os.path.join(bsdir, bb.data.expand("${PF}", e.data))
215 set_diskdata("__diskdata_task", device, e.data)
216 set_timedata("__timedata_task", e.data)
123 try: 217 try:
124 bb.mkdirhier(taskdir) 218 bb.mkdirhier(taskdir)
125 except: 219 except:
@@ -131,29 +225,13 @@ python run_buildstats () {
131 file.close() 225 file.close()
132 226
133 elif isinstance(e, bb.build.TaskSucceeded): 227 elif isinstance(e, bb.build.TaskSucceeded):
134 bn=get_bn(e) 228 bn = get_bn(e)
135 timedata = get_timedata("__timedata_task", e.data) 229 device = get_device(e)
136 if not timedata:
137 return
138 elapsedtime, cpu = timedata
139 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn) 230 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn)
140 taskdir = os.path.join(bsdir, bb.data.expand("${PF}", e.data)) 231 taskdir = os.path.join(bsdir, bb.data.expand("${PF}", e.data))
141 file = open(os.path.join(taskdir, e.task), "a") 232 write_task_data("passed", os.path.join(taskdir, e.task), device, e)
142 file.write(bb.data.expand("${PF}: %s: Elapsed time: %0.2f seconds \n" %
143 (e.task, elapsedtime), e.data))
144 file.write("Ended: %0.2f \n" % time.time())
145 if cpu:
146 file.write("CPU usage: %0.1f%% \n" % cpu)
147
148 file.write("Status: PASSED")
149 file.close()
150
151 ##############################################
152 # Alot of metric gathering occurs here.
153 # Reminder: I stripped out some in process stuff here
154 ##############################################
155
156 if e.task == "do_rootfs": 233 if e.task == "do_rootfs":
234 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn)
157 bs=os.path.join(bsdir, "build_stats") 235 bs=os.path.join(bsdir, "build_stats")
158 file = open(bs,"a") 236 file = open(bs,"a")
159 rootfs = bb.data.getVar('IMAGE_ROOTFS', e.data, True) 237 rootfs = bb.data.getVar('IMAGE_ROOTFS', e.data, True)
@@ -162,35 +240,21 @@ python run_buildstats () {
162 file.close() 240 file.close()
163 241
164 elif isinstance(e, bb.build.TaskFailed): 242 elif isinstance(e, bb.build.TaskFailed):
165 bn=get_bn(e) 243 bn = get_bn(e)
166 timedata = get_timedata("__timedata_task", e.data) 244 device = get_device(e)
167 if not timedata:
168 return
169 time, cpu = timedata
170 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn) 245 bsdir = os.path.join(bb.data.getVar('BUILDSTATS_BASE', e.data, True), bn)
171 taskdir = os.path.join(bsdir, bb.data.expand("${PF}", e.data)) 246 taskdir = os.path.join(bsdir, bb.data.expand("${PF}", e.data))
172 ############################################## 247 write_task_data("failed", os.path.join(taskdir, e.task), device, e)
173 # If the task fails dump the regular data. 248 ########################################################################
174 # fgrep -R "FAILED" <bsdir> 249 # Lets make things easier and tell people where the build failed in
175 # will grep all the events that failed. 250 # build_status. We do this here because BuildCompleted triggers no
176 ############################################## 251 # matter what the status of the build actually is
177 file = open(os.path.join(taskdir, e.task), "a") 252 ########################################################################
178 file.write(bb.data.expand("${PF}: %s: Elapsed time: %0.2f seconds \n" %
179 (e.task, time), e.data))
180 if cpu:
181 file.write("CPU usage: %0.1f%% \n" % cpu)
182 file.write("Status: FAILED")
183 file.close()
184 ##############################################
185 # Lets make things easier and tell people where the build failed in build_status
186 # We do this here because BuildCompleted triggers no matter what the status of the
187 # build actually is
188 ##############################################
189 build_status = os.path.join(bsdir, "build_stats") 253 build_status = os.path.join(bsdir, "build_stats")
190 file = open(build_status,"a") 254 file = open(build_status,"a")
191 file.write(bb.data.expand("Failed at: ${PF} at task: %s \n" % e.task, e.data)) 255 file.write(bb.data.expand("Failed at: ${PF} at task: %s \n" % e.task, e.data))
192 file.close() 256 file.close()
193 257
194} 258}
195 259
196addhandler run_buildstats 260addhandler run_buildstats