summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2015-12-17 14:53:55 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-12-18 12:18:18 +0000
commit78371629b84dcd5c9fae39a8b454c9c9f071b6ff (patch)
treee0179c8dc47f567a35b9cb4aad64847ce143cc86
parent030c033d2c3128241fc61713512cba8619d40ea6 (diff)
downloadpoky-78371629b84dcd5c9fae39a8b454c9c9f071b6ff.tar.gz
buildstats: Drop disk data from buildstats
The existing diskstats data from buildstats simply isn't useful. It gives stats on the total IO counts on some random disk within the system. This means that the count includes data from all other tasks running at the same time and from any other process running on the system. I've been unable to find any use for the data so as a start at reworking the class, remove the related code. (From OE-Core rev: f9fb02909f332365cad329352956a29cff6eba77) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/buildstats.bbclass107
1 files changed, 3 insertions, 104 deletions
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass
index 22ec571b88..04307d8812 100644
--- a/meta/classes/buildstats.bbclass
+++ b/meta/classes/buildstats.bbclass
@@ -1,6 +1,5 @@
1BUILDSTATS_BASE = "${TMPDIR}/buildstats/" 1BUILDSTATS_BASE = "${TMPDIR}/buildstats/"
2BUILDSTATS_BNFILE = "${BUILDSTATS_BASE}/.buildname" 2BUILDSTATS_BNFILE = "${BUILDSTATS_BASE}/.buildname"
3BUILDSTATS_DEVFILE = "${BUILDSTATS_BASE}/.device"
4 3
5################################################################################ 4################################################################################
6# Build statistics gathering. 5# Build statistics gathering.
@@ -36,78 +35,6 @@ def get_bn(e):
36 bn = f.readline() 35 bn = f.readline()
37 return bn 36 return bn
38 37
39def set_device(e):
40 tmpdir = e.data.getVar('TMPDIR', True)
41 devfile = e.data.getVar('BUILDSTATS_DEVFILE', True)
42 try:
43 os.remove(devfile)
44 except:
45 pass
46 ############################################################################
47 # We look for the volume TMPDIR lives on. To do all disks would make little
48 # sense and not give us any particularly useful data. In theory we could do
49 # something like stick DL_DIR on a different partition and this would
50 # throw stats gathering off. The same goes with SSTATE_DIR. However, let's
51 # get the basics in here and work on the cornercases later.
52 # A note. /proc/diskstats does not contain info on encryptfs, tmpfs, etc.
53 # If we end up hitting one of these fs, we'll just skip diskstats collection.
54 ############################################################################
55 device = os.stat(tmpdir)
56 majordev = os.major(long(device.st_dev))
57 minordev = os.minor(long(device.st_dev))
58 ############################################################################
59 # Bug 1700:
60 # Because tmpfs/encryptfs/ramfs etc inserts no entry in /proc/diskstats
61 # we set rdev to NoLogicalDevice and search for it later. If we find NLD
62 # we do not collect diskstats as the method to collect meaningful statistics
63 # for these fs types requires a bit more research.
64 ############################################################################
65 rdev = "NoLogicalDevice"
66 try:
67 with open("/proc/diskstats", "r") as f:
68 for line in f:
69 if majordev == int(line.split()[0]) and minordev == int(line.split()[1]):
70 rdev = line.split()[2]
71 except:
72 pass
73 with open(devfile, "w") as f:
74 f.write(rdev)
75
76def get_device(e):
77 with open(e.data.getVar('BUILDSTATS_DEVFILE', True)) as f:
78 device = f.readline()
79 return device
80
81def get_diskstats(dev):
82 import itertools
83 ############################################################################
84 # For info on what these are, see kernel doc file iostats.txt
85 ############################################################################
86 DSTAT_KEYS = ['ReadsComp', 'ReadsMerged', 'SectRead', 'TimeReads', 'WritesComp', 'SectWrite', 'TimeWrite', 'IOinProgress', 'TimeIO', 'WTimeIO']
87 try:
88 with open("/proc/diskstats", "r") as f:
89 for x in f:
90 if dev in x:
91 diskstats_val = x.rstrip().split()[4:]
92 except IOError as e:
93 return
94 diskstats = dict(itertools.izip(DSTAT_KEYS, diskstats_val))
95 return diskstats
96
97def set_diskdata(var, dev, data):
98 data.setVar(var, get_diskstats(dev))
99
100def get_diskdata(var, dev, data):
101 olddiskdata = data.getVar(var, False)
102 diskdata = {}
103 if olddiskdata is None:
104 return
105 newdiskdata = get_diskstats(dev)
106 for key in olddiskdata.iterkeys():
107 diskdata["Start"+key] = str(int(olddiskdata[key]))
108 diskdata["End"+key] = str(int(newdiskdata[key]))
109 return diskdata
110
111def set_timedata(var, data, server_time=None): 38def set_timedata(var, data, server_time=None):
112 import time 39 import time
113 if server_time: 40 if server_time:
@@ -137,7 +64,7 @@ def get_timedata(var, data, server_time=None):
137 cpuperc = None 64 cpuperc = None
138 return timediff, cpuperc 65 return timediff, cpuperc
139 66
140def write_task_data(status, logfile, dev, e): 67def write_task_data(status, logfile, e):
141 bn = get_bn(e) 68 bn = get_bn(e)
142 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) 69 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
143 with open(os.path.join(logfile), "a") as f: 70 with open(os.path.join(logfile), "a") as f:
@@ -148,19 +75,6 @@ def write_task_data(status, logfile, dev, e):
148 (e.task, elapsedtime), e.data)) 75 (e.task, elapsedtime), e.data))
149 if cpu: 76 if cpu:
150 f.write("CPU usage: %0.1f%% \n" % cpu) 77 f.write("CPU usage: %0.1f%% \n" % cpu)
151 ############################################################################
152 # Here we gather up disk data. In an effort to avoid lying with stats
153 # I do a bare minimum of analysis of collected data.
154 # The simple fact is, doing disk io collection on a per process basis
155 # without effecting build time would be difficult.
156 # For the best information, running things with BB_TOTAL_THREADS = "1"
157 # would return accurate per task results.
158 ############################################################################
159 if dev != "NoLogicalDevice":
160 diskdata = get_diskdata("__diskdata_task", dev, e.data)
161 if diskdata:
162 for key in sorted(diskdata.iterkeys()):
163 f.write(key + ": " + diskdata[key] + "\n")
164 if status is "passed": 78 if status is "passed":
165 f.write("Status: PASSED \n") 79 f.write("Status: PASSED \n")
166 else: 80 else:
@@ -181,13 +95,9 @@ python run_buildstats () {
181 bb.utils.mkdirhier(e.data.getVar('BUILDSTATS_BASE', True)) 95 bb.utils.mkdirhier(e.data.getVar('BUILDSTATS_BASE', True))
182 set_bn(e) 96 set_bn(e)
183 bn = get_bn(e) 97 bn = get_bn(e)
184 set_device(e)
185 device = get_device(e)
186 98
187 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) 99 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
188 bb.utils.mkdirhier(bsdir) 100 bb.utils.mkdirhier(bsdir)
189 if device != "NoLogicalDevice":
190 set_diskdata("__diskdata_build", device, e.data)
191 set_timedata("__timedata_build", e.data) 101 set_timedata("__timedata_build", e.data)
192 build_time = os.path.join(bsdir, "build_stats") 102 build_time = os.path.join(bsdir, "build_stats")
193 # write start of build into build_time 103 # write start of build into build_time
@@ -202,7 +112,6 @@ python run_buildstats () {
202 112
203 elif isinstance(e, bb.event.BuildCompleted): 113 elif isinstance(e, bb.event.BuildCompleted):
204 bn = get_bn(e) 114 bn = get_bn(e)
205 device = get_device(e)
206 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) 115 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
207 build_time = os.path.join(bsdir, "build_stats") 116 build_time = os.path.join(bsdir, "build_stats")
208 with open(build_time, "a") as f: 117 with open(build_time, "a") as f:
@@ -216,19 +125,11 @@ python run_buildstats () {
216 f.write("Elapsed time: %0.2f seconds \n" % (time)) 125 f.write("Elapsed time: %0.2f seconds \n" % (time))
217 if cpu: 126 if cpu:
218 f.write("CPU usage: %0.1f%% \n" % cpu) 127 f.write("CPU usage: %0.1f%% \n" % cpu)
219 if device != "NoLogicalDevice":
220 diskio = get_diskdata("__diskdata_build", device, e.data)
221 if diskio:
222 for key in sorted(diskio.iterkeys()):
223 f.write(key + ": " + diskio[key] + "\n")
224 128
225 if isinstance(e, bb.build.TaskStarted): 129 if isinstance(e, bb.build.TaskStarted):
226 bn = get_bn(e) 130 bn = get_bn(e)
227 device = get_device(e)
228 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) 131 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
229 taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) 132 taskdir = os.path.join(bsdir, e.data.getVar('PF', True))
230 if device != "NoLogicalDevice":
231 set_diskdata("__diskdata_task", device, e.data)
232 set_timedata("__timedata_task", e.data, e.time) 133 set_timedata("__timedata_task", e.data, e.time)
233 bb.utils.mkdirhier(taskdir) 134 bb.utils.mkdirhier(taskdir)
234 # write into the task event file the name and start time 135 # write into the task event file the name and start time
@@ -238,10 +139,9 @@ python run_buildstats () {
238 139
239 elif isinstance(e, bb.build.TaskSucceeded): 140 elif isinstance(e, bb.build.TaskSucceeded):
240 bn = get_bn(e) 141 bn = get_bn(e)
241 device = get_device(e)
242 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) 142 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
243 taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) 143 taskdir = os.path.join(bsdir, e.data.getVar('PF', True))
244 write_task_data("passed", os.path.join(taskdir, e.task), device, e) 144 write_task_data("passed", os.path.join(taskdir, e.task), e)
245 if e.task == "do_rootfs": 145 if e.task == "do_rootfs":
246 bs = os.path.join(bsdir, "build_stats") 146 bs = os.path.join(bsdir, "build_stats")
247 with open(bs, "a") as f: 147 with open(bs, "a") as f:
@@ -251,10 +151,9 @@ python run_buildstats () {
251 151
252 elif isinstance(e, bb.build.TaskFailed): 152 elif isinstance(e, bb.build.TaskFailed):
253 bn = get_bn(e) 153 bn = get_bn(e)
254 device = get_device(e)
255 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) 154 bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn)
256 taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) 155 taskdir = os.path.join(bsdir, e.data.getVar('PF', True))
257 write_task_data("failed", os.path.join(taskdir, e.task), device, e) 156 write_task_data("failed", os.path.join(taskdir, e.task), e)
258 ######################################################################## 157 ########################################################################
259 # Lets make things easier and tell people where the build failed in 158 # Lets make things easier and tell people where the build failed in
260 # build_status. We do this here because BuildCompleted triggers no 159 # build_status. We do this here because BuildCompleted triggers no