summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPatrick Ohly <patrick.ohly@intel.com>2016-11-30 10:50:01 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-12-07 10:37:59 +0000
commit8f475b78c9dbcd861784b3f06511bedb6d43f2ca (patch)
tree0fc21b0f65fc06f0537560441ca02e58d7469962
parent5956492c201157793698323e67e0b813c5d8fe51 (diff)
downloadpoky-8f475b78c9dbcd861784b3f06511bedb6d43f2ca.tar.gz
buildstats: add system state sampling
/proc/[diskstats|meminfo|stat] get sampled and written to the same proc_<filename>.log files as during normal bootchat logging. This will allow rendering the CPU, disk and memory usage charts. Right now sampling happens once a second, triggered by the heartbeat event.That produces quite a bit of data for long builds, which will be addressed in a separate commit by storing the data in a more compact form. (From OE-Core rev: 6f4e8180b5b4857eaf6caf410fd3a4a41ed85930) Signed-off-by: Patrick Ohly <patrick.ohly@intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/buildstats.bbclass24
-rw-r--r--meta/lib/buildstats.py47
2 files changed, 71 insertions, 0 deletions
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass
index 57ecc8fee8..9c0c37dcdd 100644
--- a/meta/classes/buildstats.bbclass
+++ b/meta/classes/buildstats.bbclass
@@ -188,3 +188,27 @@ python run_buildstats () {
188addhandler run_buildstats 188addhandler run_buildstats
189run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed" 189run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed"
190 190
191python runqueue_stats () {
192 import buildstats
193 from bb import event, runqueue
194 # We should not record any samples before the first task has started,
195 # because that's the first activity shown in the process chart.
196 # Besides, at that point we are sure that the build variables
197 # are available that we need to find the output directory.
198 # The persistent SystemStats is stored in the datastore and
199 # closed when the build is done.
200 system_stats = d.getVar('_buildstats_system_stats', True)
201 if not system_stats and isinstance(e, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted)):
202 system_stats = buildstats.SystemStats(d)
203 d.setVar('_buildstats_system_stats', system_stats)
204 if system_stats:
205 # Ensure that we sample at important events.
206 done = isinstance(e, bb.event.BuildCompleted)
207 system_stats.sample(force=done)
208 if done:
209 system_stats.close()
210 d.delVar('_buildstats_system_stats')
211}
212
213addhandler runqueue_stats
214runqueue_stats[eventmask] = "bb.runqueue.sceneQueueTaskStarted bb.runqueue.runQueueTaskStarted bb.event.HeartbeatEvent bb.event.BuildCompleted"
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py
new file mode 100644
index 0000000000..8ce4112c2d
--- /dev/null
+++ b/meta/lib/buildstats.py
@@ -0,0 +1,47 @@
1# Implements system state sampling. Called by buildstats.bbclass.
2# Because it is a real Python module, it can hold persistent state,
3# like open log files and the time of the last sampling.
4
5import time
6
7class SystemStats:
8 def __init__(self, d):
9 bn = d.getVar('BUILDNAME', True)
10 bsdir = os.path.join(d.getVar('BUILDSTATS_BASE', True), bn)
11 bb.utils.mkdirhier(bsdir)
12
13 self.proc_files = []
14 for filename in ('diskstats', 'meminfo', 'stat'):
15 # In practice, this class gets instantiated only once in
16 # the bitbake cooker process. Therefore 'append' mode is
17 # not strictly necessary, but using it makes the class
18 # more robust should two processes ever write
19 # concurrently.
20 self.proc_files.append((filename,
21 open(os.path.join(bsdir, 'proc_%s.log' % filename), 'ab')))
22 # Last time that we sampled data.
23 self.last = 0
24 # Minimum number of seconds between recording a sample. This
25 # becames relevant when we get called very often while many
26 # short tasks get started. Sampling during quiet periods
27 # depends on the heartbeat event, which fires less often.
28 self.min_seconds = 1
29
30 def close(self):
31 self.monitor_disk.close()
32 for _, output, _ in self.proc_files:
33 output.close()
34
35 def sample(self, force):
36 now = time.time()
37 if (now - self.last > self.min_seconds) or force:
38 for filename, output in self.proc_files:
39 with open(os.path.join('/proc', filename), 'rb') as input:
40 data = input.read()
41 # Unbuffered raw write, less overhead and useful
42 # in case that we end up with concurrent writes.
43 os.write(output.fileno(),
44 ('%.0f\n' % now).encode('ascii') +
45 data +
46 b'\n')
47 self.last = now