summaryrefslogtreecommitdiffstats
path: root/meta/lib/buildstats.py
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/buildstats.py')
-rw-r--r--meta/lib/buildstats.py57
1 files changed, 47 insertions, 10 deletions
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py
index c52b6c3b72..64ad3ef40e 100644
--- a/meta/lib/buildstats.py
+++ b/meta/lib/buildstats.py
@@ -14,13 +14,27 @@ class SystemStats:
14 bn = d.getVar('BUILDNAME') 14 bn = d.getVar('BUILDNAME')
15 bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) 15 bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn)
16 bb.utils.mkdirhier(bsdir) 16 bb.utils.mkdirhier(bsdir)
17 file_handlers = [('diskstats', self._reduce_diskstats),
18 ('meminfo', self._reduce_meminfo),
19 ('stat', self._reduce_stat)]
20
21 # Some hosts like openSUSE have readable /proc/pressure files
22 # but throw errors when these files are opened. Catch these error
23 # and ensure that the reduce_proc_pressure directory is not created.
24 if os.path.exists("/proc/pressure"):
25 try:
26 source = open('/proc/pressure/cpu', 'rb')
27 source.read()
28 pressuredir = os.path.join(bsdir, 'reduced_proc_pressure')
29 bb.utils.mkdirhier(pressuredir)
30 file_handlers.extend([('pressure/cpu', self._reduce_pressure),
31 ('pressure/io', self._reduce_pressure),
32 ('pressure/memory', self._reduce_pressure)])
33 except Exception:
34 pass
17 35
18 self.proc_files = [] 36 self.proc_files = []
19 for filename, handler in ( 37 for filename, handler in (file_handlers):
20 ('diskstats', self._reduce_diskstats),
21 ('meminfo', self._reduce_meminfo),
22 ('stat', self._reduce_stat),
23 ):
24 # The corresponding /proc files might not exist on the host. 38 # The corresponding /proc files might not exist on the host.
25 # For example, /proc/diskstats is not available in virtualized 39 # For example, /proc/diskstats is not available in virtualized
26 # environments like Linux-VServer. Silently skip collecting 40 # environments like Linux-VServer. Silently skip collecting
@@ -48,13 +62,15 @@ class SystemStats:
48 self.diskstats_ltime = None 62 self.diskstats_ltime = None
49 self.diskstats_data = None 63 self.diskstats_data = None
50 self.stat_ltimes = None 64 self.stat_ltimes = None
65 # Last time we sampled /proc/pressure. All resources stored in a single dict with the key as filename
66 self.last_pressure = {"pressure/cpu": None, "pressure/io": None, "pressure/memory": None}
51 67
52 def close(self): 68 def close(self):
53 self.monitor_disk.close() 69 self.monitor_disk.close()
54 for _, output, _ in self.proc_files: 70 for _, output, _ in self.proc_files:
55 output.close() 71 output.close()
56 72
57 def _reduce_meminfo(self, time, data): 73 def _reduce_meminfo(self, time, data, filename):
58 """ 74 """
59 Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree' 75 Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree'
60 and writes their values into a single line, in that order. 76 and writes their values into a single line, in that order.
@@ -75,7 +91,7 @@ class SystemStats:
75 disk = linetokens[2] 91 disk = linetokens[2]
76 return self.diskstats_regex.match(disk) 92 return self.diskstats_regex.match(disk)
77 93
78 def _reduce_diskstats(self, time, data): 94 def _reduce_diskstats(self, time, data, filename):
79 relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda x: x.split(), data.split(b'\n'))) 95 relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda x: x.split(), data.split(b'\n')))
80 diskdata = [0] * 3 96 diskdata = [0] * 3
81 reduced = None 97 reduced = None
@@ -104,10 +120,10 @@ class SystemStats:
104 return reduced 120 return reduced
105 121
106 122
107 def _reduce_nop(self, time, data): 123 def _reduce_nop(self, time, data, filename):
108 return (time, data) 124 return (time, data)
109 125
110 def _reduce_stat(self, time, data): 126 def _reduce_stat(self, time, data, filename):
111 if not data: 127 if not data:
112 return None 128 return None
113 # CPU times {user, nice, system, idle, io_wait, irq, softirq} from first line 129 # CPU times {user, nice, system, idle, io_wait, irq, softirq} from first line
@@ -126,6 +142,27 @@ class SystemStats:
126 self.stat_ltimes = times 142 self.stat_ltimes = times
127 return reduced 143 return reduced
128 144
145 def _reduce_pressure(self, time, data, filename):
146 """
147 Return reduced pressure: {avg10, avg60, avg300} and delta total compared to the previous sample
148 for the cpu, io and memory resources. A common function is used for all 3 resources since the
149 format of the /proc/pressure file is the same in each case.
150 """
151 if not data:
152 return None
153 tokens = data.split(b'\n', 1)[0].split()
154 avg10 = float(tokens[1].split(b'=')[1])
155 avg60 = float(tokens[2].split(b'=')[1])
156 avg300 = float(tokens[3].split(b'=')[1])
157 total = int(tokens[4].split(b'=')[1])
158
159 reduced = None
160 if self.last_pressure[filename]:
161 delta = total - self.last_pressure[filename]
162 reduced = (time, (avg10, avg60, avg300, delta))
163 self.last_pressure[filename] = total
164 return reduced
165
129 def sample(self, event, force): 166 def sample(self, event, force):
130 now = time.time() 167 now = time.time()
131 if (now - self.last_proc > self.min_seconds) or force: 168 if (now - self.last_proc > self.min_seconds) or force:
@@ -133,7 +170,7 @@ class SystemStats:
133 with open(os.path.join('/proc', filename), 'rb') as input: 170 with open(os.path.join('/proc', filename), 'rb') as input:
134 data = input.read() 171 data = input.read()
135 if handler: 172 if handler:
136 reduced = handler(now, data) 173 reduced = handler(now, data, filename)
137 else: 174 else:
138 reduced = (now, data) 175 reduced = (now, data)
139 if reduced: 176 if reduced: