diff options
Diffstat (limited to 'meta/lib/buildstats.py')
| -rw-r--r-- | meta/lib/buildstats.py | 57 |
1 files changed, 47 insertions, 10 deletions
diff --git a/meta/lib/buildstats.py b/meta/lib/buildstats.py index c52b6c3b72..64ad3ef40e 100644 --- a/meta/lib/buildstats.py +++ b/meta/lib/buildstats.py | |||
| @@ -14,13 +14,27 @@ class SystemStats: | |||
| 14 | bn = d.getVar('BUILDNAME') | 14 | bn = d.getVar('BUILDNAME') |
| 15 | bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) | 15 | bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) |
| 16 | bb.utils.mkdirhier(bsdir) | 16 | bb.utils.mkdirhier(bsdir) |
| 17 | file_handlers = [('diskstats', self._reduce_diskstats), | ||
| 18 | ('meminfo', self._reduce_meminfo), | ||
| 19 | ('stat', self._reduce_stat)] | ||
| 20 | |||
| 21 | # Some hosts like openSUSE have readable /proc/pressure files | ||
| 22 | # but throw errors when these files are opened. Catch these error | ||
| 23 | # and ensure that the reduce_proc_pressure directory is not created. | ||
| 24 | if os.path.exists("/proc/pressure"): | ||
| 25 | try: | ||
| 26 | source = open('/proc/pressure/cpu', 'rb') | ||
| 27 | source.read() | ||
| 28 | pressuredir = os.path.join(bsdir, 'reduced_proc_pressure') | ||
| 29 | bb.utils.mkdirhier(pressuredir) | ||
| 30 | file_handlers.extend([('pressure/cpu', self._reduce_pressure), | ||
| 31 | ('pressure/io', self._reduce_pressure), | ||
| 32 | ('pressure/memory', self._reduce_pressure)]) | ||
| 33 | except Exception: | ||
| 34 | pass | ||
| 17 | 35 | ||
| 18 | self.proc_files = [] | 36 | self.proc_files = [] |
| 19 | for filename, handler in ( | 37 | for filename, handler in (file_handlers): |
| 20 | ('diskstats', self._reduce_diskstats), | ||
| 21 | ('meminfo', self._reduce_meminfo), | ||
| 22 | ('stat', self._reduce_stat), | ||
| 23 | ): | ||
| 24 | # The corresponding /proc files might not exist on the host. | 38 | # The corresponding /proc files might not exist on the host. |
| 25 | # For example, /proc/diskstats is not available in virtualized | 39 | # For example, /proc/diskstats is not available in virtualized |
| 26 | # environments like Linux-VServer. Silently skip collecting | 40 | # environments like Linux-VServer. Silently skip collecting |
| @@ -48,13 +62,15 @@ class SystemStats: | |||
| 48 | self.diskstats_ltime = None | 62 | self.diskstats_ltime = None |
| 49 | self.diskstats_data = None | 63 | self.diskstats_data = None |
| 50 | self.stat_ltimes = None | 64 | self.stat_ltimes = None |
| 65 | # Last time we sampled /proc/pressure. All resources stored in a single dict with the key as filename | ||
| 66 | self.last_pressure = {"pressure/cpu": None, "pressure/io": None, "pressure/memory": None} | ||
| 51 | 67 | ||
| 52 | def close(self): | 68 | def close(self): |
| 53 | self.monitor_disk.close() | 69 | self.monitor_disk.close() |
| 54 | for _, output, _ in self.proc_files: | 70 | for _, output, _ in self.proc_files: |
| 55 | output.close() | 71 | output.close() |
| 56 | 72 | ||
| 57 | def _reduce_meminfo(self, time, data): | 73 | def _reduce_meminfo(self, time, data, filename): |
| 58 | """ | 74 | """ |
| 59 | Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree' | 75 | Extracts 'MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree' |
| 60 | and writes their values into a single line, in that order. | 76 | and writes their values into a single line, in that order. |
| @@ -75,7 +91,7 @@ class SystemStats: | |||
| 75 | disk = linetokens[2] | 91 | disk = linetokens[2] |
| 76 | return self.diskstats_regex.match(disk) | 92 | return self.diskstats_regex.match(disk) |
| 77 | 93 | ||
| 78 | def _reduce_diskstats(self, time, data): | 94 | def _reduce_diskstats(self, time, data, filename): |
| 79 | relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda x: x.split(), data.split(b'\n'))) | 95 | relevant_tokens = filter(self._diskstats_is_relevant_line, map(lambda x: x.split(), data.split(b'\n'))) |
| 80 | diskdata = [0] * 3 | 96 | diskdata = [0] * 3 |
| 81 | reduced = None | 97 | reduced = None |
| @@ -104,10 +120,10 @@ class SystemStats: | |||
| 104 | return reduced | 120 | return reduced |
| 105 | 121 | ||
| 106 | 122 | ||
| 107 | def _reduce_nop(self, time, data): | 123 | def _reduce_nop(self, time, data, filename): |
| 108 | return (time, data) | 124 | return (time, data) |
| 109 | 125 | ||
| 110 | def _reduce_stat(self, time, data): | 126 | def _reduce_stat(self, time, data, filename): |
| 111 | if not data: | 127 | if not data: |
| 112 | return None | 128 | return None |
| 113 | # CPU times {user, nice, system, idle, io_wait, irq, softirq} from first line | 129 | # CPU times {user, nice, system, idle, io_wait, irq, softirq} from first line |
| @@ -126,6 +142,27 @@ class SystemStats: | |||
| 126 | self.stat_ltimes = times | 142 | self.stat_ltimes = times |
| 127 | return reduced | 143 | return reduced |
| 128 | 144 | ||
| 145 | def _reduce_pressure(self, time, data, filename): | ||
| 146 | """ | ||
| 147 | Return reduced pressure: {avg10, avg60, avg300} and delta total compared to the previous sample | ||
| 148 | for the cpu, io and memory resources. A common function is used for all 3 resources since the | ||
| 149 | format of the /proc/pressure file is the same in each case. | ||
| 150 | """ | ||
| 151 | if not data: | ||
| 152 | return None | ||
| 153 | tokens = data.split(b'\n', 1)[0].split() | ||
| 154 | avg10 = float(tokens[1].split(b'=')[1]) | ||
| 155 | avg60 = float(tokens[2].split(b'=')[1]) | ||
| 156 | avg300 = float(tokens[3].split(b'=')[1]) | ||
| 157 | total = int(tokens[4].split(b'=')[1]) | ||
| 158 | |||
| 159 | reduced = None | ||
| 160 | if self.last_pressure[filename]: | ||
| 161 | delta = total - self.last_pressure[filename] | ||
| 162 | reduced = (time, (avg10, avg60, avg300, delta)) | ||
| 163 | self.last_pressure[filename] = total | ||
| 164 | return reduced | ||
| 165 | |||
| 129 | def sample(self, event, force): | 166 | def sample(self, event, force): |
| 130 | now = time.time() | 167 | now = time.time() |
| 131 | if (now - self.last_proc > self.min_seconds) or force: | 168 | if (now - self.last_proc > self.min_seconds) or force: |
| @@ -133,7 +170,7 @@ class SystemStats: | |||
| 133 | with open(os.path.join('/proc', filename), 'rb') as input: | 170 | with open(os.path.join('/proc', filename), 'rb') as input: |
| 134 | data = input.read() | 171 | data = input.read() |
| 135 | if handler: | 172 | if handler: |
| 136 | reduced = handler(now, data) | 173 | reduced = handler(now, data, filename) |
| 137 | else: | 174 | else: |
| 138 | reduced = (now, data) | 175 | reduced = (now, data) |
| 139 | if reduced: | 176 | if reduced: |
