diff options
| -rw-r--r-- | meta/classes/buildstats.bbclass | 107 |
1 files changed, 3 insertions, 104 deletions
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass index 22ec571b88..04307d8812 100644 --- a/meta/classes/buildstats.bbclass +++ b/meta/classes/buildstats.bbclass | |||
| @@ -1,6 +1,5 @@ | |||
| 1 | BUILDSTATS_BASE = "${TMPDIR}/buildstats/" | 1 | BUILDSTATS_BASE = "${TMPDIR}/buildstats/" |
| 2 | BUILDSTATS_BNFILE = "${BUILDSTATS_BASE}/.buildname" | 2 | BUILDSTATS_BNFILE = "${BUILDSTATS_BASE}/.buildname" |
| 3 | BUILDSTATS_DEVFILE = "${BUILDSTATS_BASE}/.device" | ||
| 4 | 3 | ||
| 5 | ################################################################################ | 4 | ################################################################################ |
| 6 | # Build statistics gathering. | 5 | # Build statistics gathering. |
| @@ -36,78 +35,6 @@ def get_bn(e): | |||
| 36 | bn = f.readline() | 35 | bn = f.readline() |
| 37 | return bn | 36 | return bn |
| 38 | 37 | ||
| 39 | def set_device(e): | ||
| 40 | tmpdir = e.data.getVar('TMPDIR', True) | ||
| 41 | devfile = e.data.getVar('BUILDSTATS_DEVFILE', True) | ||
| 42 | try: | ||
| 43 | os.remove(devfile) | ||
| 44 | except: | ||
| 45 | pass | ||
| 46 | ############################################################################ | ||
| 47 | # We look for the volume TMPDIR lives on. To do all disks would make little | ||
| 48 | # sense and not give us any particularly useful data. In theory we could do | ||
| 49 | # something like stick DL_DIR on a different partition and this would | ||
| 50 | # throw stats gathering off. The same goes with SSTATE_DIR. However, let's | ||
| 51 | # get the basics in here and work on the cornercases later. | ||
| 52 | # A note. /proc/diskstats does not contain info on encryptfs, tmpfs, etc. | ||
| 53 | # If we end up hitting one of these fs, we'll just skip diskstats collection. | ||
| 54 | ############################################################################ | ||
| 55 | device = os.stat(tmpdir) | ||
| 56 | majordev = os.major(long(device.st_dev)) | ||
| 57 | minordev = os.minor(long(device.st_dev)) | ||
| 58 | ############################################################################ | ||
| 59 | # Bug 1700: | ||
| 60 | # Because tmpfs/encryptfs/ramfs etc inserts no entry in /proc/diskstats | ||
| 61 | # we set rdev to NoLogicalDevice and search for it later. If we find NLD | ||
| 62 | # we do not collect diskstats as the method to collect meaningful statistics | ||
| 63 | # for these fs types requires a bit more research. | ||
| 64 | ############################################################################ | ||
| 65 | rdev = "NoLogicalDevice" | ||
| 66 | try: | ||
| 67 | with open("/proc/diskstats", "r") as f: | ||
| 68 | for line in f: | ||
| 69 | if majordev == int(line.split()[0]) and minordev == int(line.split()[1]): | ||
| 70 | rdev = line.split()[2] | ||
| 71 | except: | ||
| 72 | pass | ||
| 73 | with open(devfile, "w") as f: | ||
| 74 | f.write(rdev) | ||
| 75 | |||
| 76 | def get_device(e): | ||
| 77 | with open(e.data.getVar('BUILDSTATS_DEVFILE', True)) as f: | ||
| 78 | device = f.readline() | ||
| 79 | return device | ||
| 80 | |||
| 81 | def get_diskstats(dev): | ||
| 82 | import itertools | ||
| 83 | ############################################################################ | ||
| 84 | # For info on what these are, see kernel doc file iostats.txt | ||
| 85 | ############################################################################ | ||
| 86 | DSTAT_KEYS = ['ReadsComp', 'ReadsMerged', 'SectRead', 'TimeReads', 'WritesComp', 'SectWrite', 'TimeWrite', 'IOinProgress', 'TimeIO', 'WTimeIO'] | ||
| 87 | try: | ||
| 88 | with open("/proc/diskstats", "r") as f: | ||
| 89 | for x in f: | ||
| 90 | if dev in x: | ||
| 91 | diskstats_val = x.rstrip().split()[4:] | ||
| 92 | except IOError as e: | ||
| 93 | return | ||
| 94 | diskstats = dict(itertools.izip(DSTAT_KEYS, diskstats_val)) | ||
| 95 | return diskstats | ||
| 96 | |||
| 97 | def set_diskdata(var, dev, data): | ||
| 98 | data.setVar(var, get_diskstats(dev)) | ||
| 99 | |||
| 100 | def get_diskdata(var, dev, data): | ||
| 101 | olddiskdata = data.getVar(var, False) | ||
| 102 | diskdata = {} | ||
| 103 | if olddiskdata is None: | ||
| 104 | return | ||
| 105 | newdiskdata = get_diskstats(dev) | ||
| 106 | for key in olddiskdata.iterkeys(): | ||
| 107 | diskdata["Start"+key] = str(int(olddiskdata[key])) | ||
| 108 | diskdata["End"+key] = str(int(newdiskdata[key])) | ||
| 109 | return diskdata | ||
| 110 | |||
| 111 | def set_timedata(var, data, server_time=None): | 38 | def set_timedata(var, data, server_time=None): |
| 112 | import time | 39 | import time |
| 113 | if server_time: | 40 | if server_time: |
| @@ -137,7 +64,7 @@ def get_timedata(var, data, server_time=None): | |||
| 137 | cpuperc = None | 64 | cpuperc = None |
| 138 | return timediff, cpuperc | 65 | return timediff, cpuperc |
| 139 | 66 | ||
| 140 | def write_task_data(status, logfile, dev, e): | 67 | def write_task_data(status, logfile, e): |
| 141 | bn = get_bn(e) | 68 | bn = get_bn(e) |
| 142 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | 69 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) |
| 143 | with open(os.path.join(logfile), "a") as f: | 70 | with open(os.path.join(logfile), "a") as f: |
| @@ -148,19 +75,6 @@ def write_task_data(status, logfile, dev, e): | |||
| 148 | (e.task, elapsedtime), e.data)) | 75 | (e.task, elapsedtime), e.data)) |
| 149 | if cpu: | 76 | if cpu: |
| 150 | f.write("CPU usage: %0.1f%% \n" % cpu) | 77 | f.write("CPU usage: %0.1f%% \n" % cpu) |
| 151 | ############################################################################ | ||
| 152 | # Here we gather up disk data. In an effort to avoid lying with stats | ||
| 153 | # I do a bare minimum of analysis of collected data. | ||
| 154 | # The simple fact is, doing disk io collection on a per process basis | ||
| 155 | # without effecting build time would be difficult. | ||
| 156 | # For the best information, running things with BB_TOTAL_THREADS = "1" | ||
| 157 | # would return accurate per task results. | ||
| 158 | ############################################################################ | ||
| 159 | if dev != "NoLogicalDevice": | ||
| 160 | diskdata = get_diskdata("__diskdata_task", dev, e.data) | ||
| 161 | if diskdata: | ||
| 162 | for key in sorted(diskdata.iterkeys()): | ||
| 163 | f.write(key + ": " + diskdata[key] + "\n") | ||
| 164 | if status is "passed": | 78 | if status is "passed": |
| 165 | f.write("Status: PASSED \n") | 79 | f.write("Status: PASSED \n") |
| 166 | else: | 80 | else: |
| @@ -181,13 +95,9 @@ python run_buildstats () { | |||
| 181 | bb.utils.mkdirhier(e.data.getVar('BUILDSTATS_BASE', True)) | 95 | bb.utils.mkdirhier(e.data.getVar('BUILDSTATS_BASE', True)) |
| 182 | set_bn(e) | 96 | set_bn(e) |
| 183 | bn = get_bn(e) | 97 | bn = get_bn(e) |
| 184 | set_device(e) | ||
| 185 | device = get_device(e) | ||
| 186 | 98 | ||
| 187 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | 99 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) |
| 188 | bb.utils.mkdirhier(bsdir) | 100 | bb.utils.mkdirhier(bsdir) |
| 189 | if device != "NoLogicalDevice": | ||
| 190 | set_diskdata("__diskdata_build", device, e.data) | ||
| 191 | set_timedata("__timedata_build", e.data) | 101 | set_timedata("__timedata_build", e.data) |
| 192 | build_time = os.path.join(bsdir, "build_stats") | 102 | build_time = os.path.join(bsdir, "build_stats") |
| 193 | # write start of build into build_time | 103 | # write start of build into build_time |
| @@ -202,7 +112,6 @@ python run_buildstats () { | |||
| 202 | 112 | ||
| 203 | elif isinstance(e, bb.event.BuildCompleted): | 113 | elif isinstance(e, bb.event.BuildCompleted): |
| 204 | bn = get_bn(e) | 114 | bn = get_bn(e) |
| 205 | device = get_device(e) | ||
| 206 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | 115 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) |
| 207 | build_time = os.path.join(bsdir, "build_stats") | 116 | build_time = os.path.join(bsdir, "build_stats") |
| 208 | with open(build_time, "a") as f: | 117 | with open(build_time, "a") as f: |
| @@ -216,19 +125,11 @@ python run_buildstats () { | |||
| 216 | f.write("Elapsed time: %0.2f seconds \n" % (time)) | 125 | f.write("Elapsed time: %0.2f seconds \n" % (time)) |
| 217 | if cpu: | 126 | if cpu: |
| 218 | f.write("CPU usage: %0.1f%% \n" % cpu) | 127 | f.write("CPU usage: %0.1f%% \n" % cpu) |
| 219 | if device != "NoLogicalDevice": | ||
| 220 | diskio = get_diskdata("__diskdata_build", device, e.data) | ||
| 221 | if diskio: | ||
| 222 | for key in sorted(diskio.iterkeys()): | ||
| 223 | f.write(key + ": " + diskio[key] + "\n") | ||
| 224 | 128 | ||
| 225 | if isinstance(e, bb.build.TaskStarted): | 129 | if isinstance(e, bb.build.TaskStarted): |
| 226 | bn = get_bn(e) | 130 | bn = get_bn(e) |
| 227 | device = get_device(e) | ||
| 228 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | 131 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) |
| 229 | taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) | 132 | taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) |
| 230 | if device != "NoLogicalDevice": | ||
| 231 | set_diskdata("__diskdata_task", device, e.data) | ||
| 232 | set_timedata("__timedata_task", e.data, e.time) | 133 | set_timedata("__timedata_task", e.data, e.time) |
| 233 | bb.utils.mkdirhier(taskdir) | 134 | bb.utils.mkdirhier(taskdir) |
| 234 | # write into the task event file the name and start time | 135 | # write into the task event file the name and start time |
| @@ -238,10 +139,9 @@ python run_buildstats () { | |||
| 238 | 139 | ||
| 239 | elif isinstance(e, bb.build.TaskSucceeded): | 140 | elif isinstance(e, bb.build.TaskSucceeded): |
| 240 | bn = get_bn(e) | 141 | bn = get_bn(e) |
| 241 | device = get_device(e) | ||
| 242 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | 142 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) |
| 243 | taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) | 143 | taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) |
| 244 | write_task_data("passed", os.path.join(taskdir, e.task), device, e) | 144 | write_task_data("passed", os.path.join(taskdir, e.task), e) |
| 245 | if e.task == "do_rootfs": | 145 | if e.task == "do_rootfs": |
| 246 | bs = os.path.join(bsdir, "build_stats") | 146 | bs = os.path.join(bsdir, "build_stats") |
| 247 | with open(bs, "a") as f: | 147 | with open(bs, "a") as f: |
| @@ -251,10 +151,9 @@ python run_buildstats () { | |||
| 251 | 151 | ||
| 252 | elif isinstance(e, bb.build.TaskFailed): | 152 | elif isinstance(e, bb.build.TaskFailed): |
| 253 | bn = get_bn(e) | 153 | bn = get_bn(e) |
| 254 | device = get_device(e) | ||
| 255 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) | 154 | bsdir = os.path.join(e.data.getVar('BUILDSTATS_BASE', True), bn) |
| 256 | taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) | 155 | taskdir = os.path.join(bsdir, e.data.getVar('PF', True)) |
| 257 | write_task_data("failed", os.path.join(taskdir, e.task), device, e) | 156 | write_task_data("failed", os.path.join(taskdir, e.task), e) |
| 258 | ######################################################################## | 157 | ######################################################################## |
| 259 | # Lets make things easier and tell people where the build failed in | 158 | # Lets make things easier and tell people where the build failed in |
| 260 | # build_status. We do this here because BuildCompleted triggers no | 159 | # build_status. We do this here because BuildCompleted triggers no |
