diff options
Diffstat (limited to 'meta/classes/buildstats.bbclass')
-rw-r--r-- | meta/classes/buildstats.bbclass | 251 |
1 files changed, 0 insertions, 251 deletions
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass deleted file mode 100644 index a8ee6e69a6..0000000000 --- a/meta/classes/buildstats.bbclass +++ /dev/null | |||
@@ -1,251 +0,0 @@ | |||
1 | BUILDSTATS_BASE = "${TMPDIR}/buildstats/" | ||
2 | |||
3 | ################################################################################ | ||
4 | # Build statistics gathering. | ||
5 | # | ||
6 | # The CPU and Time gathering/tracking functions and bbevent inspiration | ||
7 | # were written by Christopher Larson. | ||
8 | # | ||
9 | ################################################################################ | ||
10 | |||
11 | def get_buildprocess_cputime(pid): | ||
12 | with open("/proc/%d/stat" % pid, "r") as f: | ||
13 | fields = f.readline().rstrip().split() | ||
14 | # 13: utime, 14: stime, 15: cutime, 16: cstime | ||
15 | return sum(int(field) for field in fields[13:16]) | ||
16 | |||
17 | def get_process_cputime(pid): | ||
18 | import resource | ||
19 | with open("/proc/%d/stat" % pid, "r") as f: | ||
20 | fields = f.readline().rstrip().split() | ||
21 | stats = { | ||
22 | 'utime' : fields[13], | ||
23 | 'stime' : fields[14], | ||
24 | 'cutime' : fields[15], | ||
25 | 'cstime' : fields[16], | ||
26 | } | ||
27 | iostats = {} | ||
28 | if os.path.isfile("/proc/%d/io" % pid): | ||
29 | with open("/proc/%d/io" % pid, "r") as f: | ||
30 | while True: | ||
31 | i = f.readline().strip() | ||
32 | if not i: | ||
33 | break | ||
34 | if not ":" in i: | ||
35 | # one more extra line is appended (empty or containing "0") | ||
36 | # most probably due to race condition in kernel while | ||
37 | # updating IO stats | ||
38 | break | ||
39 | i = i.split(": ") | ||
40 | iostats[i[0]] = i[1] | ||
41 | resources = resource.getrusage(resource.RUSAGE_SELF) | ||
42 | childres = resource.getrusage(resource.RUSAGE_CHILDREN) | ||
43 | return stats, iostats, resources, childres | ||
44 | |||
45 | def get_cputime(): | ||
46 | with open("/proc/stat", "r") as f: | ||
47 | fields = f.readline().rstrip().split()[1:] | ||
48 | return sum(int(field) for field in fields) | ||
49 | |||
50 | def set_timedata(var, d, server_time): | ||
51 | d.setVar(var, server_time) | ||
52 | |||
53 | def get_timedata(var, d, end_time): | ||
54 | oldtime = d.getVar(var, False) | ||
55 | if oldtime is None: | ||
56 | return | ||
57 | return end_time - oldtime | ||
58 | |||
59 | def set_buildtimedata(var, d): | ||
60 | import time | ||
61 | time = time.time() | ||
62 | cputime = get_cputime() | ||
63 | proctime = get_buildprocess_cputime(os.getpid()) | ||
64 | d.setVar(var, (time, cputime, proctime)) | ||
65 | |||
66 | def get_buildtimedata(var, d): | ||
67 | import time | ||
68 | timedata = d.getVar(var, False) | ||
69 | if timedata is None: | ||
70 | return | ||
71 | oldtime, oldcpu, oldproc = timedata | ||
72 | procdiff = get_buildprocess_cputime(os.getpid()) - oldproc | ||
73 | cpudiff = get_cputime() - oldcpu | ||
74 | end_time = time.time() | ||
75 | timediff = end_time - oldtime | ||
76 | if cpudiff > 0: | ||
77 | cpuperc = float(procdiff) * 100 / cpudiff | ||
78 | else: | ||
79 | cpuperc = None | ||
80 | return timediff, cpuperc | ||
81 | |||
82 | def write_task_data(status, logfile, e, d): | ||
83 | with open(os.path.join(logfile), "a") as f: | ||
84 | elapsedtime = get_timedata("__timedata_task", d, e.time) | ||
85 | if elapsedtime: | ||
86 | f.write(d.expand("${PF}: %s\n" % e.task)) | ||
87 | f.write(d.expand("Elapsed time: %0.2f seconds\n" % elapsedtime)) | ||
88 | cpu, iostats, resources, childres = get_process_cputime(os.getpid()) | ||
89 | if cpu: | ||
90 | f.write("utime: %s\n" % cpu['utime']) | ||
91 | f.write("stime: %s\n" % cpu['stime']) | ||
92 | f.write("cutime: %s\n" % cpu['cutime']) | ||
93 | f.write("cstime: %s\n" % cpu['cstime']) | ||
94 | for i in iostats: | ||
95 | f.write("IO %s: %s\n" % (i, iostats[i])) | ||
96 | rusages = ["ru_utime", "ru_stime", "ru_maxrss", "ru_minflt", "ru_majflt", "ru_inblock", "ru_oublock", "ru_nvcsw", "ru_nivcsw"] | ||
97 | for i in rusages: | ||
98 | f.write("rusage %s: %s\n" % (i, getattr(resources, i))) | ||
99 | for i in rusages: | ||
100 | f.write("Child rusage %s: %s\n" % (i, getattr(childres, i))) | ||
101 | if status == "passed": | ||
102 | f.write("Status: PASSED \n") | ||
103 | else: | ||
104 | f.write("Status: FAILED \n") | ||
105 | f.write("Ended: %0.2f \n" % e.time) | ||
106 | |||
107 | def write_host_data(logfile, e, d): | ||
108 | import subprocess, os, datetime | ||
109 | cmds = d.getVar('BB_LOG_HOST_STAT_CMDS') | ||
110 | if cmds is None: | ||
111 | d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") | ||
112 | d.setVar("BB_LOG_HOST_STAT_ON_FAILURE", "0") | ||
113 | bb.warn("buildstats: Collecting host data failed. Set BB_LOG_HOST_STAT_CMDS=\"command1 ; command2 ; ... \" in conf\/local.conf\n") | ||
114 | return | ||
115 | path = d.getVar("PATH") | ||
116 | opath = d.getVar("BB_ORIGENV", False).getVar("PATH") | ||
117 | ospath = os.environ['PATH'] | ||
118 | os.environ['PATH'] = path + ":" + opath + ":" + ospath | ||
119 | with open(logfile, "a") as f: | ||
120 | f.write("Event Time: %f\nDate: %s\n" % (e.time, datetime.datetime.now())) | ||
121 | for cmd in cmds.split(";"): | ||
122 | if len(cmd) == 0: | ||
123 | continue | ||
124 | try: | ||
125 | output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT, timeout=1).decode('utf-8') | ||
126 | except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as err: | ||
127 | output = "Error running command: %s\n%s\n" % (cmd, err) | ||
128 | f.write("%s\n%s\n" % (cmd, output)) | ||
129 | os.environ['PATH'] = ospath | ||
130 | |||
131 | python run_buildstats () { | ||
132 | import bb.build | ||
133 | import bb.event | ||
134 | import time, subprocess, platform | ||
135 | |||
136 | bn = d.getVar('BUILDNAME') | ||
137 | ######################################################################## | ||
138 | # bitbake fires HeartbeatEvent even before a build has been | ||
139 | # triggered, causing BUILDNAME to be None | ||
140 | ######################################################################## | ||
141 | if bn is not None: | ||
142 | bsdir = os.path.join(d.getVar('BUILDSTATS_BASE'), bn) | ||
143 | taskdir = os.path.join(bsdir, d.getVar('PF')) | ||
144 | if isinstance(e, bb.event.HeartbeatEvent) and bb.utils.to_boolean(d.getVar("BB_LOG_HOST_STAT_ON_INTERVAL")): | ||
145 | bb.utils.mkdirhier(bsdir) | ||
146 | write_host_data(os.path.join(bsdir, "host_stats"), e, d) | ||
147 | |||
148 | if isinstance(e, bb.event.BuildStarted): | ||
149 | ######################################################################## | ||
150 | # If the kernel was not configured to provide I/O statistics, issue | ||
151 | # a one time warning. | ||
152 | ######################################################################## | ||
153 | if not os.path.isfile("/proc/%d/io" % os.getpid()): | ||
154 | bb.warn("The Linux kernel on your build host was not configured to provide process I/O statistics. (CONFIG_TASK_IO_ACCOUNTING is not set)") | ||
155 | |||
156 | ######################################################################## | ||
157 | # at first pass make the buildstats hierarchy and then | ||
158 | # set the buildname | ||
159 | ######################################################################## | ||
160 | bb.utils.mkdirhier(bsdir) | ||
161 | set_buildtimedata("__timedata_build", d) | ||
162 | build_time = os.path.join(bsdir, "build_stats") | ||
163 | # write start of build into build_time | ||
164 | with open(build_time, "a") as f: | ||
165 | host_info = platform.uname() | ||
166 | f.write("Host Info: ") | ||
167 | for x in host_info: | ||
168 | if x: | ||
169 | f.write(x + " ") | ||
170 | f.write("\n") | ||
171 | f.write("Build Started: %0.2f \n" % d.getVar('__timedata_build', False)[0]) | ||
172 | |||
173 | elif isinstance(e, bb.event.BuildCompleted): | ||
174 | build_time = os.path.join(bsdir, "build_stats") | ||
175 | with open(build_time, "a") as f: | ||
176 | ######################################################################## | ||
177 | # Write build statistics for the build | ||
178 | ######################################################################## | ||
179 | timedata = get_buildtimedata("__timedata_build", d) | ||
180 | if timedata: | ||
181 | time, cpu = timedata | ||
182 | # write end of build and cpu used into build_time | ||
183 | f.write("Elapsed time: %0.2f seconds \n" % (time)) | ||
184 | if cpu: | ||
185 | f.write("CPU usage: %0.1f%% \n" % cpu) | ||
186 | |||
187 | if isinstance(e, bb.build.TaskStarted): | ||
188 | set_timedata("__timedata_task", d, e.time) | ||
189 | bb.utils.mkdirhier(taskdir) | ||
190 | # write into the task event file the name and start time | ||
191 | with open(os.path.join(taskdir, e.task), "a") as f: | ||
192 | f.write("Event: %s \n" % bb.event.getName(e)) | ||
193 | f.write("Started: %0.2f \n" % e.time) | ||
194 | |||
195 | elif isinstance(e, bb.build.TaskSucceeded): | ||
196 | write_task_data("passed", os.path.join(taskdir, e.task), e, d) | ||
197 | if e.task == "do_rootfs": | ||
198 | bs = os.path.join(bsdir, "build_stats") | ||
199 | with open(bs, "a") as f: | ||
200 | rootfs = d.getVar('IMAGE_ROOTFS') | ||
201 | if os.path.isdir(rootfs): | ||
202 | try: | ||
203 | rootfs_size = subprocess.check_output(["du", "-sh", rootfs], | ||
204 | stderr=subprocess.STDOUT).decode('utf-8') | ||
205 | f.write("Uncompressed Rootfs size: %s" % rootfs_size) | ||
206 | except subprocess.CalledProcessError as err: | ||
207 | bb.warn("Failed to get rootfs size: %s" % err.output.decode('utf-8')) | ||
208 | |||
209 | elif isinstance(e, bb.build.TaskFailed): | ||
210 | # Can have a failure before TaskStarted so need to mkdir here too | ||
211 | bb.utils.mkdirhier(taskdir) | ||
212 | write_task_data("failed", os.path.join(taskdir, e.task), e, d) | ||
213 | ######################################################################## | ||
214 | # Lets make things easier and tell people where the build failed in | ||
215 | # build_status. We do this here because BuildCompleted triggers no | ||
216 | # matter what the status of the build actually is | ||
217 | ######################################################################## | ||
218 | build_status = os.path.join(bsdir, "build_stats") | ||
219 | with open(build_status, "a") as f: | ||
220 | f.write(d.expand("Failed at: ${PF} at task: %s \n" % e.task)) | ||
221 | if bb.utils.to_boolean(d.getVar("BB_LOG_HOST_STAT_ON_FAILURE")): | ||
222 | write_host_data(build_status, e, d) | ||
223 | } | ||
224 | |||
225 | addhandler run_buildstats | ||
226 | run_buildstats[eventmask] = "bb.event.BuildStarted bb.event.BuildCompleted bb.event.HeartbeatEvent bb.build.TaskStarted bb.build.TaskSucceeded bb.build.TaskFailed" | ||
227 | |||
228 | python runqueue_stats () { | ||
229 | import buildstats | ||
230 | from bb import event, runqueue | ||
231 | # We should not record any samples before the first task has started, | ||
232 | # because that's the first activity shown in the process chart. | ||
233 | # Besides, at that point we are sure that the build variables | ||
234 | # are available that we need to find the output directory. | ||
235 | # The persistent SystemStats is stored in the datastore and | ||
236 | # closed when the build is done. | ||
237 | system_stats = d.getVar('_buildstats_system_stats', False) | ||
238 | if not system_stats and isinstance(e, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted)): | ||
239 | system_stats = buildstats.SystemStats(d) | ||
240 | d.setVar('_buildstats_system_stats', system_stats) | ||
241 | if system_stats: | ||
242 | # Ensure that we sample at important events. | ||
243 | done = isinstance(e, bb.event.BuildCompleted) | ||
244 | system_stats.sample(e, force=done) | ||
245 | if done: | ||
246 | system_stats.close() | ||
247 | d.delVar('_buildstats_system_stats') | ||
248 | } | ||
249 | |||
250 | addhandler runqueue_stats | ||
251 | runqueue_stats[eventmask] = "bb.runqueue.sceneQueueTaskStarted bb.runqueue.runQueueTaskStarted bb.event.HeartbeatEvent bb.event.BuildCompleted bb.event.MonitorDiskEvent" | ||