diff options
author | Sakib Sajal <sakib.sajal@windriver.com> | 2021-07-09 16:53:43 -0400 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2021-07-10 22:42:22 +0100 |
commit | 8ce9e8b268c3d6a8c5f22de9352656efd3e93cfb (patch) | |
tree | 10b9a795e7cce0325e2423af2270b2eaeb9ef8db | |
parent | 404463335113da2a6d0674e959109e069f9bb576 (diff) | |
download | poky-8ce9e8b268c3d6a8c5f22de9352656efd3e93cfb.tar.gz |
buildstats.bbclass: log host data on failure to task specific file
host data, for both interval and failure, was previously
logged into the same file which was difficult to read
as the files file were usually large.
host data is now logged into separate files, for each
type of logging (failure and interval) and also for each
failed task making it easier to read/parse.
(From OE-Core rev: 1a0fb3c0794f4e66086e567a297b4d9379c6b8f3)
Signed-off-by: Sakib Sajal <sakib.sajal@windriver.com>
Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | meta/classes/buildstats.bbclass | 44 |
1 files changed, 26 insertions, 18 deletions
diff --git a/meta/classes/buildstats.bbclass b/meta/classes/buildstats.bbclass index 8a1466d3fe..0de605200a 100644 --- a/meta/classes/buildstats.bbclass +++ b/meta/classes/buildstats.bbclass | |||
@@ -108,25 +108,31 @@ def write_host_data(logfile, e, d, type): | |||
108 | import subprocess, os, datetime | 108 | import subprocess, os, datetime |
109 | # minimum time allowed for each command to run, in seconds | 109 | # minimum time allowed for each command to run, in seconds |
110 | time_threshold = 0.5 | 110 | time_threshold = 0.5 |
111 | limit = 10 | ||
111 | # the total number of commands | 112 | # the total number of commands |
112 | num_cmds = 0 | 113 | num_cmds = 0 |
113 | # interval at which data will be logged | ||
114 | interval = int(d.getVar("BB_HEARTBEAT_EVENT", False)) | ||
115 | msg = "" | 114 | msg = "" |
116 | if type == "interval": | 115 | if type == "interval": |
116 | # interval at which data will be logged | ||
117 | interval = d.getVar("BB_HEARTBEAT_EVENT", False) | ||
118 | if interval is None: | ||
119 | bb.warn("buildstats: Collecting host data at intervals failed. Set BB_HEARTBEAT_EVENT=\"<interval>\" in conf/local.conf for the interval at which host data will be logged.") | ||
120 | d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") | ||
121 | return | ||
122 | interval = int(interval) | ||
117 | cmds = d.getVar('BB_LOG_HOST_STAT_CMDS_INTERVAL') | 123 | cmds = d.getVar('BB_LOG_HOST_STAT_CMDS_INTERVAL') |
118 | msg = "Host Stats: Collecting data at interval.\n" | 124 | msg = "Host Stats: Collecting data at %d second intervals.\n" % interval |
119 | if cmds is None: | 125 | if cmds is None: |
120 | d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") | 126 | d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") |
121 | bb.warn("buildstats: Collecting host data at intervals failed. Set BB_LOG_HOST_STAT_CMDS_INTERVAL=\"command1 ; command2 ; ... \" in conf/local.conf\n") | 127 | bb.warn("buildstats: Collecting host data at intervals failed. Set BB_LOG_HOST_STAT_CMDS_INTERVAL=\"command1 ; command2 ; ... \" in conf/local.conf.") |
122 | return | 128 | return |
123 | if type == "failure": | 129 | if type == "failure": |
124 | cmds = d.getVar('BB_LOG_HOST_STAT_CMDS_FAILURE') | 130 | cmds = d.getVar('BB_LOG_HOST_STAT_CMDS_FAILURE') |
125 | msg = "Host Stats: Collecting data on failure.\n" | 131 | msg = "Host Stats: Collecting data on failure.\n" |
126 | msg += "Failed at task " + e.task + "\n" | 132 | msg += "Failed at task: " + e.task + "\n" |
127 | if cmds is None: | 133 | if cmds is None: |
128 | d.setVar("BB_LOG_HOST_STAT_ON_FAILURE", "0") | 134 | d.setVar("BB_LOG_HOST_STAT_ON_FAILURE", "0") |
129 | bb.warn("buildstats: Collecting host data on failure failed. Set BB_LOG_HOST_STAT_CMDS_FAILURE=\"command1 ; command2 ; ... \" in conf/local.conf\n") | 135 | bb.warn("buildstats: Collecting host data on failure failed. Set BB_LOG_HOST_STAT_CMDS_FAILURE=\"command1 ; command2 ; ... \" in conf/local.conf.") |
130 | return | 136 | return |
131 | c_san = [] | 137 | c_san = [] |
132 | for cmd in cmds.split(";"): | 138 | for cmd in cmds.split(";"): |
@@ -134,18 +140,20 @@ def write_host_data(logfile, e, d, type): | |||
134 | continue | 140 | continue |
135 | num_cmds += 1 | 141 | num_cmds += 1 |
136 | c_san.append(cmd) | 142 | c_san.append(cmd) |
137 | if num_cmds <= 0: | 143 | if num_cmds == 0: |
138 | d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") | 144 | if type == "interval": |
139 | d.setVar("BB_LOG_HOST_STAT_ON_FAILURE", "0") | 145 | d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") |
146 | if type == "failure": | ||
147 | d.setVar("BB_LOG_HOST_STAT_ON_FAILURE", "0") | ||
140 | return | 148 | return |
141 | 149 | ||
142 | # return if the interval is not enough to run all commands within the specified BB_HEARTBEAT_EVENT interval | 150 | # return if the interval is not enough to run all commands within the specified BB_HEARTBEAT_EVENT interval |
143 | limit = interval / num_cmds | 151 | if type == "interval": |
144 | if limit <= time_threshold: | 152 | limit = interval / num_cmds |
145 | d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") | 153 | if limit <= time_threshold: |
146 | d.setVar("BB_LOG_HOST_STAT_ON_FAILURE", "0") | 154 | d.setVar("BB_LOG_HOST_STAT_ON_INTERVAL", "0") |
147 | bb.warn("buildstats: Collecting host data failed. BB_HEARTBEAT_EVENT interval not enough to run the specified commands. HINT: Increase value of BB_HEARTBEAT_EVENT in conf/local.conf\n") | 155 | bb.warn("buildstats: Collecting host data failed. BB_HEARTBEAT_EVENT interval not enough to run the specified commands. Increase value of BB_HEARTBEAT_EVENT in conf/local.conf.") |
148 | return | 156 | return |
149 | 157 | ||
150 | # set the environment variables | 158 | # set the environment variables |
151 | path = d.getVar("PATH") | 159 | path = d.getVar("PATH") |
@@ -179,7 +187,7 @@ python run_buildstats () { | |||
179 | taskdir = os.path.join(bsdir, d.getVar('PF')) | 187 | taskdir = os.path.join(bsdir, d.getVar('PF')) |
180 | if isinstance(e, bb.event.HeartbeatEvent) and bb.utils.to_boolean(d.getVar("BB_LOG_HOST_STAT_ON_INTERVAL")): | 188 | if isinstance(e, bb.event.HeartbeatEvent) and bb.utils.to_boolean(d.getVar("BB_LOG_HOST_STAT_ON_INTERVAL")): |
181 | bb.utils.mkdirhier(bsdir) | 189 | bb.utils.mkdirhier(bsdir) |
182 | write_host_data(os.path.join(bsdir, "host_stats"), e, d, "interval") | 190 | write_host_data(os.path.join(bsdir, "host_stats_interval"), e, d, "interval") |
183 | 191 | ||
184 | if isinstance(e, bb.event.BuildStarted): | 192 | if isinstance(e, bb.event.BuildStarted): |
185 | ######################################################################## | 193 | ######################################################################## |
@@ -254,8 +262,8 @@ python run_buildstats () { | |||
254 | build_status = os.path.join(bsdir, "build_stats") | 262 | build_status = os.path.join(bsdir, "build_stats") |
255 | with open(build_status, "a") as f: | 263 | with open(build_status, "a") as f: |
256 | f.write(d.expand("Failed at: ${PF} at task: %s \n" % e.task)) | 264 | f.write(d.expand("Failed at: ${PF} at task: %s \n" % e.task)) |
257 | if bb.utils.to_boolean(d.getVar("BB_LOG_HOST_STAT_ON_FAILURE")): | 265 | if bb.utils.to_boolean(d.getVar("BB_LOG_HOST_STAT_ON_FAILURE")): |
258 | write_host_data(os.path.join(bsdir, "host_stats"), e, d, "failure") | 266 | write_host_data(os.path.join(bsdir, "host_stats_%s_failure" % e.task), e, d, "failure") |
259 | } | 267 | } |
260 | 268 | ||
261 | addhandler run_buildstats | 269 | addhandler run_buildstats |