summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorElliot Smith <elliot.smith@intel.com>2015-10-29 13:59:44 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-10-29 22:05:36 +0000
commita0791c19db7d5b33ce7a4e3614a574ad4a319a84 (patch)
tree781a2469391bacf2f245a2a74acf25d4a7a39335
parent290534d3ff53b3c0a50e480863474d1785088c5a (diff)
downloadpoky-a0791c19db7d5b33ce7a4e3614a574ad4a319a84.tar.gz
bitbake: toasterui: Create per-build logs
Rather than relying on bug 8411, which is conveniently creating separate log files for each of our builds, create our own log file for each build. The log files are created in the same tmp directories that bitbake users, but are timestamped to the millisecond to avoid name collisions. Each log file is opened on a ParseStarted event (for builds triggered by Toaster) or BuildStarted event (for builds on the command line: Toaster doesn't get the ParseStarted event for command-line builds). The log file is closed on the BuildCompleted event, or if the build fails. Because we start logging on ParseStarted for Toaster builds, we're able to capture the "Build Configuration" section which bitbake writes to output. [YOCTO #8373] (Bitbake rev: 7974203cd8bc66dff1fcc55f8723dedefaf72840) Signed-off-by: Elliot Smith <elliot.smith@intel.com> Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py8
-rw-r--r--bitbake/lib/bb/ui/toasterui.py116
2 files changed, 94 insertions, 30 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 2fc1a43c40..78f1e9274f 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -784,7 +784,7 @@ class BuildInfoHelper(object):
784 ## methods to convert event/external info into objects that the ORM layer uses 784 ## methods to convert event/external info into objects that the ORM layer uses
785 785
786 786
787 def _get_build_information(self, consolelogfile): 787 def _get_build_information(self, build_log_path):
788 build_info = {} 788 build_info = {}
789 # Generate an identifier for each new build 789 # Generate an identifier for each new build
790 790
@@ -793,7 +793,7 @@ class BuildInfoHelper(object):
793 build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] 793 build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
794 build_info['started_on'] = timezone.now() 794 build_info['started_on'] = timezone.now()
795 build_info['completed_on'] = timezone.now() 795 build_info['completed_on'] = timezone.now()
796 build_info['cooker_log_path'] = consolelogfile 796 build_info['cooker_log_path'] = build_log_path
797 build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] 797 build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
798 build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] 798 build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
799 799
@@ -934,9 +934,9 @@ class BuildInfoHelper(object):
934 logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) 934 logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee)
935 935
936 936
937 def store_started_build(self, event, consolelogfile): 937 def store_started_build(self, event, build_log_path):
938 assert '_pkgs' in vars(event) 938 assert '_pkgs' in vars(event)
939 build_information = self._get_build_information(consolelogfile) 939 build_information = self._get_build_information(build_log_path)
940 940
941 build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) 941 build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project)
942 942
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index 2b3bc3f436..3d261503ea 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -21,6 +21,7 @@
21# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 21# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22 22
23from __future__ import division 23from __future__ import division
24import time
24import sys 25import sys
25try: 26try:
26 import bb 27 import bb
@@ -43,8 +44,6 @@ featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeature
43logger = logging.getLogger("ToasterLogger") 44logger = logging.getLogger("ToasterLogger")
44interactive = sys.stdout.isatty() 45interactive = sys.stdout.isatty()
45 46
46
47
48def _log_settings_from_server(server): 47def _log_settings_from_server(server):
49 # Get values of variables which control our output 48 # Get values of variables which control our output
50 includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"]) 49 includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
@@ -55,16 +54,60 @@ def _log_settings_from_server(server):
55 if error: 54 if error:
56 logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s", error) 55 logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s", error)
57 raise BaseException(error) 56 raise BaseException(error)
58 consolelogfile, error = server.runCommand(["getSetVariable", "BB_CONSOLELOG"]) 57 consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"])
59 if error: 58 if error:
60 logger.error("Unable to get the value of BB_CONSOLELOG variable: %s", error) 59 logger.error("Unable to get the value of BB_CONSOLELOG variable: %s", error)
61 raise BaseException(error) 60 raise BaseException(error)
62 return includelogs, loglines, consolelogfile 61 return consolelogfile
62
63# create a log file for a single build and direct the logger at it;
64# log file name is timestamped to the millisecond (depending
65# on system clock accuracy) to ensure it doesn't overlap with
66# other log file names
67#
68# returns (log file, path to log file) for a build
69def _open_build_log(log_dir):
70 format_str = "%(levelname)s: %(message)s"
71
72 now = time.time()
73 now_ms = int((now - int(now)) * 1000)
74 time_str = time.strftime('build_%Y%m%d_%H%M%S', time.localtime(now))
75 log_file_name = time_str + ('.%d.log' % now_ms)
76 build_log_file_path = os.path.join(log_dir, log_file_name)
77
78 build_log = logging.FileHandler(build_log_file_path)
79
80 logformat = bb.msg.BBLogFormatter(format_str)
81 build_log.setFormatter(logformat)
63 82
83 bb.msg.addDefaultlogFilter(build_log)
84 logger.addHandler(build_log)
85
86 return (build_log, build_log_file_path)
87
88# stop logging to the build log if it exists
89def _close_build_log(build_log):
90 if build_log:
91 build_log.flush()
92 build_log.close()
93 logger.removeHandler(build_log)
94
95def main(server, eventHandler, params):
96 # set to a logging.FileHandler instance when a build starts;
97 # see _open_build_log()
98 build_log = None
99
100 # set to the log path when a build starts
101 build_log_file_path = None
64 102
65def main(server, eventHandler, params ):
66 helper = uihelper.BBUIHelper() 103 helper = uihelper.BBUIHelper()
67 104
105 # TODO don't use log output to determine when bitbake has started
106 #
107 # WARNING: this log handler cannot be removed, as localhostbecontroller
108 # relies on output in the toaster_ui.log file to determine whether
109 # the bitbake server has started, which only happens if
110 # this logger is setup here (see the TODO in the loop below)
68 console = logging.StreamHandler(sys.stdout) 111 console = logging.StreamHandler(sys.stdout)
69 format_str = "%(levelname)s: %(message)s" 112 format_str = "%(levelname)s: %(message)s"
70 formatter = bb.msg.BBLogFormatter(format_str) 113 formatter = bb.msg.BBLogFormatter(format_str)
@@ -73,8 +116,6 @@ def main(server, eventHandler, params ):
73 logger.addHandler(console) 116 logger.addHandler(console)
74 logger.setLevel(logging.INFO) 117 logger.setLevel(logging.INFO)
75 118
76 _, _, consolelogfile = _log_settings_from_server(server)
77
78 # verify and warn 119 # verify and warn
79 build_history_enabled = True 120 build_history_enabled = True
80 inheritlist, _ = server.runCommand(["getVariable", "INHERIT"]) 121 inheritlist, _ = server.runCommand(["getVariable", "INHERIT"])
@@ -87,8 +128,9 @@ def main(server, eventHandler, params ):
87 logger.error("ToasterUI can only work in observer mode") 128 logger.error("ToasterUI can only work in observer mode")
88 return 1 129 return 1
89 130
90 131 # set to 1 when toasterui needs to shut down
91 main.shutdown = 0 132 main.shutdown = 0
133
92 interrupted = False 134 interrupted = False
93 return_value = 0 135 return_value = 0
94 errors = 0 136 errors = 0
@@ -98,25 +140,31 @@ def main(server, eventHandler, params ):
98 140
99 buildinfohelper = BuildInfoHelper(server, build_history_enabled) 141 buildinfohelper = BuildInfoHelper(server, build_history_enabled)
100 142
101 if buildinfohelper.brbe is not None and consolelogfile: 143 # write our own log files into bitbake's log directory;
102 # if we are under managed mode we have no other UI and we need to write our own file 144 # we're only interested in the path to the parent directory of
103 bb.utils.mkdirhier(os.path.dirname(consolelogfile)) 145 # this file, as we're writing our own logs into the same directory
104 conlogformat = bb.msg.BBLogFormatter(format_str) 146 consolelogfile = _log_settings_from_server(server)
105 consolelog = logging.FileHandler(consolelogfile) 147 log_dir = os.path.dirname(consolelogfile)
106 bb.msg.addDefaultlogFilter(consolelog) 148 bb.utils.mkdirhier(log_dir)
107 consolelog.setFormatter(conlogformat)
108 logger.addHandler(consolelog)
109
110 149
111 while True: 150 while True:
112 try: 151 try:
113 event = eventHandler.waitEvent(0.25) 152 event = eventHandler.waitEvent(0.25)
114 if first: 153 if first:
115 first = False 154 first = False
155
156 # TODO don't use log output to determine when bitbake has started
157 #
158 # this is the line localhostbecontroller needs to
159 # see in toaster_ui.log which it uses to decide whether
160 # the bitbake server has started...
116 logger.info("ToasterUI waiting for events") 161 logger.info("ToasterUI waiting for events")
117 162
118 if event is None: 163 if event is None:
119 if main.shutdown > 0: 164 if main.shutdown > 0:
165 # if shutting down, close any open build log first
166 _close_build_log(build_log)
167
120 break 168 break
121 continue 169 continue
122 170
@@ -125,8 +173,21 @@ def main(server, eventHandler, params ):
125 # pylint: disable=protected-access 173 # pylint: disable=protected-access
126 # the code will look into the protected variables of the event; no easy way around this 174 # the code will look into the protected variables of the event; no easy way around this
127 175
176 # we treat ParseStarted as the first event of toaster-triggered
177 # builds; that way we get the Build Configuration included in the log
178 # and any errors that occur before BuildStarted is fired
179 if isinstance(event, bb.event.ParseStarted):
180 if not (build_log and build_log_file_path):
181 build_log, build_log_file_path = _open_build_log(log_dir)
182 continue
183
128 if isinstance(event, bb.event.BuildStarted): 184 if isinstance(event, bb.event.BuildStarted):
129 buildinfohelper.store_started_build(event, consolelogfile) 185 # command-line builds don't fire a ParseStarted event,
186 # so we have to start the log file for those on BuildStarted instead
187 if not (build_log and build_log_file_path):
188 build_log, build_log_file_path = _open_build_log(log_dir)
189
190 buildinfohelper.store_started_build(event, build_log_file_path)
130 191
131 if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): 192 if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
132 buildinfohelper.update_and_store_task(event) 193 buildinfohelper.update_and_store_task(event)
@@ -171,8 +232,6 @@ def main(server, eventHandler, params ):
171 # timing and error informations from the parsing phase in Toaster 232 # timing and error informations from the parsing phase in Toaster
172 if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)): 233 if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)):
173 continue 234 continue
174 if isinstance(event, bb.event.ParseStarted):
175 continue
176 if isinstance(event, bb.event.ParseProgress): 235 if isinstance(event, bb.event.ParseProgress):
177 continue 236 continue
178 if isinstance(event, bb.event.ParseCompleted): 237 if isinstance(event, bb.event.ParseCompleted):
@@ -248,6 +307,12 @@ def main(server, eventHandler, params ):
248 errorcode = 1 307 errorcode = 1
249 logger.error("Command execution failed: %s", event.error) 308 logger.error("Command execution failed: %s", event.error)
250 309
310 # turn off logging to the current build log
311 _close_build_log(build_log)
312
313 # reset ready for next BuildStarted
314 build_log = None
315
251 # update the build info helper on BuildCompleted, not on CommandXXX 316 # update the build info helper on BuildCompleted, not on CommandXXX
252 buildinfohelper.update_build_information(event, errors, warnings, taskfailures) 317 buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
253 buildinfohelper.close(errorcode) 318 buildinfohelper.close(errorcode)
@@ -256,7 +321,6 @@ def main(server, eventHandler, params ):
256 321
257 # we start a new build info 322 # we start a new build info
258 if buildinfohelper.brbe is not None: 323 if buildinfohelper.brbe is not None:
259
260 logger.debug("ToasterUI under BuildEnvironment management - exiting after the build") 324 logger.debug("ToasterUI under BuildEnvironment management - exiting after the build")
261 server.terminateServer() 325 server.terminateServer()
262 else: 326 else:
@@ -298,8 +362,9 @@ def main(server, eventHandler, params ):
298 continue 362 continue
299 363
300 if isinstance(event, bb.cooker.CookerExit): 364 if isinstance(event, bb.cooker.CookerExit):
301 # exit when the server exits 365 # shutdown when bitbake server shuts down
302 break 366 main.shutdown = 1
367 continue
303 368
304 # ignore 369 # ignore
305 if isinstance(event, (bb.event.BuildBase, 370 if isinstance(event, (bb.event.BuildBase,
@@ -350,9 +415,8 @@ def main(server, eventHandler, params ):
350 # make sure we return with an error 415 # make sure we return with an error
351 return_value += 1 416 return_value += 1
352 417
353 if interrupted: 418 if interrupted and return_value == 0:
354 if return_value == 0: 419 return_value += 1
355 return_value += 1
356 420
357 logger.warn("Return value is %d", return_value) 421 logger.warn("Return value is %d", return_value)
358 return return_value 422 return return_value