summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-12-09 11:57:38 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-12-18 10:24:06 +0000
commit85a17f86ea2edf24b54aa62bd25e10ff522cb6e7 (patch)
tree69baa4d959be832b5c096b7d69b0fc2bcb2247b5
parentd086fa3aed34a05d52e73c255ca22379149a64a1 (diff)
downloadpoky-85a17f86ea2edf24b54aa62bd25e10ff522cb6e7.tar.gz
bitbake: add option to write offline event log file
This patch adds a "-w/--write-log" option to bitbake that writes an event log file for the current build. The name of the file is passed as a parameter to the "-w" argument. If the parameter is the empty string '', the file name is generated in the form bitbake_eventlog_DATE.json, where DATE is the current date and time, with second precision. The "-w" option can also be supplied as the BBEVENTLOG environment variable. We add a script, toater-eventreplay, that reads an event log file and loads the data into a Toaster database, creating a build entry. We modify the toasterui to fix minor issues with reading events from an event log file. Performance impact is undetectable under no-task executed builds. (Bitbake rev: 1befb4a783bb7b7b387d4b5ee08830d9516f1ac2) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xbitbake/bin/bitbake12
-rwxr-xr-xbitbake/bin/toaster-eventreplay179
-rw-r--r--bitbake/lib/bb/cooker.py75
-rw-r--r--bitbake/lib/bb/cookerdata.py1
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py53
-rw-r--r--bitbake/lib/bb/ui/toasterui.py2
6 files changed, 300 insertions, 22 deletions
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index 7f8449c7b3..d46c3dde3b 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -196,6 +196,9 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
196 parser.add_option("", "--status-only", help = "Check the status of the remote bitbake server.", 196 parser.add_option("", "--status-only", help = "Check the status of the remote bitbake server.",
197 action = "store_true", dest = "status_only", default = False) 197 action = "store_true", dest = "status_only", default = False)
198 198
199 parser.add_option("-w", "--write-log", help = "Writes the event log of the build to a bitbake event json file. Use '' (empty string) to assign the name automatically.",
200 action = "store", dest = "writeeventlog")
201
199 options, targets = parser.parse_args(sys.argv) 202 options, targets = parser.parse_args(sys.argv)
200 203
201 # some environmental variables set also configuration options 204 # some environmental variables set also configuration options
@@ -206,6 +209,14 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
206 if "BBTOKEN" in os.environ: 209 if "BBTOKEN" in os.environ:
207 options.xmlrpctoken = os.environ["BBTOKEN"] 210 options.xmlrpctoken = os.environ["BBTOKEN"]
208 211
212 if "BBEVENTLOG" is os.environ:
213 options.writeeventlog = os.environ["BBEVENTLOG"]
214
215 # fill in proper log name if not supplied
216 if options.writeeventlog is not None and len(options.writeeventlog) == 0:
217 import datetime
218 options.writeeventlog = "bitbake_eventlog_%s.json" % datetime.datetime.now().strftime("%Y%m%d%H%M%S")
219
209 # if BBSERVER says to autodetect, let's do that 220 # if BBSERVER says to autodetect, let's do that
210 if options.remote_server: 221 if options.remote_server:
211 [host, port] = options.remote_server.split(":", 2) 222 [host, port] = options.remote_server.split(":", 2)
@@ -266,7 +277,6 @@ def start_server(servermodule, configParams, configuration, features):
266 return server 277 return server
267 278
268 279
269
270def main(): 280def main():
271 281
272 configParams = BitBakeConfigParameters() 282 configParams = BitBakeConfigParameters()
diff --git a/bitbake/bin/toaster-eventreplay b/bitbake/bin/toaster-eventreplay
new file mode 100755
index 0000000000..624829aea0
--- /dev/null
+++ b/bitbake/bin/toaster-eventreplay
@@ -0,0 +1,179 @@
1#!/usr/bin/env python
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# Copyright (C) 2014 Alex Damian
6#
7# This file re-uses code spread throughout other Bitbake source files.
8# As such, all other copyrights belong to their own right holders.
9#
10#
11# This program is free software; you can redistribute it and/or modify
12# it under the terms of the GNU General Public License version 2 as
13# published by the Free Software Foundation.
14#
15# This program is distributed in the hope that it will be useful,
16# but WITHOUT ANY WARRANTY; without even the implied warranty of
17# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18# GNU General Public License for more details.
19#
20# You should have received a copy of the GNU General Public License along
21# with this program; if not, write to the Free Software Foundation, Inc.,
22# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
23
24
25# This command takes a filename as a single parameter. The filename is read
26# as a build eventlog, and the ToasterUI is used to process events in the file
27# and log data in the database
28
29import os
30import sys, logging
31
32# mangle syspath to allow easy import of modules
33sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
34 'lib'))
35
36
37import bb.cooker
38from bb.ui import toasterui
39import sys
40import logging
41
42logger = logging.getLogger(__name__)
43console = logging.StreamHandler(sys.stdout)
44format_str = "%(levelname)s: %(message)s"
45logging.basicConfig(format=format_str)
46
47
48import json, pickle
49
50
51class FileReadEventsServerConnection():
52 """ Emulates a connection to a bitbake server that feeds
53 events coming actually read from a saved log file.
54 """
55
56 class MockConnection():
57 """ fill-in for the proxy to the server. we just return generic data
58 """
59 def __init__(self, sc):
60 self._sc = sc
61
62 def runCommand(self, commandArray):
63 """ emulates running a command on the server; only read-only commands are accepted """
64 command_name = commandArray[0]
65
66 if command_name == "getVariable":
67 if commandArray[1] in self._sc._variables:
68 return (self._sc._variables[commandArray[1]]['v'], None)
69 return (None, "Missing variable")
70
71 elif command_name == "getAllKeysWithFlags":
72 dump = {}
73 flaglist = commandArray[1]
74 for k in self._sc._variables.keys():
75 try:
76 if not k.startswith("__"):
77 v = self._sc._variables[k]['v']
78 dump[k] = {
79 'v' : v ,
80 'history' : self._sc._variables[k]['history'],
81 }
82 for d in flaglist:
83 dump[k][d] = self._sc._variables[k][d]
84 except Exception as e:
85 print(e)
86 return (dump, None)
87 else:
88 raise Exception("Command %s not implemented" % commandArray[0])
89
90 def terminateServer(self):
91 """ do not do anything """
92 pass
93
94
95
96 class EventReader():
97 def __init__(self, sc):
98 self._sc = sc
99 self.firstraise = 0
100
101 def _create_event(self, line):
102 def _import_class(name):
103 assert len(name) > 0
104 assert "." in name, name
105
106 components = name.strip().split(".")
107 modulename = ".".join(components[:-1])
108 moduleklass = components[-1]
109
110 module = __import__(modulename, fromlist=[str(moduleklass)])
111 return getattr(module, moduleklass)
112
113 # we build a toaster event out of current event log line
114 try:
115 event_data = json.loads(line.strip())
116 event_class = _import_class(event_data['class'])
117 event_object = pickle.loads(json.loads(event_data['vars']))
118 except ValueError as e:
119 print("Failed loading ", line)
120 raise e
121
122 if not isinstance(event_object, event_class):
123 raise Exception("Error loading objects %s class %s ", event_object, event_class)
124
125 return event_object
126
127 def waitEvent(self, timeout):
128
129 nextline = self._sc._eventfile.readline()
130 if len(nextline) == 0:
131 # the build data ended, while toasterui still waits for events.
132 # this happens when the server was abruptly stopped, so we simulate this
133 self.firstraise += 1
134 if self.firstraise == 1:
135 raise KeyboardInterrupt()
136 else:
137 return None
138 else:
139 self._sc.lineno += 1
140 return self._create_event(nextline)
141
142
143 def _readVariables(self, variableline):
144 self._variables = json.loads(variableline.strip())['allvariables']
145
146
147 def __init__(self, file_name):
148 self.connection = FileReadEventsServerConnection.MockConnection(self)
149 self._eventfile = open(file_name, "r")
150
151 # we expect to have the variable dump at the start of the file
152 self.lineno = 1
153 self._readVariables(self._eventfile.readline())
154
155 self.events = FileReadEventsServerConnection.EventReader(self)
156
157
158
159
160
161class MockConfigParameters():
162 """ stand-in for cookerdata.ConfigParameters; as we don't really config a cooker, this
163 serves just to supply needed interfaces for the toaster ui to work """
164 def __init__(self):
165 self.observe_only = True # we can only read files
166
167
168# run toaster ui on our mock bitbake class
169if __name__ == "__main__":
170 if len(sys.argv) < 2:
171 logger.error("Usage: %s event.log " % sys.argv[0])
172 sys.exit(1)
173
174 file_name = sys.argv[-1]
175 mock_connection = FileReadEventsServerConnection(file_name)
176 configParams = MockConfigParameters()
177
178 # run the main program
179 toasterui.main(mock_connection.connection, mock_connection.events, configParams)
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index df9a0cab03..16fd4ad34c 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -205,6 +205,75 @@ class BBCooker:
205 self.data = self.databuilder.data 205 self.data = self.databuilder.data
206 self.data_hash = self.databuilder.data_hash 206 self.data_hash = self.databuilder.data_hash
207 207
208
209 # we log all events to a file if so directed
210 if self.configuration.writeeventlog:
211 import json, pickle
212 DEFAULT_EVENTFILE = self.configuration.writeeventlog
213 class EventLogWriteHandler():
214
215 class EventWriter():
216 def __init__(self, cooker):
217 self.file_inited = None
218 self.cooker = cooker
219 self.event_queue = []
220
221 def init_file(self):
222 try:
223 # delete the old log
224 os.remove(DEFAULT_EVENTFILE)
225 except:
226 pass
227
228 # write current configuration data
229 with open(DEFAULT_EVENTFILE, "w") as f:
230 f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
231
232 def write_event(self, event):
233 with open(DEFAULT_EVENTFILE, "a") as f:
234 try:
235 f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) }))
236 except Exception as e:
237 import traceback
238 print(e, traceback.format_exc(e))
239
240
241 def send(self, event):
242 event_class = event.__module__ + "." + event.__class__.__name__
243
244 # init on bb.event.BuildStarted
245 if self.file_inited is None:
246 if event_class == "bb.event.BuildStarted":
247 self.init_file()
248 self.file_inited = True
249
250 # write pending events
251 for e in self.event_queue:
252 self.write_event(e)
253
254 # also write the current event
255 self.write_event(event)
256
257 else:
258 # queue all events until the file is inited
259 self.event_queue.append(event)
260
261 else:
262 # we have the file, just write the event
263 self.write_event(event)
264
265 # set our handler's event processor
266 event = EventWriter(self) # self is the cooker here
267
268
269 # set up cooker features for this mock UI handler
270
271 # we need to write the dependency tree in the log
272 self.featureset.setFeature(CookerFeatures.SEND_DEPENDS_TREE)
273 # register the log file writer as UI Handler
274 bb.event.register_UIHhandler(EventLogWriteHandler())
275
276
208 # 277 #
209 # Special updated configuration we use for firing events 278 # Special updated configuration we use for firing events
210 # 279 #
@@ -505,7 +574,7 @@ class BBCooker:
505 taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False) 574 taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False)
506 575
507 return runlist, taskdata 576 return runlist, taskdata
508 577
509 ######## WARNING : this function requires cache_extra to be enabled ######## 578 ######## WARNING : this function requires cache_extra to be enabled ########
510 579
511 def generateTaskDepTreeData(self, pkgs_to_build, task): 580 def generateTaskDepTreeData(self, pkgs_to_build, task):
@@ -1550,10 +1619,10 @@ class CookerCollectFiles(object):
1550 for p in pkgfns: 1619 for p in pkgfns:
1551 realfn, cls = bb.cache.Cache.virtualfn2realfn(p) 1620 realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
1552 priorities[p] = self.calc_bbfile_priority(realfn, matched) 1621 priorities[p] = self.calc_bbfile_priority(realfn, matched)
1553 1622
1554 # Don't show the warning if the BBFILE_PATTERN did match .bbappend files 1623 # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
1555 unmatched = set() 1624 unmatched = set()
1556 for _, _, regex, pri in self.bbfile_config_priorities: 1625 for _, _, regex, pri in self.bbfile_config_priorities:
1557 if not regex in matched: 1626 if not regex in matched:
1558 unmatched.add(regex) 1627 unmatched.add(regex)
1559 1628
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 470d5381ae..2ceed2d867 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -139,6 +139,7 @@ class CookerConfiguration(object):
139 self.dry_run = False 139 self.dry_run = False
140 self.tracking = False 140 self.tracking = False
141 self.interface = [] 141 self.interface = []
142 self.writeeventlog = False
142 143
143 self.env = {} 144 self.env = {}
144 145
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 533f4cef3b..f825b57bea 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -556,7 +556,6 @@ class ORMWrapper(object):
556 assert isinstance(build_obj, Build) 556 assert isinstance(build_obj, Build)
557 557
558 helptext_objects = [] 558 helptext_objects = []
559
560 for k in vardump: 559 for k in vardump:
561 desc = vardump[k]['doc'] 560 desc = vardump[k]['doc']
562 if desc is None: 561 if desc is None:
@@ -667,9 +666,11 @@ class BuildInfoHelper(object):
667 if (path.startswith(bl.layer.local_path)): 666 if (path.startswith(bl.layer.local_path)):
668 return bl 667 return bl
669 668
670 #TODO: if we get here, we didn't read layers correctly 669 #if we get here, we didn't read layers correctly; mockup the new layer
671 assert False 670 unknown_layer, created = Layer.objects.get_or_create(name="unknown", local_path="/", layer_index_url="")
672 return None 671 unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build'])
672
673 return unknown_layer_version_obj
673 674
674 def _get_recipe_information_from_taskfile(self, taskfile): 675 def _get_recipe_information_from_taskfile(self, taskfile):
675 localfilepath = taskfile.split(":")[-1] 676 localfilepath = taskfile.split(":")[-1]
@@ -732,7 +733,6 @@ class BuildInfoHelper(object):
732 733
733 def store_started_build(self, event): 734 def store_started_build(self, event):
734 assert '_pkgs' in vars(event) 735 assert '_pkgs' in vars(event)
735 assert 'lvs' in self.internal_state, "Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass."
736 build_information = self._get_build_information() 736 build_information = self._get_build_information()
737 737
738 build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe) 738 build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe)
@@ -740,10 +740,13 @@ class BuildInfoHelper(object):
740 self.internal_state['build'] = build_obj 740 self.internal_state['build'] = build_obj
741 741
742 # save layer version information for this build 742 # save layer version information for this build
743 for layer_obj in self.internal_state['lvs']: 743 if not 'lvs' in self.internal_state:
744 self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj]) 744 logger.error("Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass.")
745 else:
746 for layer_obj in self.internal_state['lvs']:
747 self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj])
745 748
746 del self.internal_state['lvs'] 749 del self.internal_state['lvs']
747 750
748 # create target information 751 # create target information
749 target_information = {} 752 target_information = {}
@@ -753,7 +756,8 @@ class BuildInfoHelper(object):
753 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information) 756 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information)
754 757
755 # Save build configuration 758 # Save build configuration
756 self.orm_wrapper.save_build_variables(build_obj, self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]) 759 data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
760 self.orm_wrapper.save_build_variables(build_obj, [])
757 761
758 return self.brbe 762 return self.brbe
759 763
@@ -980,14 +984,29 @@ class BuildInfoHelper(object):
980 984
981 recipe_info = {} 985 recipe_info = {}
982 recipe_info['name'] = pn 986 recipe_info['name'] = pn
983 recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":")
984 recipe_info['layer_version'] = layer_version_obj 987 recipe_info['layer_version'] = layer_version_obj
985 recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] 988
986 recipe_info['license'] = event._depgraph['pn'][pn]['license'] 989 if 'version' in event._depgraph['pn'][pn]:
987 recipe_info['description'] = event._depgraph['pn'][pn]['description'] 990 recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":")
988 recipe_info['section'] = event._depgraph['pn'][pn]['section'] 991
989 recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] 992 if 'summary' in event._depgraph['pn'][pn]:
990 recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] 993 recipe_info['summary'] = event._depgraph['pn'][pn]['summary']
994
995 if 'license' in event._depgraph['pn'][pn]:
996 recipe_info['license'] = event._depgraph['pn'][pn]['license']
997
998 if 'description' in event._depgraph['pn'][pn]:
999 recipe_info['description'] = event._depgraph['pn'][pn]['description']
1000
1001 if 'section' in event._depgraph['pn'][pn]:
1002 recipe_info['section'] = event._depgraph['pn'][pn]['section']
1003
1004 if 'homepage' in event._depgraph['pn'][pn]:
1005 recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage']
1006
1007 if 'bugtracker' in event._depgraph['pn'][pn]:
1008 recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
1009
991 recipe_info['file_path'] = file_name 1010 recipe_info['file_path'] = file_name
992 recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) 1011 recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
993 recipe.is_image = False 1012 recipe.is_image = False
@@ -1146,4 +1165,4 @@ class BuildInfoHelper(object):
1146 1165
1147 if 'backlog' in self.internal_state: 1166 if 'backlog' in self.internal_state:
1148 for event in self.internal_state['backlog']: 1167 for event in self.internal_state['backlog']:
1149 print "NOTE: Unsaved log: ", event.msg 1168 logger.error("Unsaved log: %s", event.msg)
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index 7a316be57c..a85ad5a06a 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -309,7 +309,7 @@ def main(server, eventHandler, params ):
309 try: 309 try:
310 buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data)) 310 buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data))
311 except Exception as ce: 311 except Exception as ce:
312 print("CRITICAL: failed to to save toaster exception to the database: %s" % str(ce)) 312 logger.error("CRITICAL - Failed to to save toaster exception to the database: %s" % str(ce))
313 313
314 pass 314 pass
315 315