summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-12-09 11:57:38 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-12-18 10:24:06 +0000
commit85a17f86ea2edf24b54aa62bd25e10ff522cb6e7 (patch)
tree69baa4d959be832b5c096b7d69b0fc2bcb2247b5 /bitbake/lib/bb
parentd086fa3aed34a05d52e73c255ca22379149a64a1 (diff)
downloadpoky-85a17f86ea2edf24b54aa62bd25e10ff522cb6e7.tar.gz
bitbake: add option to write offline event log file
This patch adds a "-w/--write-log" option to bitbake that writes an event log file for the current build. The name of the file is passed as a parameter to the "-w" argument. If the parameter is the empty string '', the file name is generated in the form bitbake_eventlog_DATE.json, where DATE is the current date and time, with second precision. The "-w" option can also be supplied as the BBEVENTLOG environment variable. We add a script, toater-eventreplay, that reads an event log file and loads the data into a Toaster database, creating a build entry. We modify the toasterui to fix minor issues with reading events from an event log file. Performance impact is undetectable under no-task executed builds. (Bitbake rev: 1befb4a783bb7b7b387d4b5ee08830d9516f1ac2) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/cooker.py75
-rw-r--r--bitbake/lib/bb/cookerdata.py1
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py53
-rw-r--r--bitbake/lib/bb/ui/toasterui.py2
4 files changed, 110 insertions, 21 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index df9a0cab03..16fd4ad34c 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -205,6 +205,75 @@ class BBCooker:
205 self.data = self.databuilder.data 205 self.data = self.databuilder.data
206 self.data_hash = self.databuilder.data_hash 206 self.data_hash = self.databuilder.data_hash
207 207
208
209 # we log all events to a file if so directed
210 if self.configuration.writeeventlog:
211 import json, pickle
212 DEFAULT_EVENTFILE = self.configuration.writeeventlog
213 class EventLogWriteHandler():
214
215 class EventWriter():
216 def __init__(self, cooker):
217 self.file_inited = None
218 self.cooker = cooker
219 self.event_queue = []
220
221 def init_file(self):
222 try:
223 # delete the old log
224 os.remove(DEFAULT_EVENTFILE)
225 except:
226 pass
227
228 # write current configuration data
229 with open(DEFAULT_EVENTFILE, "w") as f:
230 f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
231
232 def write_event(self, event):
233 with open(DEFAULT_EVENTFILE, "a") as f:
234 try:
235 f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) }))
236 except Exception as e:
237 import traceback
238 print(e, traceback.format_exc(e))
239
240
241 def send(self, event):
242 event_class = event.__module__ + "." + event.__class__.__name__
243
244 # init on bb.event.BuildStarted
245 if self.file_inited is None:
246 if event_class == "bb.event.BuildStarted":
247 self.init_file()
248 self.file_inited = True
249
250 # write pending events
251 for e in self.event_queue:
252 self.write_event(e)
253
254 # also write the current event
255 self.write_event(event)
256
257 else:
258 # queue all events until the file is inited
259 self.event_queue.append(event)
260
261 else:
262 # we have the file, just write the event
263 self.write_event(event)
264
265 # set our handler's event processor
266 event = EventWriter(self) # self is the cooker here
267
268
269 # set up cooker features for this mock UI handler
270
271 # we need to write the dependency tree in the log
272 self.featureset.setFeature(CookerFeatures.SEND_DEPENDS_TREE)
273 # register the log file writer as UI Handler
274 bb.event.register_UIHhandler(EventLogWriteHandler())
275
276
208 # 277 #
209 # Special updated configuration we use for firing events 278 # Special updated configuration we use for firing events
210 # 279 #
@@ -505,7 +574,7 @@ class BBCooker:
505 taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False) 574 taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False)
506 575
507 return runlist, taskdata 576 return runlist, taskdata
508 577
509 ######## WARNING : this function requires cache_extra to be enabled ######## 578 ######## WARNING : this function requires cache_extra to be enabled ########
510 579
511 def generateTaskDepTreeData(self, pkgs_to_build, task): 580 def generateTaskDepTreeData(self, pkgs_to_build, task):
@@ -1550,10 +1619,10 @@ class CookerCollectFiles(object):
1550 for p in pkgfns: 1619 for p in pkgfns:
1551 realfn, cls = bb.cache.Cache.virtualfn2realfn(p) 1620 realfn, cls = bb.cache.Cache.virtualfn2realfn(p)
1552 priorities[p] = self.calc_bbfile_priority(realfn, matched) 1621 priorities[p] = self.calc_bbfile_priority(realfn, matched)
1553 1622
1554 # Don't show the warning if the BBFILE_PATTERN did match .bbappend files 1623 # Don't show the warning if the BBFILE_PATTERN did match .bbappend files
1555 unmatched = set() 1624 unmatched = set()
1556 for _, _, regex, pri in self.bbfile_config_priorities: 1625 for _, _, regex, pri in self.bbfile_config_priorities:
1557 if not regex in matched: 1626 if not regex in matched:
1558 unmatched.add(regex) 1627 unmatched.add(regex)
1559 1628
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 470d5381ae..2ceed2d867 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -139,6 +139,7 @@ class CookerConfiguration(object):
139 self.dry_run = False 139 self.dry_run = False
140 self.tracking = False 140 self.tracking = False
141 self.interface = [] 141 self.interface = []
142 self.writeeventlog = False
142 143
143 self.env = {} 144 self.env = {}
144 145
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 533f4cef3b..f825b57bea 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -556,7 +556,6 @@ class ORMWrapper(object):
556 assert isinstance(build_obj, Build) 556 assert isinstance(build_obj, Build)
557 557
558 helptext_objects = [] 558 helptext_objects = []
559
560 for k in vardump: 559 for k in vardump:
561 desc = vardump[k]['doc'] 560 desc = vardump[k]['doc']
562 if desc is None: 561 if desc is None:
@@ -667,9 +666,11 @@ class BuildInfoHelper(object):
667 if (path.startswith(bl.layer.local_path)): 666 if (path.startswith(bl.layer.local_path)):
668 return bl 667 return bl
669 668
670 #TODO: if we get here, we didn't read layers correctly 669 #if we get here, we didn't read layers correctly; mockup the new layer
671 assert False 670 unknown_layer, created = Layer.objects.get_or_create(name="unknown", local_path="/", layer_index_url="")
672 return None 671 unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build'])
672
673 return unknown_layer_version_obj
673 674
674 def _get_recipe_information_from_taskfile(self, taskfile): 675 def _get_recipe_information_from_taskfile(self, taskfile):
675 localfilepath = taskfile.split(":")[-1] 676 localfilepath = taskfile.split(":")[-1]
@@ -732,7 +733,6 @@ class BuildInfoHelper(object):
732 733
733 def store_started_build(self, event): 734 def store_started_build(self, event):
734 assert '_pkgs' in vars(event) 735 assert '_pkgs' in vars(event)
735 assert 'lvs' in self.internal_state, "Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass."
736 build_information = self._get_build_information() 736 build_information = self._get_build_information()
737 737
738 build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe) 738 build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe)
@@ -740,10 +740,13 @@ class BuildInfoHelper(object):
740 self.internal_state['build'] = build_obj 740 self.internal_state['build'] = build_obj
741 741
742 # save layer version information for this build 742 # save layer version information for this build
743 for layer_obj in self.internal_state['lvs']: 743 if not 'lvs' in self.internal_state:
744 self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj]) 744 logger.error("Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass.")
745 else:
746 for layer_obj in self.internal_state['lvs']:
747 self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj])
745 748
746 del self.internal_state['lvs'] 749 del self.internal_state['lvs']
747 750
748 # create target information 751 # create target information
749 target_information = {} 752 target_information = {}
@@ -753,7 +756,8 @@ class BuildInfoHelper(object):
753 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information) 756 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information)
754 757
755 # Save build configuration 758 # Save build configuration
756 self.orm_wrapper.save_build_variables(build_obj, self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]) 759 data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
760 self.orm_wrapper.save_build_variables(build_obj, [])
757 761
758 return self.brbe 762 return self.brbe
759 763
@@ -980,14 +984,29 @@ class BuildInfoHelper(object):
980 984
981 recipe_info = {} 985 recipe_info = {}
982 recipe_info['name'] = pn 986 recipe_info['name'] = pn
983 recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":")
984 recipe_info['layer_version'] = layer_version_obj 987 recipe_info['layer_version'] = layer_version_obj
985 recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] 988
986 recipe_info['license'] = event._depgraph['pn'][pn]['license'] 989 if 'version' in event._depgraph['pn'][pn]:
987 recipe_info['description'] = event._depgraph['pn'][pn]['description'] 990 recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":")
988 recipe_info['section'] = event._depgraph['pn'][pn]['section'] 991
989 recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] 992 if 'summary' in event._depgraph['pn'][pn]:
990 recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] 993 recipe_info['summary'] = event._depgraph['pn'][pn]['summary']
994
995 if 'license' in event._depgraph['pn'][pn]:
996 recipe_info['license'] = event._depgraph['pn'][pn]['license']
997
998 if 'description' in event._depgraph['pn'][pn]:
999 recipe_info['description'] = event._depgraph['pn'][pn]['description']
1000
1001 if 'section' in event._depgraph['pn'][pn]:
1002 recipe_info['section'] = event._depgraph['pn'][pn]['section']
1003
1004 if 'homepage' in event._depgraph['pn'][pn]:
1005 recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage']
1006
1007 if 'bugtracker' in event._depgraph['pn'][pn]:
1008 recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
1009
991 recipe_info['file_path'] = file_name 1010 recipe_info['file_path'] = file_name
992 recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) 1011 recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
993 recipe.is_image = False 1012 recipe.is_image = False
@@ -1146,4 +1165,4 @@ class BuildInfoHelper(object):
1146 1165
1147 if 'backlog' in self.internal_state: 1166 if 'backlog' in self.internal_state:
1148 for event in self.internal_state['backlog']: 1167 for event in self.internal_state['backlog']:
1149 print "NOTE: Unsaved log: ", event.msg 1168 logger.error("Unsaved log: %s", event.msg)
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index 7a316be57c..a85ad5a06a 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -309,7 +309,7 @@ def main(server, eventHandler, params ):
309 try: 309 try:
310 buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data)) 310 buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data))
311 except Exception as ce: 311 except Exception as ce:
312 print("CRITICAL: failed to to save toaster exception to the database: %s" % str(ce)) 312 logger.error("CRITICAL - Failed to to save toaster exception to the database: %s" % str(ce))
313 313
314 pass 314 pass
315 315