diff options
author | Alexandru DAMIAN <alexandru.damian@intel.com> | 2014-12-09 11:57:38 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2014-12-18 10:24:06 +0000 |
commit | 85a17f86ea2edf24b54aa62bd25e10ff522cb6e7 (patch) | |
tree | 69baa4d959be832b5c096b7d69b0fc2bcb2247b5 /bitbake/lib/bb | |
parent | d086fa3aed34a05d52e73c255ca22379149a64a1 (diff) | |
download | poky-85a17f86ea2edf24b54aa62bd25e10ff522cb6e7.tar.gz |
bitbake: add option to write offline event log file
This patch adds a "-w/--write-log" option to bitbake
that writes an event log file for the current build.
The name of the file is passed as a parameter to the "-w"
argument. If the parameter is the empty string '', the file
name is generated in the form bitbake_eventlog_DATE.json,
where DATE is the current date and time, with second precision.
The "-w" option can also be supplied as the BBEVENTLOG
environment variable.
We add a script, toater-eventreplay, that reads an event
log file and loads the data into a Toaster database, creating
a build entry.
We modify the toasterui to fix minor issues with reading
events from an event log file.
Performance impact is undetectable under no-task executed builds.
(Bitbake rev: 1befb4a783bb7b7b387d4b5ee08830d9516f1ac2)
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r-- | bitbake/lib/bb/cooker.py | 75 | ||||
-rw-r--r-- | bitbake/lib/bb/cookerdata.py | 1 | ||||
-rw-r--r-- | bitbake/lib/bb/ui/buildinfohelper.py | 53 | ||||
-rw-r--r-- | bitbake/lib/bb/ui/toasterui.py | 2 |
4 files changed, 110 insertions, 21 deletions
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py index df9a0cab03..16fd4ad34c 100644 --- a/bitbake/lib/bb/cooker.py +++ b/bitbake/lib/bb/cooker.py | |||
@@ -205,6 +205,75 @@ class BBCooker: | |||
205 | self.data = self.databuilder.data | 205 | self.data = self.databuilder.data |
206 | self.data_hash = self.databuilder.data_hash | 206 | self.data_hash = self.databuilder.data_hash |
207 | 207 | ||
208 | |||
209 | # we log all events to a file if so directed | ||
210 | if self.configuration.writeeventlog: | ||
211 | import json, pickle | ||
212 | DEFAULT_EVENTFILE = self.configuration.writeeventlog | ||
213 | class EventLogWriteHandler(): | ||
214 | |||
215 | class EventWriter(): | ||
216 | def __init__(self, cooker): | ||
217 | self.file_inited = None | ||
218 | self.cooker = cooker | ||
219 | self.event_queue = [] | ||
220 | |||
221 | def init_file(self): | ||
222 | try: | ||
223 | # delete the old log | ||
224 | os.remove(DEFAULT_EVENTFILE) | ||
225 | except: | ||
226 | pass | ||
227 | |||
228 | # write current configuration data | ||
229 | with open(DEFAULT_EVENTFILE, "w") as f: | ||
230 | f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])})) | ||
231 | |||
232 | def write_event(self, event): | ||
233 | with open(DEFAULT_EVENTFILE, "a") as f: | ||
234 | try: | ||
235 | f.write("%s\n" % json.dumps({"class":event.__module__ + "." + event.__class__.__name__, "vars":json.dumps(pickle.dumps(event)) })) | ||
236 | except Exception as e: | ||
237 | import traceback | ||
238 | print(e, traceback.format_exc(e)) | ||
239 | |||
240 | |||
241 | def send(self, event): | ||
242 | event_class = event.__module__ + "." + event.__class__.__name__ | ||
243 | |||
244 | # init on bb.event.BuildStarted | ||
245 | if self.file_inited is None: | ||
246 | if event_class == "bb.event.BuildStarted": | ||
247 | self.init_file() | ||
248 | self.file_inited = True | ||
249 | |||
250 | # write pending events | ||
251 | for e in self.event_queue: | ||
252 | self.write_event(e) | ||
253 | |||
254 | # also write the current event | ||
255 | self.write_event(event) | ||
256 | |||
257 | else: | ||
258 | # queue all events until the file is inited | ||
259 | self.event_queue.append(event) | ||
260 | |||
261 | else: | ||
262 | # we have the file, just write the event | ||
263 | self.write_event(event) | ||
264 | |||
265 | # set our handler's event processor | ||
266 | event = EventWriter(self) # self is the cooker here | ||
267 | |||
268 | |||
269 | # set up cooker features for this mock UI handler | ||
270 | |||
271 | # we need to write the dependency tree in the log | ||
272 | self.featureset.setFeature(CookerFeatures.SEND_DEPENDS_TREE) | ||
273 | # register the log file writer as UI Handler | ||
274 | bb.event.register_UIHhandler(EventLogWriteHandler()) | ||
275 | |||
276 | |||
208 | # | 277 | # |
209 | # Special updated configuration we use for firing events | 278 | # Special updated configuration we use for firing events |
210 | # | 279 | # |
@@ -505,7 +574,7 @@ class BBCooker: | |||
505 | taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False) | 574 | taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, task, False) |
506 | 575 | ||
507 | return runlist, taskdata | 576 | return runlist, taskdata |
508 | 577 | ||
509 | ######## WARNING : this function requires cache_extra to be enabled ######## | 578 | ######## WARNING : this function requires cache_extra to be enabled ######## |
510 | 579 | ||
511 | def generateTaskDepTreeData(self, pkgs_to_build, task): | 580 | def generateTaskDepTreeData(self, pkgs_to_build, task): |
@@ -1550,10 +1619,10 @@ class CookerCollectFiles(object): | |||
1550 | for p in pkgfns: | 1619 | for p in pkgfns: |
1551 | realfn, cls = bb.cache.Cache.virtualfn2realfn(p) | 1620 | realfn, cls = bb.cache.Cache.virtualfn2realfn(p) |
1552 | priorities[p] = self.calc_bbfile_priority(realfn, matched) | 1621 | priorities[p] = self.calc_bbfile_priority(realfn, matched) |
1553 | 1622 | ||
1554 | # Don't show the warning if the BBFILE_PATTERN did match .bbappend files | 1623 | # Don't show the warning if the BBFILE_PATTERN did match .bbappend files |
1555 | unmatched = set() | 1624 | unmatched = set() |
1556 | for _, _, regex, pri in self.bbfile_config_priorities: | 1625 | for _, _, regex, pri in self.bbfile_config_priorities: |
1557 | if not regex in matched: | 1626 | if not regex in matched: |
1558 | unmatched.add(regex) | 1627 | unmatched.add(regex) |
1559 | 1628 | ||
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py index 470d5381ae..2ceed2d867 100644 --- a/bitbake/lib/bb/cookerdata.py +++ b/bitbake/lib/bb/cookerdata.py | |||
@@ -139,6 +139,7 @@ class CookerConfiguration(object): | |||
139 | self.dry_run = False | 139 | self.dry_run = False |
140 | self.tracking = False | 140 | self.tracking = False |
141 | self.interface = [] | 141 | self.interface = [] |
142 | self.writeeventlog = False | ||
142 | 143 | ||
143 | self.env = {} | 144 | self.env = {} |
144 | 145 | ||
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 533f4cef3b..f825b57bea 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py | |||
@@ -556,7 +556,6 @@ class ORMWrapper(object): | |||
556 | assert isinstance(build_obj, Build) | 556 | assert isinstance(build_obj, Build) |
557 | 557 | ||
558 | helptext_objects = [] | 558 | helptext_objects = [] |
559 | |||
560 | for k in vardump: | 559 | for k in vardump: |
561 | desc = vardump[k]['doc'] | 560 | desc = vardump[k]['doc'] |
562 | if desc is None: | 561 | if desc is None: |
@@ -667,9 +666,11 @@ class BuildInfoHelper(object): | |||
667 | if (path.startswith(bl.layer.local_path)): | 666 | if (path.startswith(bl.layer.local_path)): |
668 | return bl | 667 | return bl |
669 | 668 | ||
670 | #TODO: if we get here, we didn't read layers correctly | 669 | #if we get here, we didn't read layers correctly; mockup the new layer |
671 | assert False | 670 | unknown_layer, created = Layer.objects.get_or_create(name="unknown", local_path="/", layer_index_url="") |
672 | return None | 671 | unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build']) |
672 | |||
673 | return unknown_layer_version_obj | ||
673 | 674 | ||
674 | def _get_recipe_information_from_taskfile(self, taskfile): | 675 | def _get_recipe_information_from_taskfile(self, taskfile): |
675 | localfilepath = taskfile.split(":")[-1] | 676 | localfilepath = taskfile.split(":")[-1] |
@@ -732,7 +733,6 @@ class BuildInfoHelper(object): | |||
732 | 733 | ||
733 | def store_started_build(self, event): | 734 | def store_started_build(self, event): |
734 | assert '_pkgs' in vars(event) | 735 | assert '_pkgs' in vars(event) |
735 | assert 'lvs' in self.internal_state, "Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass." | ||
736 | build_information = self._get_build_information() | 736 | build_information = self._get_build_information() |
737 | 737 | ||
738 | build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe) | 738 | build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe) |
@@ -740,10 +740,13 @@ class BuildInfoHelper(object): | |||
740 | self.internal_state['build'] = build_obj | 740 | self.internal_state['build'] = build_obj |
741 | 741 | ||
742 | # save layer version information for this build | 742 | # save layer version information for this build |
743 | for layer_obj in self.internal_state['lvs']: | 743 | if not 'lvs' in self.internal_state: |
744 | self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj]) | 744 | logger.error("Layer version information not found; Check if the bitbake server was configured to inherit toaster.bbclass.") |
745 | else: | ||
746 | for layer_obj in self.internal_state['lvs']: | ||
747 | self.orm_wrapper.get_update_layer_version_object(build_obj, layer_obj, self.internal_state['lvs'][layer_obj]) | ||
745 | 748 | ||
746 | del self.internal_state['lvs'] | 749 | del self.internal_state['lvs'] |
747 | 750 | ||
748 | # create target information | 751 | # create target information |
749 | target_information = {} | 752 | target_information = {} |
@@ -753,7 +756,8 @@ class BuildInfoHelper(object): | |||
753 | self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information) | 756 | self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information) |
754 | 757 | ||
755 | # Save build configuration | 758 | # Save build configuration |
756 | self.orm_wrapper.save_build_variables(build_obj, self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]) | 759 | data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] |
760 | self.orm_wrapper.save_build_variables(build_obj, []) | ||
757 | 761 | ||
758 | return self.brbe | 762 | return self.brbe |
759 | 763 | ||
@@ -980,14 +984,29 @@ class BuildInfoHelper(object): | |||
980 | 984 | ||
981 | recipe_info = {} | 985 | recipe_info = {} |
982 | recipe_info['name'] = pn | 986 | recipe_info['name'] = pn |
983 | recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":") | ||
984 | recipe_info['layer_version'] = layer_version_obj | 987 | recipe_info['layer_version'] = layer_version_obj |
985 | recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] | 988 | |
986 | recipe_info['license'] = event._depgraph['pn'][pn]['license'] | 989 | if 'version' in event._depgraph['pn'][pn]: |
987 | recipe_info['description'] = event._depgraph['pn'][pn]['description'] | 990 | recipe_info['version'] = event._depgraph['pn'][pn]['version'].lstrip(":") |
988 | recipe_info['section'] = event._depgraph['pn'][pn]['section'] | 991 | |
989 | recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] | 992 | if 'summary' in event._depgraph['pn'][pn]: |
990 | recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] | 993 | recipe_info['summary'] = event._depgraph['pn'][pn]['summary'] |
994 | |||
995 | if 'license' in event._depgraph['pn'][pn]: | ||
996 | recipe_info['license'] = event._depgraph['pn'][pn]['license'] | ||
997 | |||
998 | if 'description' in event._depgraph['pn'][pn]: | ||
999 | recipe_info['description'] = event._depgraph['pn'][pn]['description'] | ||
1000 | |||
1001 | if 'section' in event._depgraph['pn'][pn]: | ||
1002 | recipe_info['section'] = event._depgraph['pn'][pn]['section'] | ||
1003 | |||
1004 | if 'homepage' in event._depgraph['pn'][pn]: | ||
1005 | recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage'] | ||
1006 | |||
1007 | if 'bugtracker' in event._depgraph['pn'][pn]: | ||
1008 | recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker'] | ||
1009 | |||
991 | recipe_info['file_path'] = file_name | 1010 | recipe_info['file_path'] = file_name |
992 | recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) | 1011 | recipe = self.orm_wrapper.get_update_recipe_object(recipe_info) |
993 | recipe.is_image = False | 1012 | recipe.is_image = False |
@@ -1146,4 +1165,4 @@ class BuildInfoHelper(object): | |||
1146 | 1165 | ||
1147 | if 'backlog' in self.internal_state: | 1166 | if 'backlog' in self.internal_state: |
1148 | for event in self.internal_state['backlog']: | 1167 | for event in self.internal_state['backlog']: |
1149 | print "NOTE: Unsaved log: ", event.msg | 1168 | logger.error("Unsaved log: %s", event.msg) |
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index 7a316be57c..a85ad5a06a 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py | |||
@@ -309,7 +309,7 @@ def main(server, eventHandler, params ): | |||
309 | try: | 309 | try: |
310 | buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data)) | 310 | buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data)) |
311 | except Exception as ce: | 311 | except Exception as ce: |
312 | print("CRITICAL: failed to to save toaster exception to the database: %s" % str(ce)) | 312 | logger.error("CRITICAL - Failed to to save toaster exception to the database: %s" % str(ce)) |
313 | 313 | ||
314 | pass | 314 | pass |
315 | 315 | ||