From dd99cf957da5836dc9b48d200f15a66f0bbce245 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 11 Jul 2016 14:47:06 +0100 Subject: bitbake: toaster: show progress of recipe parsing in recent builds area Modify buildinfohelper and toasterui so that they record the recipe parse progress (from ParseProgress events in bitbake) on the Build object. Note that because the Build object is now created at the point when ParseStarted occurs, it is necessary to set the build name to the empty string initially (hence the migration). The build name can be set when the build properly starts, i.e. at the BuildStarted event. Then use this additional data to determine whether a Build is in a "Parsing" state, and report this in the JSON API. This enables the most recent builds area to show the recipe parse progress. Add additional logic to update the progress bar if the progress for a build object changes. [YOCTO #9631] (Bitbake rev: f33d51d46d70e73e04e325807c1bc4eb68462f7b) Signed-off-by: Elliot Smith Signed-off-by: Richard Purdie --- bitbake/lib/bb/ui/buildinfohelper.py | 118 ++++++++++++++++++++++------------- bitbake/lib/bb/ui/toasterui.py | 31 +++++---- 2 files changed, 96 insertions(+), 53 deletions(-) (limited to 'bitbake/lib/bb/ui') diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 8cd9371b06..b093b37a5d 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -150,55 +150,48 @@ class ORMWrapper(object): # pylint: disable=bad-continuation # we do not follow the python conventions for continuation indentation due to long lines here - def create_build_object(self, build_info, brbe, project_id): + def create_build_object(self, build_info, brbe): assert 'machine' in build_info assert 'distro' in build_info assert 'distro_version' in build_info assert 'started_on' in build_info assert 'cooker_log_path' in build_info - assert 'build_name' in build_info assert 'bitbake_version' in build_info prj = None buildrequest = None - if brbe is not None: # this build was triggered by a request from a user + if brbe is not None: + # Toaster-triggered build logger.debug(1, "buildinfohelper: brbe is %s" % brbe) br, _ = brbe.split(":") - buildrequest = BuildRequest.objects.get(pk = br) + buildrequest = BuildRequest.objects.get(pk=br) prj = buildrequest.project - - elif project_id is not None: # this build was triggered by an external system for a specific project - logger.debug(1, "buildinfohelper: project is %s" % prj) - prj = Project.objects.get(pk = project_id) - - else: # this build was triggered by a legacy system, or command line interactive mode + else: + # CLI build prj = Project.objects.get_or_create_default_project() logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) - if buildrequest is not None: build = buildrequest.build logger.info("Updating existing build, with %s", build_info) build.project = prj - build.machine=build_info['machine'] - build.distro=build_info['distro'] - build.distro_version=build_info['distro_version'] - build.cooker_log_path=build_info['cooker_log_path'] - build.build_name=build_info['build_name'] - build.bitbake_version=build_info['bitbake_version'] + build.machine = build_info['machine'] + build.distro = build_info['distro'] + build.distro_version = build_info['distro_version'] + build.cooker_log_path = build_info['cooker_log_path'] + build.bitbake_version = build_info['bitbake_version'] build.save() else: build = Build.objects.create( - project = prj, - machine=build_info['machine'], - distro=build_info['distro'], - distro_version=build_info['distro_version'], - started_on=build_info['started_on'], - completed_on=build_info['started_on'], - cooker_log_path=build_info['cooker_log_path'], - build_name=build_info['build_name'], - bitbake_version=build_info['bitbake_version']) + project=prj, + machine=build_info['machine'], + distro=build_info['distro'], + distro_version=build_info['distro_version'], + started_on=build_info['started_on'], + completed_on=build_info['started_on'], + cooker_log_path=build_info['cooker_log_path'], + bitbake_version=build_info['bitbake_version']) logger.debug(1, "buildinfohelper: build is created %s" % build) @@ -208,6 +201,10 @@ class ORMWrapper(object): return build + def update_build_name(self, build, build_name): + build.build_name = build_name + build.save() + @staticmethod def get_or_create_targets(target_info): """ @@ -924,9 +921,7 @@ class BuildInfoHelper(object): build_info['started_on'] = timezone.now() build_info['completed_on'] = timezone.now() build_info['cooker_log_path'] = build_log_path - build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] - build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0] return build_info def _get_task_information(self, event, recipe): @@ -1032,17 +1027,28 @@ class BuildInfoHelper(object): except NotExisting as nee: logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee) - - def store_started_build(self, event, build_log_path): - assert '_pkgs' in vars(event) + def store_started_build(self, build_log_path): build_information = self._get_build_information(build_log_path) + self.internal_state['build'] = \ + self.orm_wrapper.create_build_object(build_information, self.brbe) - # Update brbe and project as they can be changed for every build - self.project = build_information['project'] + def save_build_name_and_targets(self, event): + # NB the BUILDNAME variable isn't set until BuildInit (or + # BuildStarted for older bitbakes) + build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0] + self.orm_wrapper.update_build_name(self.internal_state['build'], + build_name) - build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) + # create target information + assert '_pkgs' in vars(event) + target_information = {} + target_information['targets'] = event._pkgs + target_information['build'] = self.internal_state['build'] - self.internal_state['build'] = build_obj + self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) + + def save_build_layers_and_variables(self): + build_obj = self.internal_state['build'] # save layer version information for this build if not 'lvs' in self.internal_state: @@ -1053,13 +1059,6 @@ class BuildInfoHelper(object): del self.internal_state['lvs'] - # create target information - target_information = {} - target_information['targets'] = event._pkgs - target_information['build'] = build_obj - - self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) - # Save build configuration data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] @@ -1090,6 +1089,41 @@ class BuildInfoHelper(object): return self.brbe + def set_recipes_to_parse(self, num_recipes): + """ + Set the number of recipes which need to be parsed for this build. + This is set the first time ParseStarted is received by toasterui. + """ + if self.internal_state['build']: + self.internal_state['build'].recipes_to_parse = num_recipes + self.internal_state['build'].save() + + def set_recipes_parsed(self, num_recipes): + """ + Set the number of recipes parsed so far for this build; this is updated + each time a ParseProgress or ParseCompleted event is received by + toasterui. + """ + if self.internal_state['build']: + if num_recipes <= self.internal_state['build'].recipes_to_parse: + self.internal_state['build'].recipes_parsed = num_recipes + self.internal_state['build'].save() + + def update_target_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + + for t in self.internal_state['targets']: + if t.is_image == True: + output_files = list(evdata.keys()) + for output in output_files: + if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): + self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) + + def update_artifact_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + for artifact_path in evdata.keys(): + self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) + def update_build_information(self, event, errors, warnings, taskfailures): if 'build' in self.internal_state: self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index f399a7d316..7d670bef6f 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py @@ -102,6 +102,7 @@ _evt_list = [ "bb.command.CommandExit", "bb.command.CommandFailed", "bb.cooker.CookerExit", + "bb.event.BuildInit", "bb.event.BuildCompleted", "bb.event.BuildStarted", "bb.event.CacheLoadCompleted", @@ -115,6 +116,7 @@ _evt_list = [ "bb.event.NoProvider", "bb.event.ParseCompleted", "bb.event.ParseProgress", + "bb.event.ParseStarted", "bb.event.RecipeParsed", "bb.event.SanityCheck", "bb.event.SanityCheckPassed", @@ -231,19 +233,30 @@ def main(server, eventHandler, params): # pylint: disable=protected-access # the code will look into the protected variables of the event; no easy way around this - # we treat ParseStarted as the first event of toaster-triggered - # builds; that way we get the Build Configuration included in the log - # and any errors that occur before BuildStarted is fired if isinstance(event, bb.event.ParseStarted): if not (build_log and build_log_file_path): build_log, build_log_file_path = _open_build_log(log_dir) + + buildinfohelper.store_started_build(build_log_file_path) + buildinfohelper.set_recipes_to_parse(event.total) continue - if isinstance(event, bb.event.BuildStarted): - if not (build_log and build_log_file_path): - build_log, build_log_file_path = _open_build_log(log_dir) + # create a build object in buildinfohelper from either BuildInit + # (if available) or BuildStarted (for jethro and previous versions) + if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)): + buildinfohelper.save_build_name_and_targets(event) - buildinfohelper.store_started_build(event, build_log_file_path) + # get additional data from BuildStarted + if isinstance(event, bb.event.BuildStarted): + buildinfohelper.save_build_layers_and_variables() + continue + + if isinstance(event, bb.event.ParseProgress): + buildinfohelper.set_recipes_parsed(event.current) + continue + + if isinstance(event, bb.event.ParseCompleted): + buildinfohelper.set_recipes_parsed(event.total) continue if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): @@ -289,10 +302,6 @@ def main(server, eventHandler, params): # timing and error informations from the parsing phase in Toaster if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)): continue - if isinstance(event, bb.event.ParseProgress): - continue - if isinstance(event, bb.event.ParseCompleted): - continue if isinstance(event, bb.event.CacheLoadStarted): continue if isinstance(event, bb.event.CacheLoadProgress): -- cgit v1.2.3-54-g00ecf