From dd99cf957da5836dc9b48d200f15a66f0bbce245 Mon Sep 17 00:00:00 2001 From: Elliot Smith Date: Mon, 11 Jul 2016 14:47:06 +0100 Subject: bitbake: toaster: show progress of recipe parsing in recent builds area Modify buildinfohelper and toasterui so that they record the recipe parse progress (from ParseProgress events in bitbake) on the Build object. Note that because the Build object is now created at the point when ParseStarted occurs, it is necessary to set the build name to the empty string initially (hence the migration). The build name can be set when the build properly starts, i.e. at the BuildStarted event. Then use this additional data to determine whether a Build is in a "Parsing" state, and report this in the JSON API. This enables the most recent builds area to show the recipe parse progress. Add additional logic to update the progress bar if the progress for a build object changes. [YOCTO #9631] (Bitbake rev: f33d51d46d70e73e04e325807c1bc4eb68462f7b) Signed-off-by: Elliot Smith Signed-off-by: Richard Purdie --- bitbake/lib/bb/ui/buildinfohelper.py | 118 +++++++++++++-------- bitbake/lib/bb/ui/toasterui.py | 31 ++++-- .../0013_recipe_parse_progress_fields.py | 24 +++++ .../orm/migrations/0014_allow_empty_buildname.py | 19 ++++ bitbake/lib/toaster/orm/models.py | 17 ++- bitbake/lib/toaster/toastergui/api.py | 10 +- .../lib/toaster/toastergui/static/js/mrbsection.js | 50 ++++----- .../toaster/toastergui/templates/mrb_section.html | 112 ++++++++++++------- 8 files changed, 259 insertions(+), 122 deletions(-) create mode 100644 bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py create mode 100644 bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py (limited to 'bitbake') diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index 8cd9371b06..b093b37a5d 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py @@ -150,55 +150,48 @@ class ORMWrapper(object): # pylint: disable=bad-continuation # we do not follow the python conventions for continuation indentation due to long lines here - def create_build_object(self, build_info, brbe, project_id): + def create_build_object(self, build_info, brbe): assert 'machine' in build_info assert 'distro' in build_info assert 'distro_version' in build_info assert 'started_on' in build_info assert 'cooker_log_path' in build_info - assert 'build_name' in build_info assert 'bitbake_version' in build_info prj = None buildrequest = None - if brbe is not None: # this build was triggered by a request from a user + if brbe is not None: + # Toaster-triggered build logger.debug(1, "buildinfohelper: brbe is %s" % brbe) br, _ = brbe.split(":") - buildrequest = BuildRequest.objects.get(pk = br) + buildrequest = BuildRequest.objects.get(pk=br) prj = buildrequest.project - - elif project_id is not None: # this build was triggered by an external system for a specific project - logger.debug(1, "buildinfohelper: project is %s" % prj) - prj = Project.objects.get(pk = project_id) - - else: # this build was triggered by a legacy system, or command line interactive mode + else: + # CLI build prj = Project.objects.get_or_create_default_project() logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) - if buildrequest is not None: build = buildrequest.build logger.info("Updating existing build, with %s", build_info) build.project = prj - build.machine=build_info['machine'] - build.distro=build_info['distro'] - build.distro_version=build_info['distro_version'] - build.cooker_log_path=build_info['cooker_log_path'] - build.build_name=build_info['build_name'] - build.bitbake_version=build_info['bitbake_version'] + build.machine = build_info['machine'] + build.distro = build_info['distro'] + build.distro_version = build_info['distro_version'] + build.cooker_log_path = build_info['cooker_log_path'] + build.bitbake_version = build_info['bitbake_version'] build.save() else: build = Build.objects.create( - project = prj, - machine=build_info['machine'], - distro=build_info['distro'], - distro_version=build_info['distro_version'], - started_on=build_info['started_on'], - completed_on=build_info['started_on'], - cooker_log_path=build_info['cooker_log_path'], - build_name=build_info['build_name'], - bitbake_version=build_info['bitbake_version']) + project=prj, + machine=build_info['machine'], + distro=build_info['distro'], + distro_version=build_info['distro_version'], + started_on=build_info['started_on'], + completed_on=build_info['started_on'], + cooker_log_path=build_info['cooker_log_path'], + bitbake_version=build_info['bitbake_version']) logger.debug(1, "buildinfohelper: build is created %s" % build) @@ -208,6 +201,10 @@ class ORMWrapper(object): return build + def update_build_name(self, build, build_name): + build.build_name = build_name + build.save() + @staticmethod def get_or_create_targets(target_info): """ @@ -924,9 +921,7 @@ class BuildInfoHelper(object): build_info['started_on'] = timezone.now() build_info['completed_on'] = timezone.now() build_info['cooker_log_path'] = build_log_path - build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0] build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] - build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0] return build_info def _get_task_information(self, event, recipe): @@ -1032,17 +1027,28 @@ class BuildInfoHelper(object): except NotExisting as nee: logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee) - - def store_started_build(self, event, build_log_path): - assert '_pkgs' in vars(event) + def store_started_build(self, build_log_path): build_information = self._get_build_information(build_log_path) + self.internal_state['build'] = \ + self.orm_wrapper.create_build_object(build_information, self.brbe) - # Update brbe and project as they can be changed for every build - self.project = build_information['project'] + def save_build_name_and_targets(self, event): + # NB the BUILDNAME variable isn't set until BuildInit (or + # BuildStarted for older bitbakes) + build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0] + self.orm_wrapper.update_build_name(self.internal_state['build'], + build_name) - build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) + # create target information + assert '_pkgs' in vars(event) + target_information = {} + target_information['targets'] = event._pkgs + target_information['build'] = self.internal_state['build'] - self.internal_state['build'] = build_obj + self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) + + def save_build_layers_and_variables(self): + build_obj = self.internal_state['build'] # save layer version information for this build if not 'lvs' in self.internal_state: @@ -1053,13 +1059,6 @@ class BuildInfoHelper(object): del self.internal_state['lvs'] - # create target information - target_information = {} - target_information['targets'] = event._pkgs - target_information['build'] = build_obj - - self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) - # Save build configuration data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] @@ -1090,6 +1089,41 @@ class BuildInfoHelper(object): return self.brbe + def set_recipes_to_parse(self, num_recipes): + """ + Set the number of recipes which need to be parsed for this build. + This is set the first time ParseStarted is received by toasterui. + """ + if self.internal_state['build']: + self.internal_state['build'].recipes_to_parse = num_recipes + self.internal_state['build'].save() + + def set_recipes_parsed(self, num_recipes): + """ + Set the number of recipes parsed so far for this build; this is updated + each time a ParseProgress or ParseCompleted event is received by + toasterui. + """ + if self.internal_state['build']: + if num_recipes <= self.internal_state['build'].recipes_to_parse: + self.internal_state['build'].recipes_parsed = num_recipes + self.internal_state['build'].save() + + def update_target_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + + for t in self.internal_state['targets']: + if t.is_image == True: + output_files = list(evdata.keys()) + for output in output_files: + if t.target in output and 'rootfs' in output and not output.endswith(".manifest"): + self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) + + def update_artifact_image_file(self, event): + evdata = BuildInfoHelper._get_data_from_event(event) + for artifact_path in evdata.keys(): + self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) + def update_build_information(self, event, errors, warnings, taskfailures): if 'build' in self.internal_state: self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index f399a7d316..7d670bef6f 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py @@ -102,6 +102,7 @@ _evt_list = [ "bb.command.CommandExit", "bb.command.CommandFailed", "bb.cooker.CookerExit", + "bb.event.BuildInit", "bb.event.BuildCompleted", "bb.event.BuildStarted", "bb.event.CacheLoadCompleted", @@ -115,6 +116,7 @@ _evt_list = [ "bb.event.NoProvider", "bb.event.ParseCompleted", "bb.event.ParseProgress", + "bb.event.ParseStarted", "bb.event.RecipeParsed", "bb.event.SanityCheck", "bb.event.SanityCheckPassed", @@ -231,19 +233,30 @@ def main(server, eventHandler, params): # pylint: disable=protected-access # the code will look into the protected variables of the event; no easy way around this - # we treat ParseStarted as the first event of toaster-triggered - # builds; that way we get the Build Configuration included in the log - # and any errors that occur before BuildStarted is fired if isinstance(event, bb.event.ParseStarted): if not (build_log and build_log_file_path): build_log, build_log_file_path = _open_build_log(log_dir) + + buildinfohelper.store_started_build(build_log_file_path) + buildinfohelper.set_recipes_to_parse(event.total) continue - if isinstance(event, bb.event.BuildStarted): - if not (build_log and build_log_file_path): - build_log, build_log_file_path = _open_build_log(log_dir) + # create a build object in buildinfohelper from either BuildInit + # (if available) or BuildStarted (for jethro and previous versions) + if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)): + buildinfohelper.save_build_name_and_targets(event) - buildinfohelper.store_started_build(event, build_log_file_path) + # get additional data from BuildStarted + if isinstance(event, bb.event.BuildStarted): + buildinfohelper.save_build_layers_and_variables() + continue + + if isinstance(event, bb.event.ParseProgress): + buildinfohelper.set_recipes_parsed(event.current) + continue + + if isinstance(event, bb.event.ParseCompleted): + buildinfohelper.set_recipes_parsed(event.total) continue if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): @@ -289,10 +302,6 @@ def main(server, eventHandler, params): # timing and error informations from the parsing phase in Toaster if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)): continue - if isinstance(event, bb.event.ParseProgress): - continue - if isinstance(event, bb.event.ParseCompleted): - continue if isinstance(event, bb.event.CacheLoadStarted): continue if isinstance(event, bb.event.CacheLoadProgress): diff --git a/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py b/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py new file mode 100644 index 0000000000..cc5c96d2dd --- /dev/null +++ b/bitbake/lib/toaster/orm/migrations/0013_recipe_parse_progress_fields.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0012_use_release_instead_of_up_branch'), + ] + + operations = [ + migrations.AddField( + model_name='build', + name='recipes_parsed', + field=models.IntegerField(default=0), + ), + migrations.AddField( + model_name='build', + name='recipes_to_parse', + field=models.IntegerField(default=1), + ), + ] diff --git a/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py b/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py new file mode 100644 index 0000000000..4749a14b26 --- /dev/null +++ b/bitbake/lib/toaster/orm/migrations/0014_allow_empty_buildname.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0013_recipe_parse_progress_fields'), + ] + + operations = [ + migrations.AlterField( + model_name='build', + name='build_name', + field=models.CharField(default='', max_length=100), + ), + ] diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py index 2df6d4910a..4641736add 100644 --- a/bitbake/lib/toaster/orm/models.py +++ b/bitbake/lib/toaster/orm/models.py @@ -397,9 +397,15 @@ class Build(models.Model): completed_on = models.DateTimeField() outcome = models.IntegerField(choices=BUILD_OUTCOME, default=IN_PROGRESS) cooker_log_path = models.CharField(max_length=500) - build_name = models.CharField(max_length=100) + build_name = models.CharField(max_length=100, default='') bitbake_version = models.CharField(max_length=50) + # number of recipes to parse for this build + recipes_to_parse = models.IntegerField(default=1) + + # number of recipes parsed so far for this build + recipes_parsed = models.IntegerField(default=0) + @staticmethod def get_recent(project=None): """ @@ -615,6 +621,13 @@ class Build(models.Model): else: return False + def is_parsing(self): + """ + True if the build is still parsing recipes + """ + return self.outcome == Build.IN_PROGRESS and \ + self.recipes_parsed < self.recipes_to_parse + def get_state(self): """ Get the state of the build; one of 'Succeeded', 'Failed', 'In Progress', @@ -628,6 +641,8 @@ class Build(models.Model): return 'Cancelling'; elif self.is_queued(): return 'Queued' + elif self.is_parsing(): + return 'Parsing' else: return self.get_outcome_text() diff --git a/bitbake/lib/toaster/toastergui/api.py b/bitbake/lib/toaster/toastergui/api.py index aa3cbd83b2..b57f670ba3 100644 --- a/bitbake/lib/toaster/toastergui/api.py +++ b/bitbake/lib/toaster/toastergui/api.py @@ -87,7 +87,7 @@ class XhrBuildRequest(View): br.save() except BuildRequest.DoesNotExist: - return error_response('No such build id %s' % i) + return error_response('No such build request id %s' % i) return error_response('ok') @@ -256,6 +256,14 @@ class MostRecentBuildsView(View): build['id'] = build_obj.pk build['dashboard_url'] = dashboard_url + buildrequest_id = None + if hasattr(build_obj, 'buildrequest'): + buildrequest_id = build_obj.buildrequest.pk + build['buildrequest_id'] = buildrequest_id + + build['recipes_parsed_percentage'] = \ + int((build_obj.recipes_parsed / build_obj.recipes_to_parse) * 100) + tasks_complete_percentage = 0 if build_obj.outcome in (Build.SUCCEEDED, Build.FAILED): tasks_complete_percentage = 100 diff --git a/bitbake/lib/toaster/toastergui/static/js/mrbsection.js b/bitbake/lib/toaster/toastergui/static/js/mrbsection.js index d8c3bf7750..e7fbf01731 100644 --- a/bitbake/lib/toaster/toastergui/static/js/mrbsection.js +++ b/bitbake/lib/toaster/toastergui/static/js/mrbsection.js @@ -33,8 +33,8 @@ function mrbSectionInit(ctx){ return buildData[build.id] || {}; } - // returns true if a build's state changed to "Succeeded" or "Failed" - // from some other value + // returns true if a build's state changed to "Succeeded", "Failed" + // or "Cancelled" from some other value function buildFinished(build) { var cached = getCached(build); return cached.state && @@ -49,12 +49,18 @@ function mrbSectionInit(ctx){ return (cached.state !== build.state); } - // returns true if the complete_percentage changed - function progressChanged(build) { + // returns true if the tasks_complete_percentage changed + function tasksProgressChanged(build) { var cached = getCached(build); return (cached.tasks_complete_percentage !== build.tasks_complete_percentage); } + // returns true if the number of recipes parsed/to parse changed + function recipeProgressChanged(build) { + var cached = getCached(build); + return (cached.recipes_parsed_percentage !== build.recipes_parsed_percentage); + } + function refreshMostRecentBuilds(){ libtoaster.getMostRecentBuilds( libtoaster.ctx.mostRecentBuildsUrl, @@ -68,10 +74,6 @@ function mrbSectionInit(ctx){ var colourClass; var elements; - // classes on the parent which signify the build state and affect - // the colour of the container for the build - var buildStateClasses = 'alert-info alert-success alert-danger'; - for (var i = 0; i < data.length; i++) { build = data[i]; @@ -91,31 +93,25 @@ function mrbSectionInit(ctx){ container = $(selector); container.html(html); - - // style the outermost container for this build to reflect - // the new build state (red, green, blue); - // NB class set here should be in buildStateClasses - colourClass = 'alert-info'; - if (build.state == 'Succeeded') { - colourClass = 'alert-success'; - } - else if (build.state == 'Failed') { - colourClass = 'alert-danger'; - } - - elements = $('[data-latest-build-result="' + build.id + '"]'); - elements.removeClass(buildStateClasses); - elements.addClass(colourClass); } - else if (progressChanged(build)) { - // update the progress text + else if (tasksProgressChanged(build)) { + // update the task progress text selector = '#build-pc-done-' + build.id; $(selector).html(build.tasks_complete_percentage); - // update the progress bar + // update the task progress bar selector = '#build-pc-done-bar-' + build.id; $(selector).width(build.tasks_complete_percentage + '%'); } + else if (recipeProgressChanged(build)) { + // update the recipe progress text + selector = '#recipes-parsed-percentage-' + build.id; + $(selector).html(build.recipes_parsed_percentage); + + // update the recipe progress bar + selector = '#recipes-parsed-percentage-bar-' + build.id; + $(selector).width(build.recipes_parsed_percentage + '%'); + } buildData[build.id] = build; } @@ -128,6 +124,6 @@ function mrbSectionInit(ctx){ ); } - window.setInterval(refreshMostRecentBuilds, 1000); + window.setInterval(refreshMostRecentBuilds, 1500); refreshMostRecentBuilds(); } diff --git a/bitbake/lib/toaster/toastergui/templates/mrb_section.html b/bitbake/lib/toaster/toastergui/templates/mrb_section.html index 302b4b0da4..880485d45f 100644 --- a/bitbake/lib/toaster/toastergui/templates/mrb_section.html +++ b/bitbake/lib/toaster/toastergui/templates/mrb_section.html @@ -26,7 +26,9 @@
- {{build.project.name}} + + {{build.project.name}} +
@@ -52,14 +54,18 @@ <%:targets_abbreviated%> - <%else%> + <%else targets_abbreviated !== ''%> <%:targets_abbreviated%> + <%else%> + ...targets not yet available... <%/if%> - <%if state == 'Queued'%> + <%if state == 'Parsing'%> + <%include tmpl='#parsing-recipes-build-template'/%> + <%else state == 'Queued'%> <%include tmpl='#queued-build-template'/%> <%else state == 'Succeeded' || state == 'Failed'%> <%include tmpl='#succeeded-or-failed-build-template'/%> @@ -75,21 +81,38 @@ + + + @@ -110,17 +133,9 @@ <%:tasks_complete_percentage%>% of tasks complete - <%if is_default_project_build%> - - - <%else%> - - - - Cancel - - <%/if%> + + + <%include tmpl='#cancel-template'/%> @@ -162,19 +177,8 @@
Build time: <%:buildtime%> - <%if is_default_project_build%> - - - - <%else%> - - - - Rebuild - - <%/if%> + + <%include tmpl='#rebuild-template'/%>
@@ -187,12 +191,40 @@
- +
+ + + + + + +