summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorElliot Smith <elliot.smith@intel.com>2016-07-11 14:47:06 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-08-11 00:09:26 +0100
commitdd99cf957da5836dc9b48d200f15a66f0bbce245 (patch)
treee7b8b8fa6b88520aabd6f17bc41cc8410af66761 /bitbake/lib/bb
parent952ffb3e1f4a00793e0c9c49bc0c8fb8729424c4 (diff)
downloadpoky-dd99cf957da5836dc9b48d200f15a66f0bbce245.tar.gz
bitbake: toaster: show progress of recipe parsing in recent builds area
Modify buildinfohelper and toasterui so that they record the recipe parse progress (from ParseProgress events in bitbake) on the Build object. Note that because the Build object is now created at the point when ParseStarted occurs, it is necessary to set the build name to the empty string initially (hence the migration). The build name can be set when the build properly starts, i.e. at the BuildStarted event. Then use this additional data to determine whether a Build is in a "Parsing" state, and report this in the JSON API. This enables the most recent builds area to show the recipe parse progress. Add additional logic to update the progress bar if the progress for a build object changes. [YOCTO #9631] (Bitbake rev: f33d51d46d70e73e04e325807c1bc4eb68462f7b) Signed-off-by: Elliot Smith <elliot.smith@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py118
-rw-r--r--bitbake/lib/bb/ui/toasterui.py31
2 files changed, 96 insertions, 53 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 8cd9371b06..b093b37a5d 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -150,55 +150,48 @@ class ORMWrapper(object):
150 # pylint: disable=bad-continuation 150 # pylint: disable=bad-continuation
151 # we do not follow the python conventions for continuation indentation due to long lines here 151 # we do not follow the python conventions for continuation indentation due to long lines here
152 152
153 def create_build_object(self, build_info, brbe, project_id): 153 def create_build_object(self, build_info, brbe):
154 assert 'machine' in build_info 154 assert 'machine' in build_info
155 assert 'distro' in build_info 155 assert 'distro' in build_info
156 assert 'distro_version' in build_info 156 assert 'distro_version' in build_info
157 assert 'started_on' in build_info 157 assert 'started_on' in build_info
158 assert 'cooker_log_path' in build_info 158 assert 'cooker_log_path' in build_info
159 assert 'build_name' in build_info
160 assert 'bitbake_version' in build_info 159 assert 'bitbake_version' in build_info
161 160
162 prj = None 161 prj = None
163 buildrequest = None 162 buildrequest = None
164 if brbe is not None: # this build was triggered by a request from a user 163 if brbe is not None:
164 # Toaster-triggered build
165 logger.debug(1, "buildinfohelper: brbe is %s" % brbe) 165 logger.debug(1, "buildinfohelper: brbe is %s" % brbe)
166 br, _ = brbe.split(":") 166 br, _ = brbe.split(":")
167 buildrequest = BuildRequest.objects.get(pk = br) 167 buildrequest = BuildRequest.objects.get(pk=br)
168 prj = buildrequest.project 168 prj = buildrequest.project
169 169 else:
170 elif project_id is not None: # this build was triggered by an external system for a specific project 170 # CLI build
171 logger.debug(1, "buildinfohelper: project is %s" % prj)
172 prj = Project.objects.get(pk = project_id)
173
174 else: # this build was triggered by a legacy system, or command line interactive mode
175 prj = Project.objects.get_or_create_default_project() 171 prj = Project.objects.get_or_create_default_project()
176 logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) 172 logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
177 173
178
179 if buildrequest is not None: 174 if buildrequest is not None:
180 build = buildrequest.build 175 build = buildrequest.build
181 logger.info("Updating existing build, with %s", build_info) 176 logger.info("Updating existing build, with %s", build_info)
182 build.project = prj 177 build.project = prj
183 build.machine=build_info['machine'] 178 build.machine = build_info['machine']
184 build.distro=build_info['distro'] 179 build.distro = build_info['distro']
185 build.distro_version=build_info['distro_version'] 180 build.distro_version = build_info['distro_version']
186 build.cooker_log_path=build_info['cooker_log_path'] 181 build.cooker_log_path = build_info['cooker_log_path']
187 build.build_name=build_info['build_name'] 182 build.bitbake_version = build_info['bitbake_version']
188 build.bitbake_version=build_info['bitbake_version']
189 build.save() 183 build.save()
190 184
191 else: 185 else:
192 build = Build.objects.create( 186 build = Build.objects.create(
193 project = prj, 187 project=prj,
194 machine=build_info['machine'], 188 machine=build_info['machine'],
195 distro=build_info['distro'], 189 distro=build_info['distro'],
196 distro_version=build_info['distro_version'], 190 distro_version=build_info['distro_version'],
197 started_on=build_info['started_on'], 191 started_on=build_info['started_on'],
198 completed_on=build_info['started_on'], 192 completed_on=build_info['started_on'],
199 cooker_log_path=build_info['cooker_log_path'], 193 cooker_log_path=build_info['cooker_log_path'],
200 build_name=build_info['build_name'], 194 bitbake_version=build_info['bitbake_version'])
201 bitbake_version=build_info['bitbake_version'])
202 195
203 logger.debug(1, "buildinfohelper: build is created %s" % build) 196 logger.debug(1, "buildinfohelper: build is created %s" % build)
204 197
@@ -208,6 +201,10 @@ class ORMWrapper(object):
208 201
209 return build 202 return build
210 203
204 def update_build_name(self, build, build_name):
205 build.build_name = build_name
206 build.save()
207
211 @staticmethod 208 @staticmethod
212 def get_or_create_targets(target_info): 209 def get_or_create_targets(target_info):
213 """ 210 """
@@ -924,9 +921,7 @@ class BuildInfoHelper(object):
924 build_info['started_on'] = timezone.now() 921 build_info['started_on'] = timezone.now()
925 build_info['completed_on'] = timezone.now() 922 build_info['completed_on'] = timezone.now()
926 build_info['cooker_log_path'] = build_log_path 923 build_info['cooker_log_path'] = build_log_path
927 build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
928 build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] 924 build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
929 build_info['project'] = self.project = self.server.runCommand(["getVariable", "TOASTER_PROJECT"])[0]
930 return build_info 925 return build_info
931 926
932 def _get_task_information(self, event, recipe): 927 def _get_task_information(self, event, recipe):
@@ -1032,17 +1027,28 @@ class BuildInfoHelper(object):
1032 except NotExisting as nee: 1027 except NotExisting as nee:
1033 logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee) 1028 logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee)
1034 1029
1035 1030 def store_started_build(self, build_log_path):
1036 def store_started_build(self, event, build_log_path):
1037 assert '_pkgs' in vars(event)
1038 build_information = self._get_build_information(build_log_path) 1031 build_information = self._get_build_information(build_log_path)
1032 self.internal_state['build'] = \
1033 self.orm_wrapper.create_build_object(build_information, self.brbe)
1039 1034
1040 # Update brbe and project as they can be changed for every build 1035 def save_build_name_and_targets(self, event):
1041 self.project = build_information['project'] 1036 # NB the BUILDNAME variable isn't set until BuildInit (or
1037 # BuildStarted for older bitbakes)
1038 build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
1039 self.orm_wrapper.update_build_name(self.internal_state['build'],
1040 build_name)
1042 1041
1043 build_obj = self.orm_wrapper.create_build_object(build_information, self.brbe, self.project) 1042 # create target information
1043 assert '_pkgs' in vars(event)
1044 target_information = {}
1045 target_information['targets'] = event._pkgs
1046 target_information['build'] = self.internal_state['build']
1044 1047
1045 self.internal_state['build'] = build_obj 1048 self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
1049
1050 def save_build_layers_and_variables(self):
1051 build_obj = self.internal_state['build']
1046 1052
1047 # save layer version information for this build 1053 # save layer version information for this build
1048 if not 'lvs' in self.internal_state: 1054 if not 'lvs' in self.internal_state:
@@ -1053,13 +1059,6 @@ class BuildInfoHelper(object):
1053 1059
1054 del self.internal_state['lvs'] 1060 del self.internal_state['lvs']
1055 1061
1056 # create target information
1057 target_information = {}
1058 target_information['targets'] = event._pkgs
1059 target_information['build'] = build_obj
1060
1061 self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
1062
1063 # Save build configuration 1062 # Save build configuration
1064 data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0] 1063 data = self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0]
1065 1064
@@ -1090,6 +1089,41 @@ class BuildInfoHelper(object):
1090 1089
1091 return self.brbe 1090 return self.brbe
1092 1091
1092 def set_recipes_to_parse(self, num_recipes):
1093 """
1094 Set the number of recipes which need to be parsed for this build.
1095 This is set the first time ParseStarted is received by toasterui.
1096 """
1097 if self.internal_state['build']:
1098 self.internal_state['build'].recipes_to_parse = num_recipes
1099 self.internal_state['build'].save()
1100
1101 def set_recipes_parsed(self, num_recipes):
1102 """
1103 Set the number of recipes parsed so far for this build; this is updated
1104 each time a ParseProgress or ParseCompleted event is received by
1105 toasterui.
1106 """
1107 if self.internal_state['build']:
1108 if num_recipes <= self.internal_state['build'].recipes_to_parse:
1109 self.internal_state['build'].recipes_parsed = num_recipes
1110 self.internal_state['build'].save()
1111
1112 def update_target_image_file(self, event):
1113 evdata = BuildInfoHelper._get_data_from_event(event)
1114
1115 for t in self.internal_state['targets']:
1116 if t.is_image == True:
1117 output_files = list(evdata.keys())
1118 for output in output_files:
1119 if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
1120 self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
1121
1122 def update_artifact_image_file(self, event):
1123 evdata = BuildInfoHelper._get_data_from_event(event)
1124 for artifact_path in evdata.keys():
1125 self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path])
1126
1093 def update_build_information(self, event, errors, warnings, taskfailures): 1127 def update_build_information(self, event, errors, warnings, taskfailures):
1094 if 'build' in self.internal_state: 1128 if 'build' in self.internal_state:
1095 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) 1129 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index f399a7d316..7d670bef6f 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -102,6 +102,7 @@ _evt_list = [
102 "bb.command.CommandExit", 102 "bb.command.CommandExit",
103 "bb.command.CommandFailed", 103 "bb.command.CommandFailed",
104 "bb.cooker.CookerExit", 104 "bb.cooker.CookerExit",
105 "bb.event.BuildInit",
105 "bb.event.BuildCompleted", 106 "bb.event.BuildCompleted",
106 "bb.event.BuildStarted", 107 "bb.event.BuildStarted",
107 "bb.event.CacheLoadCompleted", 108 "bb.event.CacheLoadCompleted",
@@ -115,6 +116,7 @@ _evt_list = [
115 "bb.event.NoProvider", 116 "bb.event.NoProvider",
116 "bb.event.ParseCompleted", 117 "bb.event.ParseCompleted",
117 "bb.event.ParseProgress", 118 "bb.event.ParseProgress",
119 "bb.event.ParseStarted",
118 "bb.event.RecipeParsed", 120 "bb.event.RecipeParsed",
119 "bb.event.SanityCheck", 121 "bb.event.SanityCheck",
120 "bb.event.SanityCheckPassed", 122 "bb.event.SanityCheckPassed",
@@ -231,19 +233,30 @@ def main(server, eventHandler, params):
231 # pylint: disable=protected-access 233 # pylint: disable=protected-access
232 # the code will look into the protected variables of the event; no easy way around this 234 # the code will look into the protected variables of the event; no easy way around this
233 235
234 # we treat ParseStarted as the first event of toaster-triggered
235 # builds; that way we get the Build Configuration included in the log
236 # and any errors that occur before BuildStarted is fired
237 if isinstance(event, bb.event.ParseStarted): 236 if isinstance(event, bb.event.ParseStarted):
238 if not (build_log and build_log_file_path): 237 if not (build_log and build_log_file_path):
239 build_log, build_log_file_path = _open_build_log(log_dir) 238 build_log, build_log_file_path = _open_build_log(log_dir)
239
240 buildinfohelper.store_started_build(build_log_file_path)
241 buildinfohelper.set_recipes_to_parse(event.total)
240 continue 242 continue
241 243
242 if isinstance(event, bb.event.BuildStarted): 244 # create a build object in buildinfohelper from either BuildInit
243 if not (build_log and build_log_file_path): 245 # (if available) or BuildStarted (for jethro and previous versions)
244 build_log, build_log_file_path = _open_build_log(log_dir) 246 if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)):
247 buildinfohelper.save_build_name_and_targets(event)
245 248
246 buildinfohelper.store_started_build(event, build_log_file_path) 249 # get additional data from BuildStarted
250 if isinstance(event, bb.event.BuildStarted):
251 buildinfohelper.save_build_layers_and_variables()
252 continue
253
254 if isinstance(event, bb.event.ParseProgress):
255 buildinfohelper.set_recipes_parsed(event.current)
256 continue
257
258 if isinstance(event, bb.event.ParseCompleted):
259 buildinfohelper.set_recipes_parsed(event.total)
247 continue 260 continue
248 261
249 if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): 262 if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
@@ -289,10 +302,6 @@ def main(server, eventHandler, params):
289 # timing and error informations from the parsing phase in Toaster 302 # timing and error informations from the parsing phase in Toaster
290 if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)): 303 if isinstance(event, (bb.event.SanityCheckPassed, bb.event.SanityCheck)):
291 continue 304 continue
292 if isinstance(event, bb.event.ParseProgress):
293 continue
294 if isinstance(event, bb.event.ParseCompleted):
295 continue
296 if isinstance(event, bb.event.CacheLoadStarted): 305 if isinstance(event, bb.event.CacheLoadStarted):
297 continue 306 continue
298 if isinstance(event, bb.event.CacheLoadProgress): 307 if isinstance(event, bb.event.CacheLoadProgress):