summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorElliot Smith <elliot.smith@intel.com>2016-08-04 16:32:33 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-08-11 00:09:27 +0100
commitf6261da9c003a5f571b2cb5c1347fdf8c8d13547 (patch)
tree5732ac1e45cebca0bb2ecf91269531d4e7f6d562 /bitbake/lib
parentf17ab95c799734a3effa61b9c2b5dfd40d81eb7c (diff)
downloadpoky-f6261da9c003a5f571b2cb5c1347fdf8c8d13547.tar.gz
bitbake: toasterui: ensure that the Build object is always available
Many of the methods in toasterui and buildinfohelper rely on the internal state of the buildinfohelper; in particular, they need a Build object to have been created on the buildinfohelper. If the creation of this Build object is tied to an event which may or may not occur, there's no guarantee that it will exist. This then causes assertion errors in those methods. To prevent this from happening, add an _ensure_build() method to buildinfohelper. This ensures that a minimal Build object is always available whenever it is needed, either by retrieving it from the BuildRequest or creating it; it also ensures that the Build object is up to date with whatever data is available on the bitbake server (DISTRO, MACHINE etc.). This method is then called by any other method which relies on a Build object being in the internal state, ensuring that the object is either available, or creating it. (Bitbake rev: 0990b4c73f194ec0be1762e4e48b1a525d8349fb) Signed-off-by: Elliot Smith <elliot.smith@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py196
-rw-r--r--bitbake/lib/bb/ui/toasterui.py9
2 files changed, 122 insertions, 83 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index b093b37a5d..5044324628 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -150,14 +150,7 @@ class ORMWrapper(object):
150 # pylint: disable=bad-continuation 150 # pylint: disable=bad-continuation
151 # we do not follow the python conventions for continuation indentation due to long lines here 151 # we do not follow the python conventions for continuation indentation due to long lines here
152 152
153 def create_build_object(self, build_info, brbe): 153 def get_or_create_build_object(self, brbe):
154 assert 'machine' in build_info
155 assert 'distro' in build_info
156 assert 'distro_version' in build_info
157 assert 'started_on' in build_info
158 assert 'cooker_log_path' in build_info
159 assert 'bitbake_version' in build_info
160
161 prj = None 154 prj = None
162 buildrequest = None 155 buildrequest = None
163 if brbe is not None: 156 if brbe is not None:
@@ -172,26 +165,18 @@ class ORMWrapper(object):
172 logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) 165 logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj)
173 166
174 if buildrequest is not None: 167 if buildrequest is not None:
168 # reuse existing Build object
175 build = buildrequest.build 169 build = buildrequest.build
176 logger.info("Updating existing build, with %s", build_info)
177 build.project = prj 170 build.project = prj
178 build.machine = build_info['machine']
179 build.distro = build_info['distro']
180 build.distro_version = build_info['distro_version']
181 build.cooker_log_path = build_info['cooker_log_path']
182 build.bitbake_version = build_info['bitbake_version']
183 build.save() 171 build.save()
184
185 else: 172 else:
173 # create new Build object
174 now = timezone.now()
186 build = Build.objects.create( 175 build = Build.objects.create(
187 project=prj, 176 project=prj,
188 machine=build_info['machine'], 177 started_on=now,
189 distro=build_info['distro'], 178 completed_on=now,
190 distro_version=build_info['distro_version'], 179 build_name='')
191 started_on=build_info['started_on'],
192 completed_on=build_info['started_on'],
193 cooker_log_path=build_info['cooker_log_path'],
194 bitbake_version=build_info['bitbake_version'])
195 180
196 logger.debug(1, "buildinfohelper: build is created %s" % build) 181 logger.debug(1, "buildinfohelper: build is created %s" % build)
197 182
@@ -201,8 +186,9 @@ class ORMWrapper(object):
201 186
202 return build 187 return build
203 188
204 def update_build_name(self, build, build_name): 189 def update_build(self, build, data_dict):
205 build.build_name = build_name 190 for key in data_dict:
191 setattr(build, key, data_dict[key])
206 build.save() 192 build.save()
207 193
208 @staticmethod 194 @staticmethod
@@ -227,7 +213,7 @@ class ORMWrapper(object):
227 result.append(obj) 213 result.append(obj)
228 return result 214 return result
229 215
230 def update_build_object(self, build, errors, warnings, taskfailures): 216 def update_build_stats_and_outcome(self, build, errors, warnings, taskfailures):
231 assert isinstance(build,Build) 217 assert isinstance(build,Build)
232 assert isinstance(errors, int) 218 assert isinstance(errors, int)
233 assert isinstance(warnings, int) 219 assert isinstance(warnings, int)
@@ -912,20 +898,55 @@ class BuildInfoHelper(object):
912 ################### 898 ###################
913 ## methods to convert event/external info into objects that the ORM layer uses 899 ## methods to convert event/external info into objects that the ORM layer uses
914 900
901 def _ensure_build(self):
902 """
903 Ensure the current build object exists and is up to date with
904 data on the bitbake server
905 """
906 if not 'build' in self.internal_state or not self.internal_state['build']:
907 # create the Build object
908 self.internal_state['build'] = \
909 self.orm_wrapper.get_or_create_build_object(self.brbe)
910
911 build = self.internal_state['build']
915 912
916 def _get_build_information(self, build_log_path): 913 # update missing fields on the Build object with found data
917 build_info = {} 914 build_info = {}
918 build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0] 915
919 build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0] 916 # set to True if at least one field is going to be set
920 build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0] 917 changed = False
921 build_info['started_on'] = timezone.now() 918
922 build_info['completed_on'] = timezone.now() 919 if not build.build_name:
923 build_info['cooker_log_path'] = build_log_path 920 build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
924 build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0] 921
925 return build_info 922 # only reset the build name if the one on the server is actually
923 # a valid value for the build_name field
924 if build_name != None:
925 build_info['build_name'] = build_name
926 changed = True
927
928 if not build.machine:
929 build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
930 changed = True
931
932 if not build.distro:
933 build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
934 changed = True
935
936 if not build.distro_version:
937 build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
938 changed = True
939
940 if not build.bitbake_version:
941 build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
942 changed = True
943
944 if changed:
945 self.orm_wrapper.update_build(self.internal_state['build'], build_info)
926 946
927 def _get_task_information(self, event, recipe): 947 def _get_task_information(self, event, recipe):
928 assert 'taskname' in vars(event) 948 assert 'taskname' in vars(event)
949 self._ensure_build()
929 950
930 task_information = {} 951 task_information = {}
931 task_information['build'] = self.internal_state['build'] 952 task_information['build'] = self.internal_state['build']
@@ -940,8 +961,9 @@ class BuildInfoHelper(object):
940 return task_information 961 return task_information
941 962
942 def _get_layer_version_for_path(self, path): 963 def _get_layer_version_for_path(self, path):
964 self._ensure_build()
965
943 assert path.startswith("/") 966 assert path.startswith("/")
944 assert 'build' in self.internal_state
945 967
946 def _slkey_interactive(layer_version): 968 def _slkey_interactive(layer_version):
947 assert isinstance(layer_version, Layer_Version) 969 assert isinstance(layer_version, Layer_Version)
@@ -985,6 +1007,8 @@ class BuildInfoHelper(object):
985 return recipe_info 1007 return recipe_info
986 1008
987 def _get_path_information(self, task_object): 1009 def _get_path_information(self, task_object):
1010 self._ensure_build()
1011
988 assert isinstance(task_object, Task) 1012 assert isinstance(task_object, Task)
989 build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/" 1013 build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
990 build_stats_path = [] 1014 build_stats_path = []
@@ -1027,17 +1051,18 @@ class BuildInfoHelper(object):
1027 except NotExisting as nee: 1051 except NotExisting as nee:
1028 logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee) 1052 logger.warning("buildinfohelper: cannot identify layer exception:%s ", nee)
1029 1053
1030 def store_started_build(self, build_log_path): 1054 def store_started_build(self):
1031 build_information = self._get_build_information(build_log_path) 1055 self._ensure_build()
1032 self.internal_state['build'] = \ 1056
1033 self.orm_wrapper.create_build_object(build_information, self.brbe) 1057 def save_build_log_file_path(self, build_log_path):
1058 self._ensure_build()
1034 1059
1035 def save_build_name_and_targets(self, event): 1060 if not self.internal_state['build'].cooker_log_path:
1036 # NB the BUILDNAME variable isn't set until BuildInit (or 1061 data_dict = {'cooker_log_path': build_log_path}
1037 # BuildStarted for older bitbakes) 1062 self.orm_wrapper.update_build(self.internal_state['build'], data_dict)
1038 build_name = self.server.runCommand(["getVariable", "BUILDNAME"])[0] 1063
1039 self.orm_wrapper.update_build_name(self.internal_state['build'], 1064 def save_build_targets(self, event):
1040 build_name) 1065 self._ensure_build()
1041 1066
1042 # create target information 1067 # create target information
1043 assert '_pkgs' in vars(event) 1068 assert '_pkgs' in vars(event)
@@ -1048,6 +1073,8 @@ class BuildInfoHelper(object):
1048 self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information) 1073 self.internal_state['targets'] = self.orm_wrapper.get_or_create_targets(target_information)
1049 1074
1050 def save_build_layers_and_variables(self): 1075 def save_build_layers_and_variables(self):
1076 self._ensure_build()
1077
1051 build_obj = self.internal_state['build'] 1078 build_obj = self.internal_state['build']
1052 1079
1053 # save layer version information for this build 1080 # save layer version information for this build
@@ -1094,9 +1121,9 @@ class BuildInfoHelper(object):
1094 Set the number of recipes which need to be parsed for this build. 1121 Set the number of recipes which need to be parsed for this build.
1095 This is set the first time ParseStarted is received by toasterui. 1122 This is set the first time ParseStarted is received by toasterui.
1096 """ 1123 """
1097 if self.internal_state['build']: 1124 self._ensure_build()
1098 self.internal_state['build'].recipes_to_parse = num_recipes 1125 self.internal_state['build'].recipes_to_parse = num_recipes
1099 self.internal_state['build'].save() 1126 self.internal_state['build'].save()
1100 1127
1101 def set_recipes_parsed(self, num_recipes): 1128 def set_recipes_parsed(self, num_recipes):
1102 """ 1129 """
@@ -1104,10 +1131,10 @@ class BuildInfoHelper(object):
1104 each time a ParseProgress or ParseCompleted event is received by 1131 each time a ParseProgress or ParseCompleted event is received by
1105 toasterui. 1132 toasterui.
1106 """ 1133 """
1107 if self.internal_state['build']: 1134 self._ensure_build()
1108 if num_recipes <= self.internal_state['build'].recipes_to_parse: 1135 if num_recipes <= self.internal_state['build'].recipes_to_parse:
1109 self.internal_state['build'].recipes_parsed = num_recipes 1136 self.internal_state['build'].recipes_parsed = num_recipes
1110 self.internal_state['build'].save() 1137 self.internal_state['build'].save()
1111 1138
1112 def update_target_image_file(self, event): 1139 def update_target_image_file(self, event):
1113 evdata = BuildInfoHelper._get_data_from_event(event) 1140 evdata = BuildInfoHelper._get_data_from_event(event)
@@ -1120,13 +1147,17 @@ class BuildInfoHelper(object):
1120 self.orm_wrapper.save_target_image_file_information(t, output, evdata[output]) 1147 self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
1121 1148
1122 def update_artifact_image_file(self, event): 1149 def update_artifact_image_file(self, event):
1150 self._ensure_build()
1123 evdata = BuildInfoHelper._get_data_from_event(event) 1151 evdata = BuildInfoHelper._get_data_from_event(event)
1124 for artifact_path in evdata.keys(): 1152 for artifact_path in evdata.keys():
1125 self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) 1153 self.orm_wrapper.save_artifact_information(
1154 self.internal_state['build'], artifact_path,
1155 evdata[artifact_path])
1126 1156
1127 def update_build_information(self, event, errors, warnings, taskfailures): 1157 def update_build_information(self, event, errors, warnings, taskfailures):
1128 if 'build' in self.internal_state: 1158 self._ensure_build()
1129 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) 1159 self.orm_wrapper.update_build_stats_and_outcome(
1160 self.internal_state['build'], errors, warnings, taskfailures)
1130 1161
1131 def store_started_task(self, event): 1162 def store_started_task(self, event):
1132 assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)) 1163 assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
@@ -1169,6 +1200,7 @@ class BuildInfoHelper(object):
1169 1200
1170 1201
1171 def store_tasks_stats(self, event): 1202 def store_tasks_stats(self, event):
1203 self._ensure_build()
1172 task_data = BuildInfoHelper._get_data_from_event(event) 1204 task_data = BuildInfoHelper._get_data_from_event(event)
1173 1205
1174 for (task_file, task_name, task_stats, recipe_name) in task_data: 1206 for (task_file, task_name, task_stats, recipe_name) in task_data:
@@ -1264,6 +1296,8 @@ class BuildInfoHelper(object):
1264 1296
1265 1297
1266 def store_target_package_data(self, event): 1298 def store_target_package_data(self, event):
1299 self._ensure_build()
1300
1267 # for all image targets 1301 # for all image targets
1268 for target in self.internal_state['targets']: 1302 for target in self.internal_state['targets']:
1269 if target.is_image: 1303 if target.is_image:
@@ -1297,10 +1331,9 @@ class BuildInfoHelper(object):
1297 note that this only gets called for command line builds which are 1331 note that this only gets called for command line builds which are
1298 interrupted, so it doesn't touch any BuildRequest objects 1332 interrupted, so it doesn't touch any BuildRequest objects
1299 """ 1333 """
1300 build = self.internal_state['build'] 1334 self._ensure_build()
1301 if build: 1335 self.internal_state['build'].outcome = Build.CANCELLED
1302 build.outcome = Build.CANCELLED 1336 self.internal_state['build'].save()
1303 build.save()
1304 1337
1305 def store_dependency_information(self, event): 1338 def store_dependency_information(self, event):
1306 assert '_depgraph' in vars(event) 1339 assert '_depgraph' in vars(event)
@@ -1446,6 +1479,8 @@ class BuildInfoHelper(object):
1446 1479
1447 1480
1448 def store_build_package_information(self, event): 1481 def store_build_package_information(self, event):
1482 self._ensure_build()
1483
1449 package_info = BuildInfoHelper._get_data_from_event(event) 1484 package_info = BuildInfoHelper._get_data_from_event(event)
1450 self.orm_wrapper.save_build_package_information( 1485 self.orm_wrapper.save_build_package_information(
1451 self.internal_state['build'], 1486 self.internal_state['build'],
@@ -1461,6 +1496,10 @@ class BuildInfoHelper(object):
1461 1496
1462 def _store_build_done(self, errorcode): 1497 def _store_build_done(self, errorcode):
1463 logger.info("Build exited with errorcode %d", errorcode) 1498 logger.info("Build exited with errorcode %d", errorcode)
1499
1500 if not self.brbe:
1501 return
1502
1464 br_id, be_id = self.brbe.split(":") 1503 br_id, be_id = self.brbe.split(":")
1465 be = BuildEnvironment.objects.get(pk = be_id) 1504 be = BuildEnvironment.objects.get(pk = be_id)
1466 be.lock = BuildEnvironment.LOCK_LOCK 1505 be.lock = BuildEnvironment.LOCK_LOCK
@@ -1482,7 +1521,6 @@ class BuildInfoHelper(object):
1482 br.state = BuildRequest.REQ_FAILED 1521 br.state = BuildRequest.REQ_FAILED
1483 br.save() 1522 br.save()
1484 1523
1485
1486 def store_log_error(self, text): 1524 def store_log_error(self, text):
1487 mockevent = MockEvent() 1525 mockevent = MockEvent()
1488 mockevent.levelno = formatter.ERROR 1526 mockevent.levelno = formatter.ERROR
@@ -1501,24 +1539,22 @@ class BuildInfoHelper(object):
1501 1539
1502 1540
1503 def store_log_event(self, event): 1541 def store_log_event(self, event):
1542 self._ensure_build()
1543
1504 if event.levelno < formatter.WARNING: 1544 if event.levelno < formatter.WARNING:
1505 return 1545 return
1506 1546
1507 if 'args' in vars(event): 1547 if 'args' in vars(event):
1508 event.msg = event.msg % event.args 1548 event.msg = event.msg % event.args
1509 1549
1510 if not 'build' in self.internal_state: 1550 # early return for CLI builds
1511 if self.brbe is None: 1551 if self.brbe is None:
1512 if not 'backlog' in self.internal_state: 1552 if not 'backlog' in self.internal_state:
1513 self.internal_state['backlog'] = [] 1553 self.internal_state['backlog'] = []
1514 self.internal_state['backlog'].append(event) 1554 self.internal_state['backlog'].append(event)
1515 return 1555 return
1516 else: # we're under Toaster control, the build is already created
1517 br, _ = self.brbe.split(":")
1518 buildrequest = BuildRequest.objects.get(pk = br)
1519 self.internal_state['build'] = buildrequest.build
1520 1556
1521 if 'build' in self.internal_state and 'backlog' in self.internal_state: 1557 if 'backlog' in self.internal_state:
1522 # if we have a backlog of events, do our best to save them here 1558 # if we have a backlog of events, do our best to save them here
1523 if len(self.internal_state['backlog']): 1559 if len(self.internal_state['backlog']):
1524 tempevent = self.internal_state['backlog'].pop() 1560 tempevent = self.internal_state['backlog'].pop()
@@ -1847,18 +1883,12 @@ class BuildInfoHelper(object):
1847 sdk_target) 1883 sdk_target)
1848 1884
1849 def close(self, errorcode): 1885 def close(self, errorcode):
1850 if self.brbe is not None: 1886 self._store_build_done(errorcode)
1851 self._store_build_done(errorcode)
1852 1887
1853 if 'backlog' in self.internal_state: 1888 if 'backlog' in self.internal_state:
1854 if 'build' in self.internal_state: 1889 # we save missed events in the database for the current build
1855 # we save missed events in the database for the current build 1890 tempevent = self.internal_state['backlog'].pop()
1856 tempevent = self.internal_state['backlog'].pop() 1891 self.store_log_event(tempevent)
1857 self.store_log_event(tempevent)
1858 else:
1859 # we have no build, and we still have events; something amazingly wrong happend
1860 for event in self.internal_state['backlog']:
1861 logger.error("UNSAVED log: %s", event.msg)
1862 1892
1863 if not connection.features.autocommits_when_autocommit_is_off: 1893 if not connection.features.autocommits_when_autocommit_is_off:
1864 transaction.set_autocommit(True) 1894 transaction.set_autocommit(True)
@@ -1867,3 +1897,7 @@ class BuildInfoHelper(object):
1867 # being incorrectly attached to the previous Toaster-triggered build; 1897 # being incorrectly attached to the previous Toaster-triggered build;
1868 # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021 1898 # see https://bugzilla.yoctoproject.org/show_bug.cgi?id=9021
1869 self.brbe = None 1899 self.brbe = None
1900
1901 # unset the internal Build object to prevent it being reused for the
1902 # next build
1903 self.internal_state['build'] = None
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index 7d670bef6f..c9213cc89e 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -237,14 +237,19 @@ def main(server, eventHandler, params):
237 if not (build_log and build_log_file_path): 237 if not (build_log and build_log_file_path):
238 build_log, build_log_file_path = _open_build_log(log_dir) 238 build_log, build_log_file_path = _open_build_log(log_dir)
239 239
240 buildinfohelper.store_started_build(build_log_file_path) 240 buildinfohelper.store_started_build()
241 buildinfohelper.save_build_log_file_path(build_log_file_path)
241 buildinfohelper.set_recipes_to_parse(event.total) 242 buildinfohelper.set_recipes_to_parse(event.total)
242 continue 243 continue
243 244
244 # create a build object in buildinfohelper from either BuildInit 245 # create a build object in buildinfohelper from either BuildInit
245 # (if available) or BuildStarted (for jethro and previous versions) 246 # (if available) or BuildStarted (for jethro and previous versions)
246 if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)): 247 if isinstance(event, (bb.event.BuildStarted, bb.event.BuildInit)):
247 buildinfohelper.save_build_name_and_targets(event) 248 if not (build_log and build_log_file_path):
249 build_log, build_log_file_path = _open_build_log(log_dir)
250
251 buildinfohelper.save_build_targets(event)
252 buildinfohelper.save_build_log_file_path(build_log_file_path)
248 253
249 # get additional data from BuildStarted 254 # get additional data from BuildStarted
250 if isinstance(event, bb.event.BuildStarted): 255 if isinstance(event, bb.event.BuildStarted):