summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/ui/buildinfohelper.py
diff options
context:
space:
mode:
authorElliot Smith <elliot.smith@intel.com>2016-03-08 11:32:12 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-03-09 22:45:16 +0000
commit0dcab0258e6e638db8b78fa3c7c7e485280712d4 (patch)
tree1d5fe7bafd6c45b8f935f0f516cc4d0309469eac /bitbake/lib/bb/ui/buildinfohelper.py
parentcc74a8ae26a7728828a3442ba441b1676bc2c8a1 (diff)
downloadpoky-0dcab0258e6e638db8b78fa3c7c7e485280712d4.tar.gz
bitbake: toaster: rework task buildstats storage and display
The data available from buildstats is now more fine grained than previously, so take advantage of that to enrich the data we save against tasks: * Store the CPU usage for user and system separately, and display them separately. * Disk IO is now measured in bytes, not ms. Also store the read/write bytes separately. * Store started and ended times, as well as elapsed_time. This will enable future features such as showing which tasks were running at a particular point in the build. There was also a problem with how we were looking up the Task object, which meant that the buildstats were being added to new tasks which weren't correctly associated with the build. Fix how we look up the Task (only looking for tasks which match the build, and the task and recipe names in the build stats data) so the build stats are associated with the correct task. [YOCTO #8842] (Bitbake rev: efa6f915566b979bdbad233ae195b413cef1b8da) Signed-off-by: Elliot Smith <elliot.smith@intel.com> Signed-off-by: Michael Wood <michael.g.wood@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/ui/buildinfohelper.py')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py72
1 files changed, 33 insertions, 39 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 81abede803..7fedb76911 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -50,6 +50,7 @@ from bb.msg import BBLogFormatter as formatter
50from django.db import models 50from django.db import models
51from pprint import pformat 51from pprint import pformat
52import logging 52import logging
53from datetime import datetime, timedelta
53 54
54from django.db import transaction, connection 55from django.db import transaction, connection
55 56
@@ -120,6 +121,12 @@ class ORMWrapper(object):
120 121
121 return vars(self)[dictname][key] 122 return vars(self)[dictname][key]
122 123
124 def _timestamp_to_datetime(self, secs):
125 """
126 Convert timestamp in seconds to Python datetime
127 """
128 return datetime(1970, 1, 1) + timedelta(seconds=secs)
129
123 # pylint: disable=no-self-use 130 # pylint: disable=no-self-use
124 # we disable detection of no self use in functions because the methods actually work on the object 131 # we disable detection of no self use in functions because the methods actually work on the object
125 # even if they don't touch self anywhere 132 # even if they don't touch self anywhere
@@ -223,6 +230,28 @@ class ORMWrapper(object):
223 target.license_manifest_path = license_manifest_path 230 target.license_manifest_path = license_manifest_path
224 target.save() 231 target.save()
225 232
233 def update_task_object(self, build, task_name, recipe_name, task_stats):
234 """
235 Find the task for build which matches the recipe and task name
236 to be stored
237 """
238 task_to_update = Task.objects.get(
239 build = build,
240 task_name = task_name,
241 recipe__name = recipe_name
242 )
243
244 task_to_update.started = self._timestamp_to_datetime(task_stats['started'])
245 task_to_update.ended = self._timestamp_to_datetime(task_stats['ended'])
246 task_to_update.elapsed_time = (task_stats['ended'] - task_stats['started'])
247 task_to_update.cpu_time_user = task_stats['cpu_time_user']
248 task_to_update.cpu_time_system = task_stats['cpu_time_system']
249 task_to_update.disk_io_read = task_stats['disk_io_read']
250 task_to_update.disk_io_write = task_stats['disk_io_write']
251 task_to_update.disk_io = task_stats['disk_io_read'] + task_stats['disk_io_write']
252
253 task_to_update.save()
254
226 def get_update_task_object(self, task_information, must_exist = False): 255 def get_update_task_object(self, task_information, must_exist = False):
227 assert 'build' in task_information 256 assert 'build' in task_information
228 assert 'recipe' in task_information 257 assert 'recipe' in task_information
@@ -259,14 +288,6 @@ class ORMWrapper(object):
259 task_object.sstate_result = Task.SSTATE_FAILED 288 task_object.sstate_result = Task.SSTATE_FAILED
260 object_changed = True 289 object_changed = True
261 290
262 # mark down duration if we have a start time and a current time
263 if 'start_time' in task_information.keys() and 'end_time' in task_information.keys():
264 duration = task_information['end_time'] - task_information['start_time']
265 task_object.elapsed_time = duration
266 object_changed = True
267 del task_information['start_time']
268 del task_information['end_time']
269
270 if object_changed: 291 if object_changed:
271 task_object.save() 292 task_object.save()
272 return task_object 293 return task_object
@@ -1091,31 +1112,11 @@ class BuildInfoHelper(object):
1091 1112
1092 1113
1093 def store_tasks_stats(self, event): 1114 def store_tasks_stats(self, event):
1094 for (taskfile, taskname, taskstats, recipename) in BuildInfoHelper._get_data_from_event(event): 1115 task_data = BuildInfoHelper._get_data_from_event(event)
1095 localfilepath = taskfile.split(":")[-1]
1096 assert localfilepath.startswith("/")
1097 1116
1098 recipe_information = self._get_recipe_information_from_taskfile(taskfile) 1117 for (task_file, task_name, task_stats, recipe_name) in task_data:
1099 try: 1118 build = self.internal_state['build']
1100 if recipe_information['file_path'].startswith(recipe_information['layer_version'].local_path): 1119 self.orm_wrapper.update_task_object(build, task_name, recipe_name, task_stats)
1101 recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].local_path):].lstrip("/")
1102
1103 recipe_object = Recipe.objects.get(layer_version = recipe_information['layer_version'],
1104 file_path__endswith = recipe_information['file_path'],
1105 name = recipename)
1106 except Recipe.DoesNotExist:
1107 logger.error("Could not find recipe for recipe_information %s name %s" , pformat(recipe_information), recipename)
1108 raise
1109
1110 task_information = {}
1111 task_information['build'] = self.internal_state['build']
1112 task_information['recipe'] = recipe_object
1113 task_information['task_name'] = taskname
1114 task_information['cpu_usage'] = taskstats['cpu_usage']
1115 task_information['disk_io'] = taskstats['disk_io']
1116 if 'elapsed_time' in taskstats:
1117 task_information['elapsed_time'] = taskstats['elapsed_time']
1118 self.orm_wrapper.get_update_task_object(task_information)
1119 1120
1120 def update_and_store_task(self, event): 1121 def update_and_store_task(self, event):
1121 assert 'taskfile' in vars(event) 1122 assert 'taskfile' in vars(event)
@@ -1137,13 +1138,6 @@ class BuildInfoHelper(object):
1137 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True) 1138 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information, True)
1138 task_information = self._get_task_information(event,recipe) 1139 task_information = self._get_task_information(event,recipe)
1139 1140
1140 if 'time' in vars(event):
1141 if not 'start_time' in self.internal_state['taskdata'][identifier]:
1142 self.internal_state['taskdata'][identifier]['start_time'] = event.time
1143 else:
1144 task_information['end_time'] = event.time
1145 task_information['start_time'] = self.internal_state['taskdata'][identifier]['start_time']
1146
1147 task_information['outcome'] = self.internal_state['taskdata'][identifier]['outcome'] 1141 task_information['outcome'] = self.internal_state['taskdata'][identifier]['outcome']
1148 1142
1149 if 'logfile' in vars(event): 1143 if 'logfile' in vars(event):