summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2013-09-26 12:50:50 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-10-18 11:13:49 +0100
commit9a1dce10bdc9254bb38e0e54199f23ae55e209a4 (patch)
tree40bcb13c44718a4f2d61f515e7a28250b1ee165f
parent4e21d092f9bc13e1bdade1e015c57d6ca569639b (diff)
downloadpoky-9a1dce10bdc9254bb38e0e54199f23ae55e209a4.tar.gz
bitbake: toaster: add Toaster UI interface
Adding a new bitbake UI interface named 'toasterui'. 'toasterui' listens for events and data coming from a bitbake server during a run, and records it in a data store using the Toaster object model. Adds a helper class named BuildInfoHelper that reconstructs the state of the bitbake server and saves relevant data to the data store. Code portions contributed by Calin Dragomir <calindragomir@gmail.com>. (Bitbake rev: 62200ff6694b21fbd5abf009a6f47ad93adf5309) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py719
-rw-r--r--bitbake/lib/bb/ui/toasterui.py273
2 files changed, 992 insertions, 0 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
new file mode 100644
index 0000000000..fbb2620fda
--- /dev/null
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -0,0 +1,719 @@
1#
2# BitBake ToasterUI Implementation
3#
4# Copyright (C) 2013 Intel Corporation
5#
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License version 2 as
8# published by the Free Software Foundation.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18
19import datetime
20import sys
21import bb
22import re
23import subprocess
24
25
26os.environ.setdefault("DJANGO_SETTINGS_MODULE", "toaster.toastermain.settings")
27
28import toaster.toastermain.settings as toaster_django_settings
29from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage
30from toaster.orm.models import Target_Package, Build_Package, Variable, Build_File
31from toaster.orm.models import Task_Dependency, Build_Package_Dependency, Target_Package_Dependency, Recipe_Dependency
32from bb.msg import BBLogFormatter as format
33
34class ORMWrapper(object):
35 """ This class creates the dictionaries needed to store information in the database
36 following the format defined by the Django models. It is also used to save this
37 information in the database.
38 """
39
40 def __init__(self):
41 pass
42
43
44 def create_build_object(self, build_info):
45
46 build = Build.objects.create(
47 machine=build_info['machine'],
48 image_fstypes=build_info['image_fstypes'],
49 distro=build_info['distro'],
50 distro_version=build_info['distro_version'],
51 started_on=build_info['started_on'],
52 completed_on=build_info['completed_on'],
53 cooker_log_path=build_info['cooker_log_path'],
54 build_name=build_info['build_name'],
55 bitbake_version=build_info['bitbake_version'])
56
57 return build
58
59 def create_target_objects(self, target_info):
60 targets = []
61 for tgt_name in target_info['targets']:
62 tgt_object = Target.objects.create( build = target_info['build'],
63 target = tgt_name,
64 is_image = False,
65 file_name = "",
66 file_size = 0);
67 targets.append(tgt_object)
68 return targets
69
70 def update_build_object(self, build, errors, warnings, taskfailures):
71
72 outcome = Build.SUCCEEDED
73 if errors or taskfailures:
74 outcome = Build.FAILED
75
76 build.completed_on = datetime.datetime.now()
77 build.errors_no = errors
78 build.warnings_no = warnings
79 build.outcome = outcome
80 build.save()
81
82
83 def get_update_task_object(self, task_information):
84 task_object, created = Task.objects.get_or_create(
85 build=task_information['build'],
86 recipe=task_information['recipe'],
87 task_name=task_information['task_name'],
88 )
89
90 for v in vars(task_object):
91 if v in task_information.keys():
92 vars(task_object)[v] = task_information[v]
93 # if we got covered by a setscene task, we're SSTATE
94 if task_object.outcome == Task.OUTCOME_COVERED and 1 == Task.objects.filter(task_executed=True, build = task_object.build, recipe = task_object.recipe, task_name=task_object.task_name+"_setscene").count():
95 task_object.outcome = Task.OUTCOME_SSTATE
96
97 # mark down duration if we have a start time
98 if 'start_time' in task_information.keys():
99 duration = datetime.datetime.now() - task_information['start_time']
100 task_object.elapsed_time = duration.total_seconds()
101
102 task_object.save()
103 return task_object
104
105
106 def get_update_recipe_object(self, recipe_information):
107
108 recipe_object, created = Recipe.objects.get_or_create(
109 layer_version=recipe_information['layer_version'],
110 file_path=recipe_information['file_path'])
111
112 for v in vars(recipe_object):
113 if v in recipe_information.keys():
114 vars(recipe_object)[v] = recipe_information[v]
115
116 recipe_object.save()
117
118 return recipe_object
119
120 def get_layer_version_object(self, layer_version_information):
121
122 layer_version_object = Layer_Version.objects.get_or_create(
123 layer = layer_version_information['layer'],
124 branch = layer_version_information['branch'],
125 commit = layer_version_information['commit'],
126 priority = layer_version_information['priority']
127 )
128
129 layer_version_object[0].save()
130
131 return layer_version_object[0]
132
133 def get_update_layer_object(self, layer_information):
134
135 layer_object = Layer.objects.get_or_create(
136 name=layer_information['name'],
137 local_path=layer_information['local_path'],
138 layer_index_url=layer_information['layer_index_url'])
139 layer_object[0].save()
140
141 return layer_object[0]
142
143
144 def save_target_package_information(self, target_obj, packagedict, bldpkgs, recipes):
145 for p in packagedict:
146 packagedict[p]['object'] = Target_Package.objects.create( target = target_obj,
147 name = p,
148 size = packagedict[p]['size'])
149 if p in bldpkgs:
150 packagedict[p]['object'].version = bldpkgs[p]['version']
151 packagedict[p]['object'].recipe = recipes[bldpkgs[p]['pn']]
152 packagedict[p]['object'].save()
153
154 for p in packagedict:
155 for (px,deptype) in packagedict[p]['depends']:
156 Target_Package_Dependency.objects.create( package = packagedict[p]['object'],
157 depends_on = packagedict[px]['object'],
158 dep_type = deptype);
159
160
161 def create_logmessage(self, log_information):
162 log_object = LogMessage.objects.create(
163 build = log_information['build'],
164 level = log_information['level'],
165 message = log_information['message'])
166
167 for v in vars(log_object):
168 if v in log_information.keys():
169 vars(log_object)[v] = log_information[v]
170
171 return log_object.save()
172
173
174 def save_build_package_information(self, build_obj, package_info, recipes, files):
175 # create and save the object
176 bp_object = Build_Package.objects.create( build = build_obj,
177 recipe = recipes[package_info['PN']],
178 name = package_info['PKG'],
179 version = package_info['PKGV'],
180 revision = package_info['PKGR'],
181 summary = package_info['SUMMARY'],
182 description = package_info['DESCRIPTION'],
183 size = package_info['PKGSIZE'],
184 section = package_info['SECTION'],
185 license = package_info['LICENSE'],
186 )
187 # save any attached file information
188 if bp_object.name in files.keys():
189 for path, size in files[bp_object.name]:
190 fo = Build_File.objects.create( bpackage = bp_object,
191 path = path,
192 size = size )
193 del files[bp_object.name]
194
195 # save soft dependency information
196 if package_info['RDEPENDS']:
197 for p in bb.utils.explode_deps(package_info['RDEPENDS']):
198 Build_Package_Dependency.objects.get_or_create( package = bp_object,
199 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RDEPENDS)
200 if package_info['RPROVIDES']:
201 for p in bb.utils.explode_deps(package_info['RPROVIDES']):
202 Build_Package_Dependency.objects.get_or_create( package = bp_object,
203 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RPROVIDES)
204 if package_info['RRECOMMENDS']:
205 for p in bb.utils.explode_deps(package_info['RRECOMMENDS']):
206 Build_Package_Dependency.objects.get_or_create( package = bp_object,
207 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RRECOMMENDS)
208 if package_info['RSUGGESTS']:
209 for p in bb.utils.explode_deps(package_info['RSUGGESTS']):
210 Build_Package_Dependency.objects.get_or_create( package = bp_object,
211 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RSUGGESTS)
212 if package_info['RREPLACES']:
213 for p in bb.utils.explode_deps(package_info['RREPLACES']):
214 Build_Package_Dependency.objects.get_or_create( package = bp_object,
215 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RREPLACES)
216 if package_info['RCONFLICTS']:
217 for p in bb.utils.explode_deps(package_info['RCONFLICTS']):
218 Build_Package_Dependency.objects.get_or_create( package = bp_object,
219 depends_on = p, dep_type = Build_Package_Dependency.TYPE_RCONFLICTS)
220
221 return bp_object
222
223 def save_build_variables(self, build_obj, vardump):
224 for k in vardump:
225 if not bool(vardump[k]['func']):
226 Variable.objects.create( build = build_obj,
227 variable_name = k,
228 variable_value = vardump[k]['v'],
229 description = vardump[k]['doc'])
230
231
232class BuildInfoHelper(object):
233 """ This class gathers the build information from the server and sends it
234 towards the ORM wrapper for storing in the database
235 It is instantiated once per build
236 Keeps in memory all data that needs matching before writing it to the database
237 """
238
239 def __init__(self, server, has_build_history = False):
240 self._configure_django()
241 self.internal_state = {}
242 self.task_order = 0
243 self.server = server
244 self.orm_wrapper = ORMWrapper()
245 self.has_build_history = has_build_history
246 self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
247
248 def _configure_django(self):
249 # Add toaster to sys path for importing modules
250 sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'toaster'))
251
252 ###################
253 ## methods to convert event/external info into objects that the ORM layer uses
254
255 def _get_layer_dict(self, layer_path):
256
257 layer_info = {}
258 layer_name = layer_path.split('/')[-1]
259 layer_url = 'http://layers.openembedded.org/layerindex/layer/{layer}/'
260 layer_url_name = self._get_url_map_name(layer_name)
261
262 layer_info['name'] = layer_name
263 layer_info['local_path'] = layer_path
264 layer_info['layer_index_url'] = layer_url.format(layer=layer_url_name)
265
266 return layer_info
267
268 def _get_url_map_name(self, layer_name):
269 """ Some layers have a different name on openembedded.org site,
270 this method returns the correct name to use in the URL
271 """
272
273 url_name = layer_name
274 url_mapping = {'meta': 'openembedded-core'}
275
276 for key in url_mapping.keys():
277 if key == layer_name:
278 url_name = url_mapping[key]
279
280 return url_name
281
282 def _get_layer_information(self):
283
284 layer_info = {}
285
286 return layer_info
287
288 def _get_layer_version_information(self, layer_object):
289
290 layer_version_info = {}
291 layer_version_info['build'] = self.internal_state['build']
292 layer_version_info['layer'] = layer_object
293 layer_version_info['branch'] = self._get_git_branch(layer_object.local_path)
294 layer_version_info['commit'] = self._get_git_revision(layer_object.local_path)
295 layer_version_info['priority'] = 0
296
297 return layer_version_info
298
299
300 def _get_git_branch(self, layer_path):
301 branch = subprocess.Popen("git symbolic-ref HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0]
302 branch = branch.replace('refs/heads/', '').rstrip()
303 return branch
304
305 def _get_git_revision(self, layer_path):
306 revision = subprocess.Popen("git rev-parse HEAD 2>/dev/null ", cwd=layer_path, shell=True, stdout=subprocess.PIPE).communicate()[0].rstrip()
307 return revision
308
309
310 def _get_build_information(self):
311 build_info = {}
312 # Generate an identifier for each new build
313
314 build_info['machine'] = self.server.runCommand(["getVariable", "MACHINE"])[0]
315 build_info['distro'] = self.server.runCommand(["getVariable", "DISTRO"])[0]
316 build_info['distro_version'] = self.server.runCommand(["getVariable", "DISTRO_VERSION"])[0]
317 build_info['started_on'] = datetime.datetime.now()
318 build_info['completed_on'] = datetime.datetime.now()
319 build_info['image_fstypes'] = self._remove_redundant(self.server.runCommand(["getVariable", "IMAGE_FSTYPES"])[0] or "")
320 build_info['cooker_log_path'] = self.server.runCommand(["getVariable", "BB_CONSOLELOG"])[0]
321 build_info['build_name'] = self.server.runCommand(["getVariable", "BUILDNAME"])[0]
322 build_info['bitbake_version'] = self.server.runCommand(["getVariable", "BB_VERSION"])[0]
323
324 return build_info
325
326 def _get_task_information(self, event, recipe):
327
328
329 task_information = {}
330 task_information['build'] = self.internal_state['build']
331 task_information['outcome'] = Task.OUTCOME_NA
332 task_information['recipe'] = recipe
333 task_information['task_name'] = event.taskname
334 try:
335 # some tasks don't come with a hash. and that's ok
336 task_information['sstate_checksum'] = event.taskhash
337 except AttributeError:
338 pass
339 return task_information
340
341 def _get_layer_version_for_path(self, path):
342 def _slkey(layer_version):
343 return len(layer_version.layer.local_path)
344
345 # Heuristics: we always match recipe to the deepest layer path that
346 # we can match to the recipe file path
347 for bl in sorted(self.internal_state['layer_versions'], reverse=True, key=_slkey):
348 if (path.startswith(bl.layer.local_path)):
349 return bl
350
351 #TODO: if we get here, we didn't read layers correctly
352 assert False
353 return None
354
355 def _get_recipe_information_from_build_event(self, event):
356
357 layer_version_obj = self._get_layer_version_for_path(re.split(':', event.taskfile)[-1])
358
359 recipe_info = {}
360 recipe_info['layer_version'] = layer_version_obj
361 recipe_info['file_path'] = re.split(':', event.taskfile)[-1]
362
363 return recipe_info
364
365 def _get_task_build_stats(self, task_object):
366 bs_path = self._get_path_information(task_object)
367 for bp in bs_path: # TODO: split for each target
368 task_build_stats = self._get_build_stats_from_file(bp, task_object.task_name)
369
370 return task_build_stats
371
372 def _get_path_information(self, task_object):
373 build_stats_format = "{tmpdir}/buildstats/{target}-{machine}/{buildname}/{package}/"
374 build_stats_path = []
375
376 for t in self.internal_state['targets']:
377 target = t.target
378 machine = self.internal_state['build'].machine
379 buildname = self.internal_state['build'].build_name
380 package = task_object.recipe.name + "-" + task_object.recipe.version.strip(":")
381
382 build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir, target=target,
383 machine=machine, buildname=buildname,
384 package=package))
385
386 return build_stats_path
387
388 def _get_build_stats_from_file(self, bs_path, task_name):
389
390 task_bs_filename = str(bs_path) + str(task_name)
391 task_bs = open(task_bs_filename, 'r')
392
393 cpu_usage = 0
394 disk_io = 0
395 startio = ''
396 endio = ''
397
398 for line in task_bs.readlines():
399 if line.startswith('CPU usage: '):
400 cpu_usage = line[11:]
401 elif line.startswith('EndTimeIO: '):
402 endio = line[11:]
403 elif line.startswith('StartTimeIO: '):
404 startio = line[13:]
405
406 task_bs.close()
407
408 if startio and endio:
409 disk_io = int(endio.strip('\n ')) - int(startio.strip('\n '))
410
411 if cpu_usage:
412 cpu_usage = float(cpu_usage.strip('% \n'))
413
414 task_build_stats = {'cpu_usage': cpu_usage, 'disk_io': disk_io}
415
416 return task_build_stats
417
418 def _remove_redundant(self, string):
419 ret = []
420 for i in string.split():
421 if i not in ret:
422 ret.append(i)
423 return " ".join(ret)
424
425
426 ################################
427 ## external available methods to store information
428
429 def store_layer_info(self):
430 layers = self.server.runCommand(["getVariable", "BBLAYERS"])[0].strip().split(" ")
431 self.internal_state['layers'] = []
432 for layer_path in { l for l in layers if len(l) }:
433 layer_information = self._get_layer_dict(layer_path)
434 self.internal_state['layers'].append(self.orm_wrapper.get_update_layer_object(layer_information))
435
436 def store_started_build(self, event):
437
438 build_information = self._get_build_information()
439
440 build_obj = self.orm_wrapper.create_build_object(build_information)
441 self.internal_state['build'] = build_obj
442
443 # create target information
444 target_information = {}
445 target_information['targets'] = event.getPkgs()
446 target_information['build'] = build_obj
447
448 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information)
449
450 # Load layer information for the build
451 self.internal_state['layer_versions'] = []
452 for layer_object in self.internal_state['layers']:
453 layer_version_information = self._get_layer_version_information(layer_object)
454 self.internal_state['layer_versions'].append(self.orm_wrapper.get_layer_version_object(layer_version_information))
455
456 del self.internal_state['layers']
457 # Save build configuration
458 self.orm_wrapper.save_build_variables(build_obj, self.server.runCommand(["getAllKeysWithFlags", ["doc", "func"]])[0])
459
460
461 def update_build_information(self, event, errors, warnings, taskfailures):
462 if 'build' in self.internal_state:
463 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
464
465 def store_started_task(self, event):
466 identifier = event.taskfile + event.taskname
467
468 recipe_information = self._get_recipe_information_from_build_event(event)
469 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
470
471 task_information = self._get_task_information(event, recipe)
472 task_information['outcome'] = Task.OUTCOME_NA
473
474 if isinstance(event, bb.runqueue.runQueueTaskSkipped):
475 task_information['task_executed'] = False
476 if event.reason == "covered":
477 task_information['outcome'] = Task.OUTCOME_COVERED
478 if event.reason == "existing":
479 task_information['outcome'] = Task.OUTCOME_EXISTING
480 else:
481 task_information['task_executed'] = True
482
483 self.task_order += 1
484 task_information['order'] = self.task_order
485 task_obj = self.orm_wrapper.get_update_task_object(task_information)
486
487 self.internal_state[identifier] = {'start_time': datetime.datetime.now()}
488
489 def update_and_store_task(self, event):
490 identifier = event.taskfile + event.taskname
491 recipe_information = self._get_recipe_information_from_build_event(event)
492 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
493 task_information = self._get_task_information(event,recipe)
494 try:
495 task_information['start_time'] = self.internal_state[identifier]['start_time']
496 except:
497 pass
498
499 if 'logfile' in vars(event):
500 task_information['logfile'] = event.logfile
501
502 if '_message' in vars(event):
503 task_information['message'] = event._message
504
505 if 'ispython' in vars(event):
506 if event.ispython:
507 task_information['script_type'] = Task.CODING_PYTHON
508 else:
509 task_information['script_type'] = Task.CODING_SHELL
510
511 if isinstance(event, (bb.runqueue.runQueueTaskCompleted, bb.runqueue.sceneQueueTaskCompleted)):
512 task_information['outcome'] = Task.OUTCOME_SUCCESS
513 task_build_stats = self._get_task_build_stats(self.orm_wrapper.get_update_task_object(task_information))
514 task_information['cpu_usage'] = task_build_stats['cpu_usage']
515 task_information['disk_io'] = task_build_stats['disk_io']
516 del self.internal_state[identifier]
517
518 if isinstance(event, bb.runqueue.runQueueTaskFailed):
519 task_information['outcome'] = Task.OUTCOME_FAILED
520 del self.internal_state[identifier]
521
522 self.orm_wrapper.get_update_task_object(task_information)
523
524
525 def read_target_package_dep_data(self, event):
526 # for all targets
527 for target in self.internal_state['targets']:
528 # verify that we have something to read
529 if not target.is_image or not self.has_build_history:
530 print "not collecting package info ", target.is_image, self.has_build_history
531 break
532
533 # TODO this is a temporary replication of the code in buildhistory.bbclass
534 # This MUST be changed to query the actual BUILD_DIR_IMAGE in the target context when
535 # the capability will be implemented in Bitbake
536
537 MACHINE_ARCH, error = self.server.runCommand(['getVariable', 'MACHINE_ARCH'])
538 TCLIBC, error = self.server.runCommand(['getVariable', 'TCLIBC'])
539 BUILDHISTORY_DIR = self.server.runCommand(['getVariable', 'BUILDHISTORY_DIR'])
540 BUILDHISTORY_DIR_IMAGE = "%s/images/%s/%s/%s" % (BUILDHISTORY_DIR, MACHINE_ARCH, TCLIBC, target.target)
541
542 self.internal_state['packages'] = {}
543
544 with open("%s/installed-package-sizes.txt" % BUILDHISTORY_DIR_IMAGE, "r") as fin:
545 for line in fin:
546 line = line.rstrip(";")
547 psize, px = line.split("\t")
548 punit, pname = px.split(" ")
549 self.internal_state['packages'][pname.strip()] = {'size':int(psize)*1024, 'depends' : []}
550
551 with open("%s/depends.dot" % BUILDHISTORY_DIR_IMAGE, "r") as fin:
552 p = re.compile(r' -> ')
553 dot = re.compile(r'.*style=dotted')
554 for line in fin:
555 line = line.rstrip(';')
556 linesplit = p.split(line)
557 if len(linesplit) == 2:
558 pname = linesplit[0].rstrip('"').strip('"')
559 dependsname = linesplit[1].split(" ")[0].strip().strip(";").strip('"').rstrip('"')
560 deptype = Target_Package_Dependency.TYPE_DEPENDS
561 if dot.match(line):
562 deptype = Target_Package_Dependency.TYPE_RECOMMENDS
563 if not pname in self.internal_state['packages']:
564 self.internal_state['packages'][pname] = {'size': 0, 'depends' : []}
565 if not dependsname in self.internal_state['packages']:
566 self.internal_state['packages'][dependsname] = {'size': 0, 'depends' : []}
567 self.internal_state['packages'][pname]['depends'].append((dependsname, deptype))
568
569 self.orm_wrapper.save_target_package_information(target,
570 self.internal_state['packages'],
571 self.internal_state['bldpkgs'], self.internal_state['recipes'])
572
573
574 def store_dependency_information(self, event):
575 # save layer version priorities
576 if 'layer-priorities' in event._depgraph.keys():
577 for lv in event._depgraph['layer-priorities']:
578 (name, path, regexp, priority) = lv
579 layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^
580 assert layer_version_obj is not None
581 layer_version_obj.priority = priority
582 layer_version_obj.save()
583
584 # save build time package information
585 self.internal_state['bldpkgs'] = {}
586 for pkg in event._depgraph['packages']:
587 self.internal_state['bldpkgs'][pkg] = event._depgraph['packages'][pkg]
588
589 # save recipe information
590 self.internal_state['recipes'] = {}
591 for pn in event._depgraph['pn']:
592
593 file_name = re.split(':', event._depgraph['pn'][pn]['filename'])[-1]
594 layer_version_obj = self._get_layer_version_for_path(re.split(':', file_name)[-1])
595
596 assert layer_version_obj is not None
597
598 recipe_info = {}
599 recipe_info['name'] = pn
600 recipe_info['version'] = event._depgraph['pn'][pn]['version']
601 recipe_info['layer_version'] = layer_version_obj
602 recipe_info['summary'] = event._depgraph['pn'][pn]['summary']
603 recipe_info['license'] = event._depgraph['pn'][pn]['license']
604 recipe_info['description'] = event._depgraph['pn'][pn]['description']
605 recipe_info['section'] = event._depgraph['pn'][pn]['section']
606 recipe_info['licensing_info'] = 'Not Available'
607 recipe_info['homepage'] = event._depgraph['pn'][pn]['homepage']
608 recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
609 recipe_info['author'] = 'Not Available'
610 recipe_info['file_path'] = file_name
611 recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
612 if 'inherits' in event._depgraph['pn'][pn].keys():
613 recipe.is_image = True in map(lambda x: x.endswith('image.bbclass'), event._depgraph['pn'][pn]['inherits'])
614 else:
615 recipe.is_image = False
616 if recipe.is_image:
617 for t in self.internal_state['targets']:
618 if pn == t.target:
619 t.is_image = True
620 t.save()
621 self.internal_state['recipes'][pn] = recipe
622
623 # save recipe dependency
624 # buildtime
625 for recipe in event._depgraph['depends']:
626 try:
627 target = self.internal_state['recipes'][recipe]
628 for dep in event._depgraph['depends'][recipe]:
629 dependency = self.internal_state['recipes'][dep]
630 Recipe_Dependency.objects.get_or_create( recipe = target,
631 depends_on = dependency, dep_type = Recipe_Dependency.TYPE_DEPENDS)
632 except KeyError: # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED
633 pass
634
635 # runtime
636 for recipe in event._depgraph['rdepends-pn']:
637 try:
638 target = self.internal_state['recipes'][recipe]
639 for dep in event._depgraph['rdepends-pn'][recipe]:
640 dependency = self.internal_state['recipes'][dep]
641 Recipe_Dependency.objects.get_or_create( recipe = target,
642 depends_on = dependency, dep_type = Recipe_Dependency.TYPE_RDEPENDS)
643
644 except KeyError: # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED
645 pass
646
647 # save all task information
648 def _save_a_task(taskdesc):
649 spec = re.split(r'\.', taskdesc);
650 pn = ".".join(spec[0:-1])
651 taskname = spec[-1]
652 e = event
653 e.taskname = pn
654 recipe = self.internal_state['recipes'][pn]
655 task_info = self._get_task_information(e, recipe)
656 task_info['task_name'] = taskname
657 task_obj = self.orm_wrapper.get_update_task_object(task_info)
658 return task_obj
659
660 for taskdesc in event._depgraph['tdepends']:
661 target = _save_a_task(taskdesc)
662 for taskdesc1 in event._depgraph['tdepends'][taskdesc]:
663 dep = _save_a_task(taskdesc1)
664 Task_Dependency.objects.get_or_create( task = target, depends_on = dep )
665
666 def store_build_package_information(self, event):
667 package_info = event.data
668 self.orm_wrapper.save_build_package_information(self.internal_state['build'],
669 package_info,
670 self.internal_state['recipes'],
671 self.internal_state['package_files'])
672
673
674 def store_package_file_information(self, event):
675 if not 'package_files' in self.internal_state.keys():
676 self.internal_state['package_files'] = {}
677
678 data = event.data
679 self.internal_state['package_files'][data['PKG']] = data['FILES']
680
681 def _store_log_information(self, level, text):
682 log_information = {}
683 log_information['build'] = self.internal_state['build']
684 log_information['level'] = level
685 log_information['message'] = text
686 self.orm_wrapper.create_logmessage(log_information)
687
688 def store_log_info(self, text):
689 self._store_log_information(LogMessage.INFO, text)
690
691 def store_log_warn(self, text):
692 self._store_log_information(LogMessage.WARNING, text)
693
694 def store_log_error(self, text):
695 self._store_log_information(LogMessage.ERROR, text)
696
697 def store_log_event(self, event):
698 # look up license files info from insane.bbclass
699 m = re.match("([^:]*): md5 checksum matched for ([^;]*)", event.msg)
700 if m:
701 (pn, fn) = m.groups()
702 self.internal_state['recipes'][pn].licensing_info = fn
703 self.internal_state['recipes'][pn].save()
704
705 if event.levelno < format.WARNING:
706 return
707 if not 'build' in self.internal_state:
708 return
709 log_information = {}
710 log_information['build'] = self.internal_state['build']
711 if event.levelno >= format.ERROR:
712 log_information['level'] = LogMessage.ERROR
713 elif event.levelno == format.WARNING:
714 log_information['level'] = LogMessage.WARNING
715 log_information['message'] = event.msg
716 log_information['pathname'] = event.pathname
717 log_information['lineno'] = event.lineno
718 self.orm_wrapper.create_logmessage(log_information)
719
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
new file mode 100644
index 0000000000..ab87092e63
--- /dev/null
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -0,0 +1,273 @@
1#
2# BitBake ToasterUI Implementation
3# based on (No)TTY UI Implementation by Richard Purdie
4#
5# Handling output to TTYs or files (no TTY)
6#
7# Copyright (C) 2006-2012 Richard Purdie
8# Copyright (C) 2013 Intel Corporation
9#
10# This program is free software; you can redistribute it and/or modify
11# it under the terms of the GNU General Public License version 2 as
12# published by the Free Software Foundation.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License along
20# with this program; if not, write to the Free Software Foundation, Inc.,
21# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
22
23from __future__ import division
24try:
25 import bb
26except RuntimeError as exc:
27 sys.exit(str(exc))
28
29from bb.ui import uihelper
30from bb.ui.buildinfohelper import BuildInfoHelper
31
32import bb.msg
33import copy
34import fcntl
35import logging
36import os
37import progressbar
38import signal
39import struct
40import sys
41import time
42import xmlrpclib
43
44featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE]
45
46logger = logging.getLogger("BitBake")
47interactive = sys.stdout.isatty()
48
49
50
51def _log_settings_from_server(server):
52 # Get values of variables which control our output
53 includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"])
54 if error:
55 logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error)
56 raise BaseException(error)
57 loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"])
58 if error:
59 logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error)
60 raise BaseException(error)
61 return includelogs, loglines
62
63def main(server, eventHandler, params ):
64
65 includelogs, loglines = _log_settings_from_server(server)
66
67 # verify and warn
68 build_history_enabled = True
69 inheritlist, error = server.runCommand(["getVariable", "INHERIT"])
70 if not "buildhistory" in inheritlist.split(" "):
71 logger.warn("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.")
72 build_history_enabled = False
73
74 helper = uihelper.BBUIHelper()
75
76 console = logging.StreamHandler(sys.stdout)
77 format_str = "%(levelname)s: %(message)s"
78 format = bb.msg.BBLogFormatter(format_str)
79 bb.msg.addDefaultlogFilter(console)
80 console.setFormatter(format)
81 logger.addHandler(console)
82
83 if not params.observe_only:
84 logger.error("ToasterUI can only work in observer mode")
85 return
86
87
88 main.shutdown = 0
89 interrupted = False
90 return_value = 0
91 errors = 0
92 warnings = 0
93 taskfailures = []
94
95 buildinfohelper = BuildInfoHelper(server, build_history_enabled)
96 buildinfohelper.store_layer_info()
97
98
99 while True:
100 try:
101 event = eventHandler.waitEvent(0.25)
102
103 if event is None:
104 if main.shutdown > 0:
105 break
106 continue
107
108 helper.eventHandler(event)
109
110 if isinstance(event, bb.event.BuildStarted):
111 buildinfohelper.store_started_build(event)
112
113 if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)):
114 buildinfohelper.update_and_store_task(event)
115 continue
116
117 if isinstance(event, bb.event.LogExecTTY):
118 logger.warn(event.msg)
119 continue
120
121 if isinstance(event, logging.LogRecord):
122 buildinfohelper.store_log_event(event)
123 if event.levelno >= format.ERROR:
124 errors = errors + 1
125 return_value = 1
126 elif event.levelno == format.WARNING:
127 warnings = warnings + 1
128 # For "normal" logging conditions, don't show note logs from tasks
129 # but do show them if the user has changed the default log level to
130 # include verbose/debug messages
131 if event.taskpid != 0 and event.levelno <= format.NOTE:
132 continue
133
134 logger.handle(event)
135 continue
136
137 if isinstance(event, bb.build.TaskFailed):
138 buildinfohelper.update_and_store_task(event)
139 return_value = 1
140 logfile = event.logfile
141 if logfile and os.path.exists(logfile):
142 bb.error("Logfile of failure stored in: %s" % logfile)
143
144 # these events are unprocessed now, but may be used in the future to log
145 # timing and error informations from the parsing phase in Toaster
146 if isinstance(event, bb.event.ParseStarted):
147 continue
148 if isinstance(event, bb.event.ParseProgress):
149 continue
150 if isinstance(event, bb.event.ParseCompleted):
151 continue
152 if isinstance(event, bb.event.CacheLoadStarted):
153 continue
154 if isinstance(event, bb.event.CacheLoadProgress):
155 continue
156 if isinstance(event, bb.event.CacheLoadCompleted):
157 continue
158 if isinstance(event, bb.event.MultipleProviders):
159 continue
160 if isinstance(event, bb.event.NoProvider):
161 return_value = 1
162 errors = errors + 1
163 if event._runtime:
164 r = "R"
165 else:
166 r = ""
167
168 if event._dependees:
169 text = "Nothing %sPROVIDES '%s' (but %s %sDEPENDS on or otherwise requires it)" % (r, event._item, ", ".join(event._dependees), r)
170 else:
171 text = "Nothing %sPROVIDES '%s'" % (r, event._item)
172
173 logger.error(text)
174 if event._reasons:
175 for reason in event._reasons:
176 logger.error("%s", reason)
177 text += reason
178 buildinfohelper.store_log_error(text)
179 continue
180
181 if isinstance(event, bb.event.ConfigParsed):
182 continue
183 if isinstance(event, bb.event.RecipeParsed):
184 continue
185
186 # end of saved events
187
188 if isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped)):
189 buildinfohelper.store_started_task(event)
190 continue
191
192 if isinstance(event, bb.runqueue.runQueueTaskCompleted):
193 buildinfohelper.update_and_store_task(event)
194 continue
195
196 if isinstance(event, bb.runqueue.runQueueTaskFailed):
197 buildinfohelper.update_and_store_task(event)
198 taskfailures.append(event.taskstring)
199 logger.error("Task %s (%s) failed with exit code '%s'",
200 event.taskid, event.taskstring, event.exitcode)
201 continue
202
203 if isinstance(event, (bb.runqueue.sceneQueueTaskCompleted, bb.runqueue.sceneQueueTaskFailed)):
204 buildinfohelper.update_and_store_task(event)
205 continue
206
207
208 if isinstance(event, (bb.event.TreeDataPreparationStarted, bb.event.TreeDataPreparationCompleted)):
209 continue
210
211 if isinstance(event, (bb.event.BuildCompleted)):
212 buildinfohelper.read_target_package_dep_data(event)
213 buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
214 continue
215
216 if isinstance(event, (bb.command.CommandCompleted,
217 bb.command.CommandFailed,
218 bb.command.CommandExit)):
219
220 buildinfohelper.update_build_information(event, errors, warnings, taskfailures)
221
222 # we start a new build info
223 errors = 0
224 warnings = 0
225 taskfailures = []
226 buildinfohelper = BuildInfoHelper(server, build_history_enabled)
227 buildinfohelper.store_layer_info()
228 continue
229
230 if isinstance(event, bb.event.MetadataEvent):
231 if event.type == "SinglePackageInfo":
232 buildinfohelper.store_build_package_information(event)
233 elif event.type == "PackageFileSize":
234 buildinfohelper.store_package_file_information(event)
235 continue
236
237 # ignore
238 if isinstance(event, (bb.event.BuildBase,
239 bb.event.StampUpdate,
240 bb.event.RecipePreFinalise,
241 bb.runqueue.runQueueEvent,
242 bb.runqueue.runQueueExitWait,
243 bb.event.OperationProgress,
244 bb.command.CommandFailed,
245 bb.command.CommandExit,
246 bb.command.CommandCompleted,
247 bb.cooker.CookerExit)):
248 continue
249
250 if isinstance(event, bb.event.DepTreeGenerated):
251 buildinfohelper.store_dependency_information(event)
252 continue
253
254 logger.error("Unknown event: %s", event)
255
256 except EnvironmentError as ioerror:
257 # ignore interrupted io
258 if ioerror.args[0] == 4:
259 pass
260 except KeyboardInterrupt:
261 main.shutdown = 1
262 pass
263 except Exception as e:
264 logger.error(e)
265 import traceback
266 traceback.print_exc()
267 pass
268
269 if interrupted:
270 if return_value == 0:
271 return_value = 1
272
273 return return_value