summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py142
1 files changed, 105 insertions, 37 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 160cb03743..b1a418bca9 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -43,6 +43,14 @@ class ORMWrapper(object):
43 43
44 44
45 def create_build_object(self, build_info): 45 def create_build_object(self, build_info):
46 assert 'machine' in build_info
47 assert 'image_fstypes' in build_info
48 assert 'distro' in build_info
49 assert 'distro_version' in build_info
50 assert 'started_on' in build_info
51 assert 'cooker_log_path' in build_info
52 assert 'build_name' in build_info
53 assert 'bitbake_version' in build_info
46 54
47 build = Build.objects.create( 55 build = Build.objects.create(
48 machine=build_info['machine'], 56 machine=build_info['machine'],
@@ -50,7 +58,7 @@ class ORMWrapper(object):
50 distro=build_info['distro'], 58 distro=build_info['distro'],
51 distro_version=build_info['distro_version'], 59 distro_version=build_info['distro_version'],
52 started_on=build_info['started_on'], 60 started_on=build_info['started_on'],
53 completed_on=build_info['completed_on'], 61 completed_on=build_info['started_on'],
54 cooker_log_path=build_info['cooker_log_path'], 62 cooker_log_path=build_info['cooker_log_path'],
55 build_name=build_info['build_name'], 63 build_name=build_info['build_name'],
56 bitbake_version=build_info['bitbake_version']) 64 bitbake_version=build_info['bitbake_version'])
@@ -58,6 +66,9 @@ class ORMWrapper(object):
58 return build 66 return build
59 67
60 def create_target_objects(self, target_info): 68 def create_target_objects(self, target_info):
69 assert 'build' in target_info
70 assert 'targets' in target_info
71
61 targets = [] 72 targets = []
62 for tgt_name in target_info['targets']: 73 for tgt_name in target_info['targets']:
63 tgt_object = Target.objects.create( build = target_info['build'], 74 tgt_object = Target.objects.create( build = target_info['build'],
@@ -69,6 +80,9 @@ class ORMWrapper(object):
69 return targets 80 return targets
70 81
71 def update_build_object(self, build, errors, warnings, taskfailures): 82 def update_build_object(self, build, errors, warnings, taskfailures):
83 assert isinstance(build,Build)
84 assert isinstance(errors, int)
85 assert isinstance(warnings, int)
72 86
73 outcome = Build.SUCCEEDED 87 outcome = Build.SUCCEEDED
74 if errors or taskfailures: 88 if errors or taskfailures:
@@ -82,13 +96,20 @@ class ORMWrapper(object):
82 build.save() 96 build.save()
83 97
84 98
85 def get_update_task_object(self, task_information): 99 def get_update_task_object(self, task_information, must_exist = False):
100 assert 'build' in task_information
101 assert 'recipe' in task_information
102 assert 'task_name' in task_information
103
86 task_object, created = Task.objects.get_or_create( 104 task_object, created = Task.objects.get_or_create(
87 build=task_information['build'], 105 build=task_information['build'],
88 recipe=task_information['recipe'], 106 recipe=task_information['recipe'],
89 task_name=task_information['task_name'], 107 task_name=task_information['task_name'],
90 ) 108 )
91 109
110 if must_exist and created:
111 raise Exception("Task object created when expected to exist")
112
92 for v in vars(task_object): 113 for v in vars(task_object):
93 if v in task_information.keys(): 114 if v in task_information.keys():
94 vars(task_object)[v] = task_information[v] 115 vars(task_object)[v] = task_information[v]
@@ -112,6 +133,8 @@ class ORMWrapper(object):
112 133
113 134
114 def get_update_recipe_object(self, recipe_information): 135 def get_update_recipe_object(self, recipe_information):
136 assert 'layer_version' in recipe_information
137 assert 'file_path' in recipe_information
115 138
116 recipe_object, created = Recipe.objects.get_or_create( 139 recipe_object, created = Recipe.objects.get_or_create(
117 layer_version=recipe_information['layer_version'], 140 layer_version=recipe_information['layer_version'],
@@ -126,8 +149,13 @@ class ORMWrapper(object):
126 return recipe_object 149 return recipe_object
127 150
128 def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information): 151 def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information):
152 assert isinstance(build_obj, Build)
153 assert isinstance(layer_obj, Layer)
154 assert 'branch' in layer_version_information
155 assert 'commit' in layer_version_information
156 assert 'priority' in layer_version_information
129 157
130 layer_version_object = Layer_Version.objects.get_or_create( 158 layer_version_object, created = Layer_Version.objects.get_or_create(
131 build = build_obj, 159 build = build_obj,
132 layer = layer_obj, 160 layer = layer_obj,
133 branch = layer_version_information['branch'], 161 branch = layer_version_information['branch'],
@@ -135,22 +163,26 @@ class ORMWrapper(object):
135 priority = layer_version_information['priority'] 163 priority = layer_version_information['priority']
136 ) 164 )
137 165
138 layer_version_object[0].save() 166 return layer_version_object
139
140 return layer_version_object[0]
141 167
142 def get_update_layer_object(self, layer_information): 168 def get_update_layer_object(self, layer_information):
169 assert 'name' in layer_information
170 assert 'local_path' in layer_information
171 assert 'layer_index_url' in layer_information
143 172
144 layer_object = Layer.objects.get_or_create( 173 layer_object, created = Layer.objects.get_or_create(
145 name=layer_information['name'], 174 name=layer_information['name'],
146 local_path=layer_information['local_path'], 175 local_path=layer_information['local_path'],
147 layer_index_url=layer_information['layer_index_url']) 176 layer_index_url=layer_information['layer_index_url'])
148 layer_object[0].save()
149 177
150 return layer_object[0] 178 return layer_object
151 179
152 180
153 def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes): 181 def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes):
182 assert isinstance(build_obj, Build)
183 assert isinstance(target_obj, Target)
184
185 errormsg = ""
154 for p in packagedict: 186 for p in packagedict:
155 searchname = p 187 searchname = p
156 if 'OPKGN' in pkgpnmap[p].keys(): 188 if 'OPKGN' in pkgpnmap[p].keys():
@@ -178,7 +210,7 @@ class ORMWrapper(object):
178 path = targetpath, 210 path = targetpath,
179 size = targetfilesize) 211 size = targetfilesize)
180 except KeyError as e: 212 except KeyError as e:
181 print "Key error, package", p, "key", e 213 errormsg += " stpi: Key error, package %s key %s \n" % ( p, e )
182 214
183 # save disk installed size 215 # save disk installed size
184 packagedict[p]['object'].installed_size = packagedict[p]['size'] 216 packagedict[p]['object'].installed_size = packagedict[p]['size']
@@ -198,8 +230,15 @@ class ORMWrapper(object):
198 dep_type = tdeptype, 230 dep_type = tdeptype,
199 target = target_obj); 231 target = target_obj);
200 232
233 if (len(errormsg) > 0):
234 raise Exception(errormsg)
235
201 236
202 def create_logmessage(self, log_information): 237 def create_logmessage(self, log_information):
238 assert 'build' in log_information
239 assert 'level' in log_information
240 assert 'message' in log_information
241
203 log_object = LogMessage.objects.create( 242 log_object = LogMessage.objects.create(
204 build = log_information['build'], 243 build = log_information['build'],
205 level = log_information['level'], 244 level = log_information['level'],
@@ -213,8 +252,10 @@ class ORMWrapper(object):
213 252
214 253
215 def save_build_package_information(self, build_obj, package_info, recipes): 254 def save_build_package_information(self, build_obj, package_info, recipes):
255 assert isinstance(build_obj, Build)
256
216 # create and save the object 257 # create and save the object
217 pname = package_info['PKG'] 258 pname = package_info['PKG']
218 if 'OPKGN' in package_info.keys(): 259 if 'OPKGN' in package_info.keys():
219 pname = package_info['OPKGN'] 260 pname = package_info['OPKGN']
220 261
@@ -274,6 +315,8 @@ class ORMWrapper(object):
274 return bp_object 315 return bp_object
275 316
276 def save_build_variables(self, build_obj, vardump): 317 def save_build_variables(self, build_obj, vardump):
318 assert isinstance(build_obj, Build)
319
277 for k in vardump: 320 for k in vardump:
278 if not bool(vardump[k]['func']): 321 if not bool(vardump[k]['func']):
279 value = vardump[k]['v']; 322 value = vardump[k]['v'];
@@ -340,7 +383,7 @@ class BuildInfoHelper(object):
340 return build_info 383 return build_info
341 384
342 def _get_task_information(self, event, recipe): 385 def _get_task_information(self, event, recipe):
343 386 assert 'taskname' in vars(event)
344 387
345 task_information = {} 388 task_information = {}
346 task_information['build'] = self.internal_state['build'] 389 task_information['build'] = self.internal_state['build']
@@ -355,7 +398,11 @@ class BuildInfoHelper(object):
355 return task_information 398 return task_information
356 399
357 def _get_layer_version_for_path(self, path): 400 def _get_layer_version_for_path(self, path):
401 assert path.startswith("/")
402 assert 'build' in self.internal_state
403
358 def _slkey(layer_version): 404 def _slkey(layer_version):
405 assert isinstance(layer_version, Layer_Version)
359 return len(layer_version.layer.local_path) 406 return len(layer_version.layer.local_path)
360 407
361 # Heuristics: we always match recipe to the deepest layer path that 408 # Heuristics: we always match recipe to the deepest layer path that
@@ -369,16 +416,17 @@ class BuildInfoHelper(object):
369 return None 416 return None
370 417
371 def _get_recipe_information_from_taskfile(self, taskfile): 418 def _get_recipe_information_from_taskfile(self, taskfile):
372 419 localfilepath = taskfile.split(":")[-1]
373 layer_version_obj = self._get_layer_version_for_path(re.split(':', taskfile)[-1]) 420 layer_version_obj = self._get_layer_version_for_path(localfilepath)
374 421
375 recipe_info = {} 422 recipe_info = {}
376 recipe_info['layer_version'] = layer_version_obj 423 recipe_info['layer_version'] = layer_version_obj
377 recipe_info['file_path'] = re.split(':', taskfile)[-1] 424 recipe_info['file_path'] = taskfile
378 425
379 return recipe_info 426 return recipe_info
380 427
381 def _get_path_information(self, task_object): 428 def _get_path_information(self, task_object):
429 assert isinstance(task_object, Task)
382 build_stats_format = "{tmpdir}/buildstats/{target}-{machine}/{buildname}/{package}/" 430 build_stats_format = "{tmpdir}/buildstats/{target}-{machine}/{buildname}/{package}/"
383 build_stats_path = [] 431 build_stats_path = []
384 432
@@ -410,6 +458,7 @@ class BuildInfoHelper(object):
410 ## external available methods to store information 458 ## external available methods to store information
411 459
412 def store_layer_info(self, event): 460 def store_layer_info(self, event):
461 assert 'data' in vars(event)
413 layerinfos = event.data 462 layerinfos = event.data
414 self.internal_state['lvs'] = {} 463 self.internal_state['lvs'] = {}
415 for layer in layerinfos: 464 for layer in layerinfos:
@@ -417,7 +466,7 @@ class BuildInfoHelper(object):
417 466
418 467
419 def store_started_build(self, event): 468 def store_started_build(self, event):
420 469 assert '_pkgs' in vars(event)
421 build_information = self._get_build_information() 470 build_information = self._get_build_information()
422 471
423 build_obj = self.orm_wrapper.create_build_object(build_information) 472 build_obj = self.orm_wrapper.create_build_object(build_information)
@@ -431,7 +480,7 @@ class BuildInfoHelper(object):
431 480
432 # create target information 481 # create target information
433 target_information = {} 482 target_information = {}
434 target_information['targets'] = event.getPkgs() 483 target_information['targets'] = event._pkgs
435 target_information['build'] = build_obj 484 target_information['build'] = build_obj
436 485
437 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information) 486 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information)
@@ -445,7 +494,12 @@ class BuildInfoHelper(object):
445 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) 494 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
446 495
447 def store_started_task(self, event): 496 def store_started_task(self, event):
448 identifier = event.taskfile.split(":")[-1] + ":" + event.taskname 497 assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
498 assert 'taskfile' in vars(event)
499 localfilepath = event.taskfile.split(":")[-1]
500 assert localfilepath.startswith("/")
501
502 identifier = event.taskfile + ":" + event.taskname
449 503
450 recipe_information = self._get_recipe_information_from_taskfile(event.taskfile) 504 recipe_information = self._get_recipe_information_from_taskfile(event.taskfile)
451 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) 505 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
@@ -454,6 +508,7 @@ class BuildInfoHelper(object):
454 task_information['outcome'] = Task.OUTCOME_NA 508 task_information['outcome'] = Task.OUTCOME_NA
455 509
456 if isinstance(event, bb.runqueue.runQueueTaskSkipped): 510 if isinstance(event, bb.runqueue.runQueueTaskSkipped):
511 assert 'reason' in vars(event)
457 task_information['task_executed'] = False 512 task_information['task_executed'] = False
458 if event.reason == "covered": 513 if event.reason == "covered":
459 task_information['outcome'] = Task.OUTCOME_COVERED 514 task_information['outcome'] = Task.OUTCOME_COVERED
@@ -481,6 +536,9 @@ class BuildInfoHelper(object):
481 536
482 def store_tasks_stats(self, event): 537 def store_tasks_stats(self, event):
483 for (taskfile, taskname, taskstats) in event.data: 538 for (taskfile, taskname, taskstats) in event.data:
539 localfilepath = taskfile.split(":")[-1]
540 assert localfilepath.startswith("/")
541
484 recipe_information = self._get_recipe_information_from_taskfile(taskfile) 542 recipe_information = self._get_recipe_information_from_taskfile(taskfile)
485 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) 543 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
486 544
@@ -490,10 +548,14 @@ class BuildInfoHelper(object):
490 task_information['task_name'] = taskname 548 task_information['task_name'] = taskname
491 task_information['cpu_usage'] = taskstats['cpu_usage'] 549 task_information['cpu_usage'] = taskstats['cpu_usage']
492 task_information['disk_io'] = taskstats['disk_io'] 550 task_information['disk_io'] = taskstats['disk_io']
493 task_obj = self.orm_wrapper.get_update_task_object(task_information) 551 task_obj = self.orm_wrapper.get_update_task_object(task_information, True) # must exist
494 552
495 def update_and_store_task(self, event): 553 def update_and_store_task(self, event):
496 identifier = event.taskfile.split(":")[-1] + ":" + event.taskname 554 assert 'taskfile' in vars(event)
555 localfilepath = event.taskfile.split(":")[-1]
556 assert localfilepath.startswith("/")
557
558 identifier = event.taskfile + ":" + event.taskname
497 assert identifier in self.internal_state['taskdata'] 559 assert identifier in self.internal_state['taskdata']
498 560
499 recipe_information = self._get_recipe_information_from_taskfile(event.taskfile) 561 recipe_information = self._get_recipe_information_from_taskfile(event.taskfile)
@@ -525,10 +587,11 @@ class BuildInfoHelper(object):
525 task_information['outcome'] = Task.OUTCOME_FAILED 587 task_information['outcome'] = Task.OUTCOME_FAILED
526 del self.internal_state['taskdata'][identifier] 588 del self.internal_state['taskdata'][identifier]
527 589
528 self.orm_wrapper.get_update_task_object(task_information) 590 self.orm_wrapper.get_update_task_object(task_information, True) # must exist
529 591
530 592
531 def store_target_package_data(self, event): 593 def store_target_package_data(self, event):
594 assert 'data' in vars(event)
532 # for all image targets 595 # for all image targets
533 for target in self.internal_state['targets']: 596 for target in self.internal_state['targets']:
534 if target.is_image: 597 if target.is_image:
@@ -537,6 +600,13 @@ class BuildInfoHelper(object):
537 self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes']) 600 self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'])
538 601
539 def store_dependency_information(self, event): 602 def store_dependency_information(self, event):
603 assert '_depgraph' in vars(event)
604 assert 'layer-priorities' in event._depgraph
605 assert 'pn' in event._depgraph
606 assert 'tdepends' in event._depgraph
607
608 errormsg = ""
609
540 # save layer version priorities 610 # save layer version priorities
541 if 'layer-priorities' in event._depgraph.keys(): 611 if 'layer-priorities' in event._depgraph.keys():
542 for lv in event._depgraph['layer-priorities']: 612 for lv in event._depgraph['layer-priorities']:
@@ -550,8 +620,8 @@ class BuildInfoHelper(object):
550 self.internal_state['recipes'] = {} 620 self.internal_state['recipes'] = {}
551 for pn in event._depgraph['pn']: 621 for pn in event._depgraph['pn']:
552 622
553 file_name = re.split(':', event._depgraph['pn'][pn]['filename'])[-1] 623 file_name = event._depgraph['pn'][pn]['filename']
554 layer_version_obj = self._get_layer_version_for_path(re.split(':', file_name)[-1]) 624 layer_version_obj = self._get_layer_version_for_path(file_name.split(":")[-1])
555 625
556 assert layer_version_obj is not None 626 assert layer_version_obj is not None
557 627
@@ -579,6 +649,10 @@ class BuildInfoHelper(object):
579 t.save() 649 t.save()
580 self.internal_state['recipes'][pn] = recipe 650 self.internal_state['recipes'][pn] = recipe
581 651
652 # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED
653
654 assume_provided = self.server.runCommand(["getVariable", "ASSUME_PROVIDED"])[0].split()
655
582 # save recipe dependency 656 # save recipe dependency
583 # buildtime 657 # buildtime
584 for recipe in event._depgraph['depends']: 658 for recipe in event._depgraph['depends']:
@@ -588,20 +662,9 @@ class BuildInfoHelper(object):
588 dependency = self.internal_state['recipes'][dep] 662 dependency = self.internal_state['recipes'][dep]
589 Recipe_Dependency.objects.get_or_create( recipe = target, 663 Recipe_Dependency.objects.get_or_create( recipe = target,
590 depends_on = dependency, dep_type = Recipe_Dependency.TYPE_DEPENDS) 664 depends_on = dependency, dep_type = Recipe_Dependency.TYPE_DEPENDS)
591 except KeyError: # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED 665 except KeyError as e:
592 pass 666 if e not in assume_provided and not str(e).startswith("virtual/"):
593 667 errormsg += " stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, e)
594 # runtime
595 for recipe in event._depgraph['rdepends-pn']:
596 try:
597 target = self.internal_state['recipes'][recipe]
598 for dep in event._depgraph['rdepends-pn'][recipe]:
599 dependency = self.internal_state['recipes'][dep]
600 Recipe_Dependency.objects.get_or_create( recipe = target,
601 depends_on = dependency, dep_type = Recipe_Dependency.TYPE_RDEPENDS)
602
603 except KeyError: # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED
604 pass
605 668
606 # save all task information 669 # save all task information
607 def _save_a_task(taskdesc): 670 def _save_a_task(taskdesc):
@@ -632,7 +695,12 @@ class BuildInfoHelper(object):
632 dep = tasks[taskdep] 695 dep = tasks[taskdep]
633 Task_Dependency.objects.get_or_create( task = target, depends_on = dep ) 696 Task_Dependency.objects.get_or_create( task = target, depends_on = dep )
634 697
698 if (len(errormsg) > 0):
699 raise Exception(errormsg)
700
701
635 def store_build_package_information(self, event): 702 def store_build_package_information(self, event):
703 assert 'data' in vars(event)
636 package_info = event.data 704 package_info = event.data
637 self.orm_wrapper.save_build_package_information(self.internal_state['build'], 705 self.orm_wrapper.save_build_package_information(self.internal_state['build'],
638 package_info, 706 package_info,