summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-02-19 17:26:18 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-03-09 12:23:58 -0700
commit93aa4aba74843aff06434d03c4003a86439678c2 (patch)
treef0a05214610a82d97c900cd34baf139b7720ca16 /bitbake/lib
parent0daa189028fac906f11b7fbb6bf8ed5444667712 (diff)
downloadpoky-93aa4aba74843aff06434d03c4003a86439678c2.tar.gz
bitbake: toasterui: add asserts on expected values
We add assert statements that validate expectations of correct values and context when the data collection code is running. These checks will help pinpointing unexpected data or call flows, reducing debugging time. Also contains a couple of very small fixes discovered through these checks, including a virtual:native conflict with regular tasks. [YOCTO #5553] (Bitbake rev: e2fbd5c6fa6b53514e2cb23d42aa639020d8a475) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py142
1 files changed, 105 insertions, 37 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 160cb03743..b1a418bca9 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -43,6 +43,14 @@ class ORMWrapper(object):
43 43
44 44
45 def create_build_object(self, build_info): 45 def create_build_object(self, build_info):
46 assert 'machine' in build_info
47 assert 'image_fstypes' in build_info
48 assert 'distro' in build_info
49 assert 'distro_version' in build_info
50 assert 'started_on' in build_info
51 assert 'cooker_log_path' in build_info
52 assert 'build_name' in build_info
53 assert 'bitbake_version' in build_info
46 54
47 build = Build.objects.create( 55 build = Build.objects.create(
48 machine=build_info['machine'], 56 machine=build_info['machine'],
@@ -50,7 +58,7 @@ class ORMWrapper(object):
50 distro=build_info['distro'], 58 distro=build_info['distro'],
51 distro_version=build_info['distro_version'], 59 distro_version=build_info['distro_version'],
52 started_on=build_info['started_on'], 60 started_on=build_info['started_on'],
53 completed_on=build_info['completed_on'], 61 completed_on=build_info['started_on'],
54 cooker_log_path=build_info['cooker_log_path'], 62 cooker_log_path=build_info['cooker_log_path'],
55 build_name=build_info['build_name'], 63 build_name=build_info['build_name'],
56 bitbake_version=build_info['bitbake_version']) 64 bitbake_version=build_info['bitbake_version'])
@@ -58,6 +66,9 @@ class ORMWrapper(object):
58 return build 66 return build
59 67
60 def create_target_objects(self, target_info): 68 def create_target_objects(self, target_info):
69 assert 'build' in target_info
70 assert 'targets' in target_info
71
61 targets = [] 72 targets = []
62 for tgt_name in target_info['targets']: 73 for tgt_name in target_info['targets']:
63 tgt_object = Target.objects.create( build = target_info['build'], 74 tgt_object = Target.objects.create( build = target_info['build'],
@@ -69,6 +80,9 @@ class ORMWrapper(object):
69 return targets 80 return targets
70 81
71 def update_build_object(self, build, errors, warnings, taskfailures): 82 def update_build_object(self, build, errors, warnings, taskfailures):
83 assert isinstance(build,Build)
84 assert isinstance(errors, int)
85 assert isinstance(warnings, int)
72 86
73 outcome = Build.SUCCEEDED 87 outcome = Build.SUCCEEDED
74 if errors or taskfailures: 88 if errors or taskfailures:
@@ -82,13 +96,20 @@ class ORMWrapper(object):
82 build.save() 96 build.save()
83 97
84 98
85 def get_update_task_object(self, task_information): 99 def get_update_task_object(self, task_information, must_exist = False):
100 assert 'build' in task_information
101 assert 'recipe' in task_information
102 assert 'task_name' in task_information
103
86 task_object, created = Task.objects.get_or_create( 104 task_object, created = Task.objects.get_or_create(
87 build=task_information['build'], 105 build=task_information['build'],
88 recipe=task_information['recipe'], 106 recipe=task_information['recipe'],
89 task_name=task_information['task_name'], 107 task_name=task_information['task_name'],
90 ) 108 )
91 109
110 if must_exist and created:
111 raise Exception("Task object created when expected to exist")
112
92 for v in vars(task_object): 113 for v in vars(task_object):
93 if v in task_information.keys(): 114 if v in task_information.keys():
94 vars(task_object)[v] = task_information[v] 115 vars(task_object)[v] = task_information[v]
@@ -112,6 +133,8 @@ class ORMWrapper(object):
112 133
113 134
114 def get_update_recipe_object(self, recipe_information): 135 def get_update_recipe_object(self, recipe_information):
136 assert 'layer_version' in recipe_information
137 assert 'file_path' in recipe_information
115 138
116 recipe_object, created = Recipe.objects.get_or_create( 139 recipe_object, created = Recipe.objects.get_or_create(
117 layer_version=recipe_information['layer_version'], 140 layer_version=recipe_information['layer_version'],
@@ -126,8 +149,13 @@ class ORMWrapper(object):
126 return recipe_object 149 return recipe_object
127 150
128 def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information): 151 def get_update_layer_version_object(self, build_obj, layer_obj, layer_version_information):
152 assert isinstance(build_obj, Build)
153 assert isinstance(layer_obj, Layer)
154 assert 'branch' in layer_version_information
155 assert 'commit' in layer_version_information
156 assert 'priority' in layer_version_information
129 157
130 layer_version_object = Layer_Version.objects.get_or_create( 158 layer_version_object, created = Layer_Version.objects.get_or_create(
131 build = build_obj, 159 build = build_obj,
132 layer = layer_obj, 160 layer = layer_obj,
133 branch = layer_version_information['branch'], 161 branch = layer_version_information['branch'],
@@ -135,22 +163,26 @@ class ORMWrapper(object):
135 priority = layer_version_information['priority'] 163 priority = layer_version_information['priority']
136 ) 164 )
137 165
138 layer_version_object[0].save() 166 return layer_version_object
139
140 return layer_version_object[0]
141 167
142 def get_update_layer_object(self, layer_information): 168 def get_update_layer_object(self, layer_information):
169 assert 'name' in layer_information
170 assert 'local_path' in layer_information
171 assert 'layer_index_url' in layer_information
143 172
144 layer_object = Layer.objects.get_or_create( 173 layer_object, created = Layer.objects.get_or_create(
145 name=layer_information['name'], 174 name=layer_information['name'],
146 local_path=layer_information['local_path'], 175 local_path=layer_information['local_path'],
147 layer_index_url=layer_information['layer_index_url']) 176 layer_index_url=layer_information['layer_index_url'])
148 layer_object[0].save()
149 177
150 return layer_object[0] 178 return layer_object
151 179
152 180
153 def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes): 181 def save_target_package_information(self, build_obj, target_obj, packagedict, pkgpnmap, recipes):
182 assert isinstance(build_obj, Build)
183 assert isinstance(target_obj, Target)
184
185 errormsg = ""
154 for p in packagedict: 186 for p in packagedict:
155 searchname = p 187 searchname = p
156 if 'OPKGN' in pkgpnmap[p].keys(): 188 if 'OPKGN' in pkgpnmap[p].keys():
@@ -178,7 +210,7 @@ class ORMWrapper(object):
178 path = targetpath, 210 path = targetpath,
179 size = targetfilesize) 211 size = targetfilesize)
180 except KeyError as e: 212 except KeyError as e:
181 print "Key error, package", p, "key", e 213 errormsg += " stpi: Key error, package %s key %s \n" % ( p, e )
182 214
183 # save disk installed size 215 # save disk installed size
184 packagedict[p]['object'].installed_size = packagedict[p]['size'] 216 packagedict[p]['object'].installed_size = packagedict[p]['size']
@@ -198,8 +230,15 @@ class ORMWrapper(object):
198 dep_type = tdeptype, 230 dep_type = tdeptype,
199 target = target_obj); 231 target = target_obj);
200 232
233 if (len(errormsg) > 0):
234 raise Exception(errormsg)
235
201 236
202 def create_logmessage(self, log_information): 237 def create_logmessage(self, log_information):
238 assert 'build' in log_information
239 assert 'level' in log_information
240 assert 'message' in log_information
241
203 log_object = LogMessage.objects.create( 242 log_object = LogMessage.objects.create(
204 build = log_information['build'], 243 build = log_information['build'],
205 level = log_information['level'], 244 level = log_information['level'],
@@ -213,8 +252,10 @@ class ORMWrapper(object):
213 252
214 253
215 def save_build_package_information(self, build_obj, package_info, recipes): 254 def save_build_package_information(self, build_obj, package_info, recipes):
255 assert isinstance(build_obj, Build)
256
216 # create and save the object 257 # create and save the object
217 pname = package_info['PKG'] 258 pname = package_info['PKG']
218 if 'OPKGN' in package_info.keys(): 259 if 'OPKGN' in package_info.keys():
219 pname = package_info['OPKGN'] 260 pname = package_info['OPKGN']
220 261
@@ -274,6 +315,8 @@ class ORMWrapper(object):
274 return bp_object 315 return bp_object
275 316
276 def save_build_variables(self, build_obj, vardump): 317 def save_build_variables(self, build_obj, vardump):
318 assert isinstance(build_obj, Build)
319
277 for k in vardump: 320 for k in vardump:
278 if not bool(vardump[k]['func']): 321 if not bool(vardump[k]['func']):
279 value = vardump[k]['v']; 322 value = vardump[k]['v'];
@@ -340,7 +383,7 @@ class BuildInfoHelper(object):
340 return build_info 383 return build_info
341 384
342 def _get_task_information(self, event, recipe): 385 def _get_task_information(self, event, recipe):
343 386 assert 'taskname' in vars(event)
344 387
345 task_information = {} 388 task_information = {}
346 task_information['build'] = self.internal_state['build'] 389 task_information['build'] = self.internal_state['build']
@@ -355,7 +398,11 @@ class BuildInfoHelper(object):
355 return task_information 398 return task_information
356 399
357 def _get_layer_version_for_path(self, path): 400 def _get_layer_version_for_path(self, path):
401 assert path.startswith("/")
402 assert 'build' in self.internal_state
403
358 def _slkey(layer_version): 404 def _slkey(layer_version):
405 assert isinstance(layer_version, Layer_Version)
359 return len(layer_version.layer.local_path) 406 return len(layer_version.layer.local_path)
360 407
361 # Heuristics: we always match recipe to the deepest layer path that 408 # Heuristics: we always match recipe to the deepest layer path that
@@ -369,16 +416,17 @@ class BuildInfoHelper(object):
369 return None 416 return None
370 417
371 def _get_recipe_information_from_taskfile(self, taskfile): 418 def _get_recipe_information_from_taskfile(self, taskfile):
372 419 localfilepath = taskfile.split(":")[-1]
373 layer_version_obj = self._get_layer_version_for_path(re.split(':', taskfile)[-1]) 420 layer_version_obj = self._get_layer_version_for_path(localfilepath)
374 421
375 recipe_info = {} 422 recipe_info = {}
376 recipe_info['layer_version'] = layer_version_obj 423 recipe_info['layer_version'] = layer_version_obj
377 recipe_info['file_path'] = re.split(':', taskfile)[-1] 424 recipe_info['file_path'] = taskfile
378 425
379 return recipe_info 426 return recipe_info
380 427
381 def _get_path_information(self, task_object): 428 def _get_path_information(self, task_object):
429 assert isinstance(task_object, Task)
382 build_stats_format = "{tmpdir}/buildstats/{target}-{machine}/{buildname}/{package}/" 430 build_stats_format = "{tmpdir}/buildstats/{target}-{machine}/{buildname}/{package}/"
383 build_stats_path = [] 431 build_stats_path = []
384 432
@@ -410,6 +458,7 @@ class BuildInfoHelper(object):
410 ## external available methods to store information 458 ## external available methods to store information
411 459
412 def store_layer_info(self, event): 460 def store_layer_info(self, event):
461 assert 'data' in vars(event)
413 layerinfos = event.data 462 layerinfos = event.data
414 self.internal_state['lvs'] = {} 463 self.internal_state['lvs'] = {}
415 for layer in layerinfos: 464 for layer in layerinfos:
@@ -417,7 +466,7 @@ class BuildInfoHelper(object):
417 466
418 467
419 def store_started_build(self, event): 468 def store_started_build(self, event):
420 469 assert '_pkgs' in vars(event)
421 build_information = self._get_build_information() 470 build_information = self._get_build_information()
422 471
423 build_obj = self.orm_wrapper.create_build_object(build_information) 472 build_obj = self.orm_wrapper.create_build_object(build_information)
@@ -431,7 +480,7 @@ class BuildInfoHelper(object):
431 480
432 # create target information 481 # create target information
433 target_information = {} 482 target_information = {}
434 target_information['targets'] = event.getPkgs() 483 target_information['targets'] = event._pkgs
435 target_information['build'] = build_obj 484 target_information['build'] = build_obj
436 485
437 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information) 486 self.internal_state['targets'] = self.orm_wrapper.create_target_objects(target_information)
@@ -445,7 +494,12 @@ class BuildInfoHelper(object):
445 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) 494 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
446 495
447 def store_started_task(self, event): 496 def store_started_task(self, event):
448 identifier = event.taskfile.split(":")[-1] + ":" + event.taskname 497 assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
498 assert 'taskfile' in vars(event)
499 localfilepath = event.taskfile.split(":")[-1]
500 assert localfilepath.startswith("/")
501
502 identifier = event.taskfile + ":" + event.taskname
449 503
450 recipe_information = self._get_recipe_information_from_taskfile(event.taskfile) 504 recipe_information = self._get_recipe_information_from_taskfile(event.taskfile)
451 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) 505 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
@@ -454,6 +508,7 @@ class BuildInfoHelper(object):
454 task_information['outcome'] = Task.OUTCOME_NA 508 task_information['outcome'] = Task.OUTCOME_NA
455 509
456 if isinstance(event, bb.runqueue.runQueueTaskSkipped): 510 if isinstance(event, bb.runqueue.runQueueTaskSkipped):
511 assert 'reason' in vars(event)
457 task_information['task_executed'] = False 512 task_information['task_executed'] = False
458 if event.reason == "covered": 513 if event.reason == "covered":
459 task_information['outcome'] = Task.OUTCOME_COVERED 514 task_information['outcome'] = Task.OUTCOME_COVERED
@@ -481,6 +536,9 @@ class BuildInfoHelper(object):
481 536
482 def store_tasks_stats(self, event): 537 def store_tasks_stats(self, event):
483 for (taskfile, taskname, taskstats) in event.data: 538 for (taskfile, taskname, taskstats) in event.data:
539 localfilepath = taskfile.split(":")[-1]
540 assert localfilepath.startswith("/")
541
484 recipe_information = self._get_recipe_information_from_taskfile(taskfile) 542 recipe_information = self._get_recipe_information_from_taskfile(taskfile)
485 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) 543 recipe = self.orm_wrapper.get_update_recipe_object(recipe_information)
486 544
@@ -490,10 +548,14 @@ class BuildInfoHelper(object):
490 task_information['task_name'] = taskname 548 task_information['task_name'] = taskname
491 task_information['cpu_usage'] = taskstats['cpu_usage'] 549 task_information['cpu_usage'] = taskstats['cpu_usage']
492 task_information['disk_io'] = taskstats['disk_io'] 550 task_information['disk_io'] = taskstats['disk_io']
493 task_obj = self.orm_wrapper.get_update_task_object(task_information) 551 task_obj = self.orm_wrapper.get_update_task_object(task_information, True) # must exist
494 552
495 def update_and_store_task(self, event): 553 def update_and_store_task(self, event):
496 identifier = event.taskfile.split(":")[-1] + ":" + event.taskname 554 assert 'taskfile' in vars(event)
555 localfilepath = event.taskfile.split(":")[-1]
556 assert localfilepath.startswith("/")
557
558 identifier = event.taskfile + ":" + event.taskname
497 assert identifier in self.internal_state['taskdata'] 559 assert identifier in self.internal_state['taskdata']
498 560
499 recipe_information = self._get_recipe_information_from_taskfile(event.taskfile) 561 recipe_information = self._get_recipe_information_from_taskfile(event.taskfile)
@@ -525,10 +587,11 @@ class BuildInfoHelper(object):
525 task_information['outcome'] = Task.OUTCOME_FAILED 587 task_information['outcome'] = Task.OUTCOME_FAILED
526 del self.internal_state['taskdata'][identifier] 588 del self.internal_state['taskdata'][identifier]
527 589
528 self.orm_wrapper.get_update_task_object(task_information) 590 self.orm_wrapper.get_update_task_object(task_information, True) # must exist
529 591
530 592
531 def store_target_package_data(self, event): 593 def store_target_package_data(self, event):
594 assert 'data' in vars(event)
532 # for all image targets 595 # for all image targets
533 for target in self.internal_state['targets']: 596 for target in self.internal_state['targets']:
534 if target.is_image: 597 if target.is_image:
@@ -537,6 +600,13 @@ class BuildInfoHelper(object):
537 self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes']) 600 self.orm_wrapper.save_target_package_information(self.internal_state['build'], target, imgdata, pkgdata, self.internal_state['recipes'])
538 601
539 def store_dependency_information(self, event): 602 def store_dependency_information(self, event):
603 assert '_depgraph' in vars(event)
604 assert 'layer-priorities' in event._depgraph
605 assert 'pn' in event._depgraph
606 assert 'tdepends' in event._depgraph
607
608 errormsg = ""
609
540 # save layer version priorities 610 # save layer version priorities
541 if 'layer-priorities' in event._depgraph.keys(): 611 if 'layer-priorities' in event._depgraph.keys():
542 for lv in event._depgraph['layer-priorities']: 612 for lv in event._depgraph['layer-priorities']:
@@ -550,8 +620,8 @@ class BuildInfoHelper(object):
550 self.internal_state['recipes'] = {} 620 self.internal_state['recipes'] = {}
551 for pn in event._depgraph['pn']: 621 for pn in event._depgraph['pn']:
552 622
553 file_name = re.split(':', event._depgraph['pn'][pn]['filename'])[-1] 623 file_name = event._depgraph['pn'][pn]['filename']
554 layer_version_obj = self._get_layer_version_for_path(re.split(':', file_name)[-1]) 624 layer_version_obj = self._get_layer_version_for_path(file_name.split(":")[-1])
555 625
556 assert layer_version_obj is not None 626 assert layer_version_obj is not None
557 627
@@ -579,6 +649,10 @@ class BuildInfoHelper(object):
579 t.save() 649 t.save()
580 self.internal_state['recipes'][pn] = recipe 650 self.internal_state['recipes'][pn] = recipe
581 651
652 # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED
653
654 assume_provided = self.server.runCommand(["getVariable", "ASSUME_PROVIDED"])[0].split()
655
582 # save recipe dependency 656 # save recipe dependency
583 # buildtime 657 # buildtime
584 for recipe in event._depgraph['depends']: 658 for recipe in event._depgraph['depends']:
@@ -588,20 +662,9 @@ class BuildInfoHelper(object):
588 dependency = self.internal_state['recipes'][dep] 662 dependency = self.internal_state['recipes'][dep]
589 Recipe_Dependency.objects.get_or_create( recipe = target, 663 Recipe_Dependency.objects.get_or_create( recipe = target,
590 depends_on = dependency, dep_type = Recipe_Dependency.TYPE_DEPENDS) 664 depends_on = dependency, dep_type = Recipe_Dependency.TYPE_DEPENDS)
591 except KeyError: # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED 665 except KeyError as e:
592 pass 666 if e not in assume_provided and not str(e).startswith("virtual/"):
593 667 errormsg += " stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, e)
594 # runtime
595 for recipe in event._depgraph['rdepends-pn']:
596 try:
597 target = self.internal_state['recipes'][recipe]
598 for dep in event._depgraph['rdepends-pn'][recipe]:
599 dependency = self.internal_state['recipes'][dep]
600 Recipe_Dependency.objects.get_or_create( recipe = target,
601 depends_on = dependency, dep_type = Recipe_Dependency.TYPE_RDEPENDS)
602
603 except KeyError: # we'll not get recipes for key w/ values listed in ASSUME_PROVIDED
604 pass
605 668
606 # save all task information 669 # save all task information
607 def _save_a_task(taskdesc): 670 def _save_a_task(taskdesc):
@@ -632,7 +695,12 @@ class BuildInfoHelper(object):
632 dep = tasks[taskdep] 695 dep = tasks[taskdep]
633 Task_Dependency.objects.get_or_create( task = target, depends_on = dep ) 696 Task_Dependency.objects.get_or_create( task = target, depends_on = dep )
634 697
698 if (len(errormsg) > 0):
699 raise Exception(errormsg)
700
701
635 def store_build_package_information(self, event): 702 def store_build_package_information(self, event):
703 assert 'data' in vars(event)
636 package_info = event.data 704 package_info = event.data
637 self.orm_wrapper.save_build_package_information(self.internal_state['build'], 705 self.orm_wrapper.save_build_package_information(self.internal_state['build'],
638 package_info, 706 package_info,