diff options
| -rw-r--r-- | bitbake/lib/bb/ui/buildinfohelper.py | 130 | ||||
| -rw-r--r-- | bitbake/lib/bb/ui/toasterui.py | 57 |
2 files changed, 98 insertions, 89 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index cce6da5fad..09e8853338 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py | |||
| @@ -19,12 +19,20 @@ | |||
| 19 | import sys | 19 | import sys |
| 20 | import bb | 20 | import bb |
| 21 | import re | 21 | import re |
| 22 | import ast | 22 | import os |
| 23 | 23 | ||
| 24 | os.environ["DJANGO_SETTINGS_MODULE"] = "toaster.toastermain.settings" | 24 | os.environ["DJANGO_SETTINGS_MODULE"] = "toaster.toastermain.settings" |
| 25 | 25 | ||
| 26 | |||
| 26 | from django.utils import timezone | 27 | from django.utils import timezone |
| 27 | import toaster.toastermain.settings as toaster_django_settings | 28 | |
| 29 | |||
| 30 | def _configure_toaster(): | ||
| 31 | """ Add toaster to sys path for importing modules | ||
| 32 | """ | ||
| 33 | sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'toaster')) | ||
| 34 | _configure_toaster() | ||
| 35 | |||
| 28 | from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText | 36 | from toaster.orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText |
| 29 | from toaster.orm.models import Target_Image_File, BuildArtifact | 37 | from toaster.orm.models import Target_Image_File, BuildArtifact |
| 30 | from toaster.orm.models import Variable, VariableHistory | 38 | from toaster.orm.models import Variable, VariableHistory |
| @@ -33,14 +41,17 @@ from toaster.orm.models import Task_Dependency, Package_Dependency | |||
| 33 | from toaster.orm.models import Recipe_Dependency | 41 | from toaster.orm.models import Recipe_Dependency |
| 34 | 42 | ||
| 35 | from toaster.orm.models import Project | 43 | from toaster.orm.models import Project |
| 44 | from bldcontrol.models import BuildEnvironment, BuildRequest | ||
| 36 | 45 | ||
| 37 | from bb.msg import BBLogFormatter as format | 46 | from bb.msg import BBLogFormatter as formatter |
| 38 | from django.db import models | 47 | from django.db import models |
| 39 | from pprint import pformat | 48 | from pprint import pformat |
| 40 | import logging | 49 | import logging |
| 41 | 50 | ||
| 42 | from django.db import transaction, connection | 51 | from django.db import transaction, connection |
| 43 | 52 | ||
| 53 | # pylint: disable=invalid-name | ||
| 54 | # the logger name is standard throughout BitBake | ||
| 44 | logger = logging.getLogger("ToasterLogger") | 55 | logger = logging.getLogger("ToasterLogger") |
| 45 | 56 | ||
| 46 | 57 | ||
| @@ -57,7 +68,6 @@ class ORMWrapper(object): | |||
| 57 | self.layer_version_objects = [] | 68 | self.layer_version_objects = [] |
| 58 | self.task_objects = {} | 69 | self.task_objects = {} |
| 59 | self.recipe_objects = {} | 70 | self.recipe_objects = {} |
| 60 | pass | ||
| 61 | 71 | ||
| 62 | @staticmethod | 72 | @staticmethod |
| 63 | def _build_key(**kwargs): | 73 | def _build_key(**kwargs): |
| @@ -106,6 +116,13 @@ class ORMWrapper(object): | |||
| 106 | 116 | ||
| 107 | return vars(self)[dictname][key] | 117 | return vars(self)[dictname][key] |
| 108 | 118 | ||
| 119 | # pylint: disable=no-self-use | ||
| 120 | # we disable detection of no self use in functions because the methods actually work on the object | ||
| 121 | # even if they don't touch self anywhere | ||
| 122 | |||
| 123 | # pylint: disable=bad-continuation | ||
| 124 | # we do not follow the python conventions for continuation indentation due to long lines here | ||
| 125 | |||
| 109 | def create_build_object(self, build_info, brbe, project_id): | 126 | def create_build_object(self, build_info, brbe, project_id): |
| 110 | assert 'machine' in build_info | 127 | assert 'machine' in build_info |
| 111 | assert 'distro' in build_info | 128 | assert 'distro' in build_info |
| @@ -119,8 +136,7 @@ class ORMWrapper(object): | |||
| 119 | buildrequest = None | 136 | buildrequest = None |
| 120 | if brbe is not None: # this build was triggered by a request from a user | 137 | if brbe is not None: # this build was triggered by a request from a user |
| 121 | logger.debug(1, "buildinfohelper: brbe is %s" % brbe) | 138 | logger.debug(1, "buildinfohelper: brbe is %s" % brbe) |
| 122 | from bldcontrol.models import BuildEnvironment, BuildRequest | 139 | br, _ = brbe.split(":") |
| 123 | br, be = brbe.split(":") | ||
| 124 | buildrequest = BuildRequest.objects.get(pk = br) | 140 | buildrequest = BuildRequest.objects.get(pk = br) |
| 125 | prj = buildrequest.project | 141 | prj = buildrequest.project |
| 126 | 142 | ||
| @@ -129,18 +145,16 @@ class ORMWrapper(object): | |||
| 129 | prj = Project.objects.get(pk = project_id) | 145 | prj = Project.objects.get(pk = project_id) |
| 130 | 146 | ||
| 131 | else: # this build was triggered by a legacy system, or command line interactive mode | 147 | else: # this build was triggered by a legacy system, or command line interactive mode |
| 132 | prj, created = Project.objects.get_or_create(pk=0, name="Default Project") | 148 | prj, _ = Project.objects.get_or_create(pk=0, name="Default Project") |
| 133 | logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) | 149 | logger.debug(1, "buildinfohelper: project is not specified, defaulting to %s" % prj) |
| 134 | 150 | ||
| 135 | 151 | ||
| 136 | if buildrequest is not None: | 152 | if buildrequest is not None: |
| 137 | build = buildrequest.build | 153 | build = buildrequest.build |
| 138 | logger.info("Updating existing build, with %s" % build_info) | 154 | logger.info("Updating existing build, with %s", build_info) |
| 139 | build.machine=build_info['machine'] | 155 | build.machine=build_info['machine'] |
| 140 | build.distro=build_info['distro'] | 156 | build.distro=build_info['distro'] |
| 141 | build.distro_version=build_info['distro_version'] | 157 | build.distro_version=build_info['distro_version'] |
| 142 | started_on=build_info['started_on'] | ||
| 143 | completed_on=build_info['started_on'] | ||
| 144 | build.cooker_log_path=build_info['cooker_log_path'] | 158 | build.cooker_log_path=build_info['cooker_log_path'] |
| 145 | build.build_name=build_info['build_name'] | 159 | build.build_name=build_info['build_name'] |
| 146 | build.bitbake_version=build_info['bitbake_version'] | 160 | build.bitbake_version=build_info['bitbake_version'] |
| @@ -210,8 +224,8 @@ class ORMWrapper(object): | |||
| 210 | task_name=task_information['task_name'] | 224 | task_name=task_information['task_name'] |
| 211 | ) | 225 | ) |
| 212 | if created and must_exist: | 226 | if created and must_exist: |
| 213 | task_information['debug'] = "build id %d, recipe id %d" % (task_information['build'].pk, task_information['recipe'].pk) | 227 | task_information['debug'] = "build id %d, recipe id %d" % (task_information['build'].pk, task_information['recipe'].pk) |
| 214 | raise NotExisting("Task object created when expected to exist", task_information) | 228 | raise NotExisting("Task object created when expected to exist", task_information) |
| 215 | 229 | ||
| 216 | object_changed = False | 230 | object_changed = False |
| 217 | for v in vars(task_object): | 231 | for v in vars(task_object): |
| @@ -278,7 +292,7 @@ class ORMWrapper(object): | |||
| 278 | assert 'priority' in layer_version_information | 292 | assert 'priority' in layer_version_information |
| 279 | assert 'local_path' in layer_version_information | 293 | assert 'local_path' in layer_version_information |
| 280 | 294 | ||
| 281 | layer_version_object, created = Layer_Version.objects.get_or_create( | 295 | layer_version_object, _ = Layer_Version.objects.get_or_create( |
| 282 | build = build_obj, | 296 | build = build_obj, |
| 283 | layer = layer_obj, | 297 | layer = layer_obj, |
| 284 | branch = layer_version_information['branch'], | 298 | branch = layer_version_information['branch'], |
| @@ -296,13 +310,12 @@ class ORMWrapper(object): | |||
| 296 | assert 'layer_index_url' in layer_information | 310 | assert 'layer_index_url' in layer_information |
| 297 | 311 | ||
| 298 | if brbe is None: | 312 | if brbe is None: |
| 299 | layer_object, created = Layer.objects.get_or_create( | 313 | layer_object, _ = Layer.objects.get_or_create( |
| 300 | name=layer_information['name'], | 314 | name=layer_information['name'], |
| 301 | layer_index_url=layer_information['layer_index_url']) | 315 | layer_index_url=layer_information['layer_index_url']) |
| 302 | return layer_object | 316 | return layer_object |
| 303 | else: | 317 | else: |
| 304 | # we are under managed mode; we must match the layer used in the Project Layer | 318 | # we are under managed mode; we must match the layer used in the Project Layer |
| 305 | from bldcontrol.models import BuildEnvironment, BuildRequest | ||
| 306 | br_id, be_id = brbe.split(":") | 319 | br_id, be_id = brbe.split(":") |
| 307 | 320 | ||
| 308 | # find layer by checkout path; | 321 | # find layer by checkout path; |
| @@ -422,7 +435,7 @@ class ORMWrapper(object): | |||
| 422 | 435 | ||
| 423 | try: | 436 | try: |
| 424 | filetarget_obj = Target_File.objects.get(target = target_obj, path = filetarget_path) | 437 | filetarget_obj = Target_File.objects.get(target = target_obj, path = filetarget_path) |
| 425 | except Exception as e: | 438 | except Target_File.DoesNotExist: |
| 426 | # we might have an invalid link; no way to detect this. just set it to None | 439 | # we might have an invalid link; no way to detect this. just set it to None |
| 427 | filetarget_obj = None | 440 | filetarget_obj = None |
| 428 | 441 | ||
| @@ -498,11 +511,11 @@ class ORMWrapper(object): | |||
| 498 | if len(packagedeps_objs) > 0: | 511 | if len(packagedeps_objs) > 0: |
| 499 | Package_Dependency.objects.bulk_create(packagedeps_objs) | 512 | Package_Dependency.objects.bulk_create(packagedeps_objs) |
| 500 | 513 | ||
| 501 | if (len(errormsg) > 0): | 514 | if len(errormsg) > 0: |
| 502 | logger.warn("buildinfohelper: target_package_info could not identify recipes: \n%s" % errormsg) | 515 | logger.warn("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) |
| 503 | 516 | ||
| 504 | def save_target_image_file_information(self, target_obj, file_name, file_size): | 517 | def save_target_image_file_information(self, target_obj, file_name, file_size): |
| 505 | target_image_file = Target_Image_File.objects.create( target = target_obj, | 518 | Target_Image_File.objects.create( target = target_obj, |
| 506 | file_name = file_name, | 519 | file_name = file_name, |
| 507 | file_size = file_size) | 520 | file_size = file_size) |
| 508 | 521 | ||
| @@ -542,7 +555,7 @@ class ORMWrapper(object): | |||
| 542 | if 'OPKGN' in package_info.keys(): | 555 | if 'OPKGN' in package_info.keys(): |
| 543 | pname = package_info['OPKGN'] | 556 | pname = package_info['OPKGN'] |
| 544 | 557 | ||
| 545 | bp_object, created = Package.objects.get_or_create( build = build_obj, | 558 | bp_object, _ = Package.objects.get_or_create( build = build_obj, |
| 546 | name = pname ) | 559 | name = pname ) |
| 547 | 560 | ||
| 548 | bp_object.installed_name = package_info['PKG'] | 561 | bp_object.installed_name = package_info['PKG'] |
| @@ -559,7 +572,7 @@ class ORMWrapper(object): | |||
| 559 | # save any attached file information | 572 | # save any attached file information |
| 560 | packagefile_objects = [] | 573 | packagefile_objects = [] |
| 561 | for path in package_info['FILES_INFO']: | 574 | for path in package_info['FILES_INFO']: |
| 562 | packagefile_objects.append(Package_File( package = bp_object, | 575 | packagefile_objects.append(Package_File( package = bp_object, |
| 563 | path = path, | 576 | path = path, |
| 564 | size = package_info['FILES_INFO'][path] )) | 577 | size = package_info['FILES_INFO'][path] )) |
| 565 | if len(packagefile_objects): | 578 | if len(packagefile_objects): |
| @@ -643,7 +656,19 @@ class ORMWrapper(object): | |||
| 643 | 656 | ||
| 644 | HelpText.objects.bulk_create(helptext_objects) | 657 | HelpText.objects.bulk_create(helptext_objects) |
| 645 | 658 | ||
| 646 | class MockEvent: pass # sometimes we mock an event, declare it here | 659 | |
| 660 | class MockEvent(object): | ||
| 661 | """ This object is used to create event, for which normal event-processing methods can | ||
| 662 | be used, out of data that is not coming via an actual event | ||
| 663 | """ | ||
| 664 | def __init__(self): | ||
| 665 | self.msg = None | ||
| 666 | self.levelno = None | ||
| 667 | self.taskname = None | ||
| 668 | self.taskhash = None | ||
| 669 | self.pathname = None | ||
| 670 | self.lineno = None | ||
| 671 | |||
| 647 | 672 | ||
| 648 | class BuildInfoHelper(object): | 673 | class BuildInfoHelper(object): |
| 649 | """ This class gathers the build information from the server and sends it | 674 | """ This class gathers the build information from the server and sends it |
| @@ -652,9 +677,12 @@ class BuildInfoHelper(object): | |||
| 652 | Keeps in memory all data that needs matching before writing it to the database | 677 | Keeps in memory all data that needs matching before writing it to the database |
| 653 | """ | 678 | """ |
| 654 | 679 | ||
| 680 | # pylint: disable=protected-access | ||
| 681 | # the code will look into the protected variables of the event; no easy way around this | ||
| 682 | # pylint: disable=bad-continuation | ||
| 683 | # we do not follow the python conventions for continuation indentation due to long lines here | ||
| 655 | 684 | ||
| 656 | def __init__(self, server, has_build_history = False): | 685 | def __init__(self, server, has_build_history = False): |
| 657 | self._configure_django() | ||
| 658 | self.internal_state = {} | 686 | self.internal_state = {} |
| 659 | self.internal_state['taskdata'] = {} | 687 | self.internal_state['taskdata'] = {} |
| 660 | self.task_order = 0 | 688 | self.task_order = 0 |
| @@ -671,10 +699,6 @@ class BuildInfoHelper(object): | |||
| 671 | logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self)) | 699 | logger.debug(1, "buildinfohelper: Build info helper inited %s" % vars(self)) |
| 672 | 700 | ||
| 673 | 701 | ||
| 674 | def _configure_django(self): | ||
| 675 | # Add toaster to sys path for importing modules | ||
| 676 | sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'toaster')) | ||
| 677 | |||
| 678 | ################### | 702 | ################### |
| 679 | ## methods to convert event/external info into objects that the ORM layer uses | 703 | ## methods to convert event/external info into objects that the ORM layer uses |
| 680 | 704 | ||
| @@ -727,7 +751,6 @@ class BuildInfoHelper(object): | |||
| 727 | else: | 751 | else: |
| 728 | br_id, be_id = self.brbe.split(":") | 752 | br_id, be_id = self.brbe.split(":") |
| 729 | from bldcontrol.bbcontroller import getBuildEnvironmentController | 753 | from bldcontrol.bbcontroller import getBuildEnvironmentController |
| 730 | from bldcontrol.models import BuildRequest | ||
| 731 | bc = getBuildEnvironmentController(pk = be_id) | 754 | bc = getBuildEnvironmentController(pk = be_id) |
| 732 | 755 | ||
| 733 | def _slkey_managed(layer_version): | 756 | def _slkey_managed(layer_version): |
| @@ -747,11 +770,11 @@ class BuildInfoHelper(object): | |||
| 747 | return lvo | 770 | return lvo |
| 748 | 771 | ||
| 749 | #if we get here, we didn't read layers correctly; dump whatever information we have on the error log | 772 | #if we get here, we didn't read layers correctly; dump whatever information we have on the error log |
| 750 | logger.warn("Could not match layer version for recipe path %s : %s" % (path, self.orm_wrapper.layer_version_objects)) | 773 | logger.warn("Could not match layer version for recipe path %s : %s", path, self.orm_wrapper.layer_version_objects) |
| 751 | 774 | ||
| 752 | #mockup the new layer | 775 | #mockup the new layer |
| 753 | unknown_layer, created = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", layer_index_url="") | 776 | unknown_layer, _ = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", layer_index_url="") |
| 754 | unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build']) | 777 | unknown_layer_version_obj, _ = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build']) |
| 755 | 778 | ||
| 756 | # append it so we don't run into this error again and again | 779 | # append it so we don't run into this error again and again |
| 757 | self.orm_wrapper.layer_version_objects.append(unknown_layer_version_obj) | 780 | self.orm_wrapper.layer_version_objects.append(unknown_layer_version_obj) |
| @@ -798,13 +821,6 @@ class BuildInfoHelper(object): | |||
| 798 | 821 | ||
| 799 | return build_stats_path | 822 | return build_stats_path |
| 800 | 823 | ||
| 801 | def _remove_redundant(self, string): | ||
| 802 | ret = [] | ||
| 803 | for i in string.split(): | ||
| 804 | if i not in ret: | ||
| 805 | ret.append(i) | ||
| 806 | return " ".join(sorted(ret)) | ||
| 807 | |||
| 808 | 824 | ||
| 809 | ################################ | 825 | ################################ |
| 810 | ## external available methods to store information | 826 | ## external available methods to store information |
| @@ -827,7 +843,7 @@ class BuildInfoHelper(object): | |||
| 827 | self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version'] | 843 | self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version'] |
| 828 | self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path'] | 844 | self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path'] |
| 829 | except NotExisting as nee: | 845 | except NotExisting as nee: |
| 830 | logger.warn("buildinfohelper: cannot identify layer exception:%s " % nee) | 846 | logger.warn("buildinfohelper: cannot identify layer exception:%s ", nee) |
| 831 | 847 | ||
| 832 | 848 | ||
| 833 | def store_started_build(self, event): | 849 | def store_started_build(self, event): |
| @@ -861,8 +877,7 @@ class BuildInfoHelper(object): | |||
| 861 | path_prefixes = [] | 877 | path_prefixes = [] |
| 862 | 878 | ||
| 863 | if self.brbe is not None: | 879 | if self.brbe is not None: |
| 864 | br_id, be_id = self.brbe.split(":") | 880 | _, be_id = self.brbe.split(":") |
| 865 | from bldcontrol.models import BuildEnvironment, BuildRequest | ||
| 866 | be = BuildEnvironment.objects.get(pk = be_id) | 881 | be = BuildEnvironment.objects.get(pk = be_id) |
| 867 | path_prefixes.append(be.builddir) | 882 | path_prefixes.append(be.builddir) |
| 868 | 883 | ||
| @@ -887,7 +902,6 @@ class BuildInfoHelper(object): | |||
| 887 | 902 | ||
| 888 | 903 | ||
| 889 | def update_target_image_file(self, event): | 904 | def update_target_image_file(self, event): |
| 890 | image_fstypes = self.server.runCommand(["getVariable", "IMAGE_FSTYPES"])[0] | ||
| 891 | evdata = BuildInfoHelper._get_data_from_event(event) | 905 | evdata = BuildInfoHelper._get_data_from_event(event) |
| 892 | 906 | ||
| 893 | for t in self.internal_state['targets']: | 907 | for t in self.internal_state['targets']: |
| @@ -949,7 +963,7 @@ class BuildInfoHelper(object): | |||
| 949 | self.task_order += 1 | 963 | self.task_order += 1 |
| 950 | task_information['order'] = self.task_order | 964 | task_information['order'] = self.task_order |
| 951 | 965 | ||
| 952 | task_obj = self.orm_wrapper.get_update_task_object(task_information) | 966 | self.orm_wrapper.get_update_task_object(task_information) |
| 953 | 967 | ||
| 954 | self.internal_state['taskdata'][identifier] = { | 968 | self.internal_state['taskdata'][identifier] = { |
| 955 | 'outcome': task_information['outcome'], | 969 | 'outcome': task_information['outcome'], |
| @@ -970,7 +984,7 @@ class BuildInfoHelper(object): | |||
| 970 | file_path__endswith = recipe_information['file_path'], | 984 | file_path__endswith = recipe_information['file_path'], |
| 971 | name = recipename) | 985 | name = recipename) |
| 972 | except Recipe.DoesNotExist: | 986 | except Recipe.DoesNotExist: |
| 973 | logger.error("Could not find recipe for recipe_information %s name %s" % (pformat(recipe_information), recipename)) | 987 | logger.error("Could not find recipe for recipe_information %s name %s" , pformat(recipe_information), recipename) |
| 974 | raise | 988 | raise |
| 975 | 989 | ||
| 976 | task_information = {} | 990 | task_information = {} |
| @@ -981,7 +995,7 @@ class BuildInfoHelper(object): | |||
| 981 | task_information['disk_io'] = taskstats['disk_io'] | 995 | task_information['disk_io'] = taskstats['disk_io'] |
| 982 | if 'elapsed_time' in taskstats: | 996 | if 'elapsed_time' in taskstats: |
| 983 | task_information['elapsed_time'] = taskstats['elapsed_time'] | 997 | task_information['elapsed_time'] = taskstats['elapsed_time'] |
| 984 | task_obj = self.orm_wrapper.get_update_task_object(task_information, True) # must exist | 998 | self.orm_wrapper.get_update_task_object(task_information, True) # must exist |
| 985 | 999 | ||
| 986 | def update_and_store_task(self, event): | 1000 | def update_and_store_task(self, event): |
| 987 | assert 'taskfile' in vars(event) | 1001 | assert 'taskfile' in vars(event) |
| @@ -1047,7 +1061,7 @@ class BuildInfoHelper(object): | |||
| 1047 | def store_missed_state_tasks(self, event): | 1061 | def store_missed_state_tasks(self, event): |
| 1048 | for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['missed']: | 1062 | for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['missed']: |
| 1049 | 1063 | ||
| 1050 | identifier = fn + taskname + "_setscene" | 1064 | # identifier = fn + taskname + "_setscene" |
| 1051 | recipe_information = self._get_recipe_information_from_taskfile(fn) | 1065 | recipe_information = self._get_recipe_information_from_taskfile(fn) |
| 1052 | recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) | 1066 | recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) |
| 1053 | mevent = MockEvent() | 1067 | mevent = MockEvent() |
| @@ -1065,7 +1079,7 @@ class BuildInfoHelper(object): | |||
| 1065 | 1079 | ||
| 1066 | for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['found']: | 1080 | for (fn, taskname, taskhash, sstatefile) in BuildInfoHelper._get_data_from_event(event)['found']: |
| 1067 | 1081 | ||
| 1068 | identifier = fn + taskname + "_setscene" | 1082 | # identifier = fn + taskname + "_setscene" |
| 1069 | recipe_information = self._get_recipe_information_from_taskfile(fn) | 1083 | recipe_information = self._get_recipe_information_from_taskfile(fn) |
| 1070 | recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) | 1084 | recipe = self.orm_wrapper.get_update_recipe_object(recipe_information) |
| 1071 | mevent = MockEvent() | 1085 | mevent = MockEvent() |
| @@ -1105,7 +1119,7 @@ class BuildInfoHelper(object): | |||
| 1105 | # save layer version priorities | 1119 | # save layer version priorities |
| 1106 | if 'layer-priorities' in event._depgraph.keys(): | 1120 | if 'layer-priorities' in event._depgraph.keys(): |
| 1107 | for lv in event._depgraph['layer-priorities']: | 1121 | for lv in event._depgraph['layer-priorities']: |
| 1108 | (name, path, regexp, priority) = lv | 1122 | (_, path, _, priority) = lv |
| 1109 | layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^ | 1123 | layer_version_obj = self._get_layer_version_for_path(path[1:]) # paths start with a ^ |
| 1110 | assert layer_version_obj is not None | 1124 | assert layer_version_obj is not None |
| 1111 | layer_version_obj.priority = priority | 1125 | layer_version_obj.priority = priority |
| @@ -1218,8 +1232,8 @@ class BuildInfoHelper(object): | |||
| 1218 | taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep )) | 1232 | taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep )) |
| 1219 | Task_Dependency.objects.bulk_create(taskdeps_objects) | 1233 | Task_Dependency.objects.bulk_create(taskdeps_objects) |
| 1220 | 1234 | ||
| 1221 | if (len(errormsg) > 0): | 1235 | if len(errormsg) > 0: |
| 1222 | logger.warn("buildinfohelper: dependency info not identify recipes: \n%s" % errormsg) | 1236 | logger.warn("buildinfohelper: dependency info not identify recipes: \n%s", errormsg) |
| 1223 | 1237 | ||
| 1224 | 1238 | ||
| 1225 | def store_build_package_information(self, event): | 1239 | def store_build_package_information(self, event): |
| @@ -1232,7 +1246,6 @@ class BuildInfoHelper(object): | |||
| 1232 | def _store_build_done(self, errorcode): | 1246 | def _store_build_done(self, errorcode): |
| 1233 | logger.info("Build exited with errorcode %d", errorcode) | 1247 | logger.info("Build exited with errorcode %d", errorcode) |
| 1234 | br_id, be_id = self.brbe.split(":") | 1248 | br_id, be_id = self.brbe.split(":") |
| 1235 | from bldcontrol.models import BuildEnvironment, BuildRequest | ||
| 1236 | be = BuildEnvironment.objects.get(pk = be_id) | 1249 | be = BuildEnvironment.objects.get(pk = be_id) |
| 1237 | be.lock = BuildEnvironment.LOCK_LOCK | 1250 | be.lock = BuildEnvironment.LOCK_LOCK |
| 1238 | be.save() | 1251 | be.save() |
| @@ -1247,7 +1260,7 @@ class BuildInfoHelper(object): | |||
| 1247 | 1260 | ||
| 1248 | def store_log_error(self, text): | 1261 | def store_log_error(self, text): |
| 1249 | mockevent = MockEvent() | 1262 | mockevent = MockEvent() |
| 1250 | mockevent.levelno = format.ERROR | 1263 | mockevent.levelno = formatter.ERROR |
| 1251 | mockevent.msg = text | 1264 | mockevent.msg = text |
| 1252 | mockevent.pathname = '-- None' | 1265 | mockevent.pathname = '-- None' |
| 1253 | mockevent.lineno = LogMessage.ERROR | 1266 | mockevent.lineno = LogMessage.ERROR |
| @@ -1263,7 +1276,7 @@ class BuildInfoHelper(object): | |||
| 1263 | 1276 | ||
| 1264 | 1277 | ||
| 1265 | def store_log_event(self, event): | 1278 | def store_log_event(self, event): |
| 1266 | if event.levelno < format.WARNING: | 1279 | if event.levelno < formatter.WARNING: |
| 1267 | return | 1280 | return |
| 1268 | 1281 | ||
| 1269 | if 'args' in vars(event): | 1282 | if 'args' in vars(event): |
| @@ -1276,8 +1289,7 @@ class BuildInfoHelper(object): | |||
| 1276 | self.internal_state['backlog'].append(event) | 1289 | self.internal_state['backlog'].append(event) |
| 1277 | return | 1290 | return |
| 1278 | else: # we're under Toaster control, the build is already created | 1291 | else: # we're under Toaster control, the build is already created |
| 1279 | from bldcontrol.models import BuildRequest, BRError | 1292 | br, _ = self.brbe.split(":") |
| 1280 | br, be = self.brbe.split(":") | ||
| 1281 | buildrequest = BuildRequest.objects.get(pk = br) | 1293 | buildrequest = BuildRequest.objects.get(pk = br) |
| 1282 | self.internal_state['build'] = buildrequest.build | 1294 | self.internal_state['build'] = buildrequest.build |
| 1283 | 1295 | ||
| @@ -1293,9 +1305,9 @@ class BuildInfoHelper(object): | |||
| 1293 | 1305 | ||
| 1294 | log_information = {} | 1306 | log_information = {} |
| 1295 | log_information['build'] = self.internal_state['build'] | 1307 | log_information['build'] = self.internal_state['build'] |
| 1296 | if event.levelno == format.ERROR: | 1308 | if event.levelno == formatter.ERROR: |
| 1297 | log_information['level'] = LogMessage.ERROR | 1309 | log_information['level'] = LogMessage.ERROR |
| 1298 | elif event.levelno == format.WARNING: | 1310 | elif event.levelno == formatter.WARNING: |
| 1299 | log_information['level'] = LogMessage.WARNING | 1311 | log_information['level'] = LogMessage.WARNING |
| 1300 | elif event.levelno == -2: # toaster self-logging | 1312 | elif event.levelno == -2: # toaster self-logging |
| 1301 | log_information['level'] = -2 | 1313 | log_information['level'] = -2 |
| @@ -1305,7 +1317,7 @@ class BuildInfoHelper(object): | |||
| 1305 | log_information['message'] = event.msg | 1317 | log_information['message'] = event.msg |
| 1306 | log_information['pathname'] = event.pathname | 1318 | log_information['pathname'] = event.pathname |
| 1307 | log_information['lineno'] = event.lineno | 1319 | log_information['lineno'] = event.lineno |
| 1308 | logger.info("Logging error 2: %s" % log_information) | 1320 | logger.info("Logging error 2: %s", log_information) |
| 1309 | self.orm_wrapper.create_logmessage(log_information) | 1321 | self.orm_wrapper.create_logmessage(log_information) |
| 1310 | 1322 | ||
| 1311 | def close(self, errorcode): | 1323 | def close(self, errorcode): |
| @@ -1320,7 +1332,7 @@ class BuildInfoHelper(object): | |||
| 1320 | else: | 1332 | else: |
| 1321 | # we have no build, and we still have events; something amazingly wrong happend | 1333 | # we have no build, and we still have events; something amazingly wrong happend |
| 1322 | for event in self.internal_state['backlog']: | 1334 | for event in self.internal_state['backlog']: |
| 1323 | logger.error("UNSAVED log: %s", event.msg) | 1335 | logger.error("UNSAVED log: %s", event.msg) |
| 1324 | 1336 | ||
| 1325 | if not connection.features.autocommits_when_autocommit_is_off: | 1337 | if not connection.features.autocommits_when_autocommit_is_off: |
| 1326 | transaction.set_autocommit(True) | 1338 | transaction.set_autocommit(True) |
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index 767bfabe31..9c7e87dd1e 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py | |||
| @@ -31,16 +31,12 @@ from bb.ui import uihelper | |||
| 31 | from bb.ui.buildinfohelper import BuildInfoHelper | 31 | from bb.ui.buildinfohelper import BuildInfoHelper |
| 32 | 32 | ||
| 33 | import bb.msg | 33 | import bb.msg |
| 34 | import copy | ||
| 35 | import fcntl | ||
| 36 | import logging | 34 | import logging |
| 37 | import os | 35 | import os |
| 38 | import progressbar | 36 | |
| 39 | import signal | 37 | # pylint: disable=invalid-name |
| 40 | import struct | 38 | # module properties for UI modules are read by bitbake and the contract should not be broken |
| 41 | import sys | 39 | |
| 42 | import time | ||
| 43 | import xmlrpclib | ||
| 44 | 40 | ||
| 45 | featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING, bb.cooker.CookerFeatures.SEND_SANITYEVENTS] | 41 | featureSet = [bb.cooker.CookerFeatures.HOB_EXTRA_CACHES, bb.cooker.CookerFeatures.SEND_DEPENDS_TREE, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING, bb.cooker.CookerFeatures.SEND_SANITYEVENTS] |
| 46 | 42 | ||
| @@ -53,15 +49,15 @@ def _log_settings_from_server(server): | |||
| 53 | # Get values of variables which control our output | 49 | # Get values of variables which control our output |
| 54 | includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"]) | 50 | includelogs, error = server.runCommand(["getVariable", "BBINCLUDELOGS"]) |
| 55 | if error: | 51 | if error: |
| 56 | logger.error("Unable to get the value of BBINCLUDELOGS variable: %s" % error) | 52 | logger.error("Unable to get the value of BBINCLUDELOGS variable: %s", error) |
| 57 | raise BaseException(error) | 53 | raise BaseException(error) |
| 58 | loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"]) | 54 | loglines, error = server.runCommand(["getVariable", "BBINCLUDELOGS_LINES"]) |
| 59 | if error: | 55 | if error: |
| 60 | logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s" % error) | 56 | logger.error("Unable to get the value of BBINCLUDELOGS_LINES variable: %s", error) |
| 61 | raise BaseException(error) | 57 | raise BaseException(error) |
| 62 | consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"]) | 58 | consolelogfile, error = server.runCommand(["getVariable", "BB_CONSOLELOG"]) |
| 63 | if error: | 59 | if error: |
| 64 | logger.error("Unable to get the value of BB_CONSOLELOG variable: %s" % error) | 60 | logger.error("Unable to get the value of BB_CONSOLELOG variable: %s", error) |
| 65 | raise BaseException(error) | 61 | raise BaseException(error) |
| 66 | return includelogs, loglines, consolelogfile | 62 | return includelogs, loglines, consolelogfile |
| 67 | 63 | ||
| @@ -71,17 +67,17 @@ def main(server, eventHandler, params ): | |||
| 71 | 67 | ||
| 72 | console = logging.StreamHandler(sys.stdout) | 68 | console = logging.StreamHandler(sys.stdout) |
| 73 | format_str = "%(levelname)s: %(message)s" | 69 | format_str = "%(levelname)s: %(message)s" |
| 74 | format = bb.msg.BBLogFormatter(format_str) | 70 | formatter = bb.msg.BBLogFormatter(format_str) |
| 75 | bb.msg.addDefaultlogFilter(console) | 71 | bb.msg.addDefaultlogFilter(console) |
| 76 | console.setFormatter(format) | 72 | console.setFormatter(formatter) |
| 77 | logger.addHandler(console) | 73 | logger.addHandler(console) |
| 78 | logger.setLevel(logging.INFO) | 74 | logger.setLevel(logging.INFO) |
| 79 | 75 | ||
| 80 | includelogs, loglines, consolelogfile = _log_settings_from_server(server) | 76 | _, _, consolelogfile = _log_settings_from_server(server) |
| 81 | 77 | ||
| 82 | # verify and warn | 78 | # verify and warn |
| 83 | build_history_enabled = True | 79 | build_history_enabled = True |
| 84 | inheritlist, error = server.runCommand(["getVariable", "INHERIT"]) | 80 | inheritlist, _ = server.runCommand(["getVariable", "INHERIT"]) |
| 85 | 81 | ||
| 86 | if not "buildhistory" in inheritlist.split(" "): | 82 | if not "buildhistory" in inheritlist.split(" "): |
| 87 | logger.warn("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.") | 83 | logger.warn("buildhistory is not enabled. Please enable INHERIT += \"buildhistory\" to see image details.") |
| @@ -126,12 +122,15 @@ def main(server, eventHandler, params ): | |||
| 126 | 122 | ||
| 127 | helper.eventHandler(event) | 123 | helper.eventHandler(event) |
| 128 | 124 | ||
| 125 | # pylint: disable=protected-access | ||
| 126 | # the code will look into the protected variables of the event; no easy way around this | ||
| 127 | |||
| 129 | if isinstance(event, bb.event.BuildStarted): | 128 | if isinstance(event, bb.event.BuildStarted): |
| 130 | buildinfohelper.store_started_build(event) | 129 | buildinfohelper.store_started_build(event) |
| 131 | 130 | ||
| 132 | if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): | 131 | if isinstance(event, (bb.build.TaskStarted, bb.build.TaskSucceeded, bb.build.TaskFailedSilent)): |
| 133 | buildinfohelper.update_and_store_task(event) | 132 | buildinfohelper.update_and_store_task(event) |
| 134 | logger.warn("Logfile for task %s" % event.logfile) | 133 | logger.warn("Logfile for task %s", event.logfile) |
| 135 | continue | 134 | continue |
| 136 | 135 | ||
| 137 | if isinstance(event, bb.build.TaskBase): | 136 | if isinstance(event, bb.build.TaskBase): |
| @@ -143,17 +142,17 @@ def main(server, eventHandler, params ): | |||
| 143 | 142 | ||
| 144 | if isinstance(event, logging.LogRecord): | 143 | if isinstance(event, logging.LogRecord): |
| 145 | if event.levelno == -1: | 144 | if event.levelno == -1: |
| 146 | event.levelno = format.ERROR | 145 | event.levelno = formatter.ERROR |
| 147 | 146 | ||
| 148 | buildinfohelper.store_log_event(event) | 147 | buildinfohelper.store_log_event(event) |
| 149 | if event.levelno >= format.ERROR: | 148 | if event.levelno >= formatter.ERROR: |
| 150 | errors = errors + 1 | 149 | errors = errors + 1 |
| 151 | elif event.levelno == format.WARNING: | 150 | elif event.levelno == formatter.WARNING: |
| 152 | warnings = warnings + 1 | 151 | warnings = warnings + 1 |
| 153 | # For "normal" logging conditions, don't show note logs from tasks | 152 | # For "normal" logging conditions, don't show note logs from tasks |
| 154 | # but do show them if the user has changed the default log level to | 153 | # but do show them if the user has changed the default log level to |
| 155 | # include verbose/debug messages | 154 | # include verbose/debug messages |
| 156 | if event.taskpid != 0 and event.levelno <= format.NOTE: | 155 | if event.taskpid != 0 and event.levelno <= formatter.NOTE: |
| 157 | continue | 156 | continue |
| 158 | 157 | ||
| 159 | logger.handle(event) | 158 | logger.handle(event) |
| @@ -241,8 +240,8 @@ def main(server, eventHandler, params ): | |||
| 241 | 240 | ||
| 242 | if isinstance(event, (bb.event.BuildCompleted, bb.command.CommandFailed)): | 241 | if isinstance(event, (bb.event.BuildCompleted, bb.command.CommandFailed)): |
| 243 | 242 | ||
| 244 | errorcode = 0 | 243 | errorcode = 0 |
| 245 | if (isinstance(event, bb.command.CommandFailed)): | 244 | if isinstance(event, bb.command.CommandFailed): |
| 246 | errors += 1 | 245 | errors += 1 |
| 247 | errorcode = 1 | 246 | errorcode = 1 |
| 248 | logger.error("Command execution failed: %s", event.error) | 247 | logger.error("Command execution failed: %s", event.error) |
| @@ -251,7 +250,7 @@ def main(server, eventHandler, params ): | |||
| 251 | buildinfohelper.update_build_information(event, errors, warnings, taskfailures) | 250 | buildinfohelper.update_build_information(event, errors, warnings, taskfailures) |
| 252 | buildinfohelper.close(errorcode) | 251 | buildinfohelper.close(errorcode) |
| 253 | # mark the log output; controllers may kill the toasterUI after seeing this log | 252 | # mark the log output; controllers may kill the toasterUI after seeing this log |
| 254 | logger.info("ToasterUI build done 1, brbe: %s" % buildinfohelper.brbe ) | 253 | logger.info("ToasterUI build done 1, brbe: %s", buildinfohelper.brbe ) |
| 255 | 254 | ||
| 256 | # we start a new build info | 255 | # we start a new build info |
| 257 | if buildinfohelper.brbe is not None: | 256 | if buildinfohelper.brbe is not None: |
| @@ -293,7 +292,7 @@ def main(server, eventHandler, params ): | |||
| 293 | elif event.type == "LicenseManifestPath": | 292 | elif event.type == "LicenseManifestPath": |
| 294 | buildinfohelper.store_license_manifest_path(event) | 293 | buildinfohelper.store_license_manifest_path(event) |
| 295 | else: | 294 | else: |
| 296 | logger.error("Unprocessed MetadataEvent %s " % str(event)) | 295 | logger.error("Unprocessed MetadataEvent %s ", str(event)) |
| 297 | continue | 296 | continue |
| 298 | 297 | ||
| 299 | if isinstance(event, bb.cooker.CookerExit): | 298 | if isinstance(event, bb.cooker.CookerExit): |
| @@ -325,30 +324,28 @@ def main(server, eventHandler, params ): | |||
| 325 | pass | 324 | pass |
| 326 | except KeyboardInterrupt: | 325 | except KeyboardInterrupt: |
| 327 | main.shutdown = 1 | 326 | main.shutdown = 1 |
| 328 | pass | ||
| 329 | except Exception as e: | 327 | except Exception as e: |
| 330 | # print errors to log | 328 | # print errors to log |
| 331 | import traceback | 329 | import traceback |
| 332 | from pprint import pformat | 330 | from pprint import pformat |
| 333 | exception_data = traceback.format_exc() | 331 | exception_data = traceback.format_exc() |
| 334 | logger.error("%s\n%s" % (e, exception_data)) | 332 | logger.error("%s\n%s" , e, exception_data) |
| 335 | 333 | ||
| 336 | exc_type, exc_value, tb = sys.exc_info() | 334 | _, _, tb = sys.exc_info() |
| 337 | if tb is not None: | 335 | if tb is not None: |
| 338 | curr = tb | 336 | curr = tb |
| 339 | while curr is not None: | 337 | while curr is not None: |
| 340 | logger.warn("Error data dump %s\n%s\n" % (traceback.format_tb(curr,1), pformat(curr.tb_frame.f_locals))) | 338 | logger.warn("Error data dump %s\n%s\n" , traceback.format_tb(curr,1), pformat(curr.tb_frame.f_locals)) |
| 341 | curr = curr.tb_next | 339 | curr = curr.tb_next |
| 342 | 340 | ||
| 343 | # save them to database, if possible; if it fails, we already logged to console. | 341 | # save them to database, if possible; if it fails, we already logged to console. |
| 344 | try: | 342 | try: |
| 345 | buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data)) | 343 | buildinfohelper.store_log_exception("%s\n%s" % (str(e), exception_data)) |
| 346 | except Exception as ce: | 344 | except Exception as ce: |
| 347 | logger.error("CRITICAL - Failed to to save toaster exception to the database: %s" % str(ce)) | 345 | logger.error("CRITICAL - Failed to to save toaster exception to the database: %s", str(ce)) |
| 348 | 346 | ||
| 349 | # make sure we return with an error | 347 | # make sure we return with an error |
| 350 | return_value += 1 | 348 | return_value += 1 |
| 351 | pass | ||
| 352 | 349 | ||
| 353 | if interrupted: | 350 | if interrupted: |
| 354 | if return_value == 0: | 351 | if return_value == 0: |
