diff options
author | Marlon Rodriguez Garcia <marlon.rodriguez-garcia@savoirfairelinux.com> | 2023-12-11 11:47:05 -0500 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-12-12 15:58:57 +0000 |
commit | df5c8d6471bf2484db61c7f180c9758fad4182e1 (patch) | |
tree | b27ed4efa0ec97ecee080e5ae887a6e7488cd442 /bitbake/lib/toaster/toastergui/views.py | |
parent | 4bb222e0d71a4cb159b8a4f1a90b65b1af32ac10 (diff) | |
download | poky-df5c8d6471bf2484db61c7f180c9758fad4182e1.tar.gz |
bitbake: toaster: Added new feature to import eventlogs from command line into toaster using replay functionality
Added a new button on the base template to access a new template.
Added a model register the information on the builds and generate access links
Added a form to include the option to load specific files
Added jquery and ajax functions to block screen and redirect to build page when import eventlogs is trigger
Added a new button on landing page linked to import build page, and set min-height of buttons in landing page for uniformity
Removed test assertion to check command line build in content, because new button contains text
Updated toaster_eventreplay to use library
Fix test in test_layerdetails_page
Rebased from master
This feature uses the value from the variable BB_DEFAULT_EVENTLOG to read the files created by bitbake
Exclude listing of files that don't contain the allvariables definitions used to replay builds
This part of the feature should be revisited. Over a long period of time, the BB_DEFAULT_EVENTLOG
will exponentially increase the size of the log file and cause bottlenecks when importing.
(Bitbake rev: ab96cafe03d8bab33c1de09602cc62bd6974f157)
Signed-off-by: Marlon Rodriguez Garcia <marlon.rodriguez-garcia@savoirfairelinux.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster/toastergui/views.py')
-rw-r--r-- | bitbake/lib/toaster/toastergui/views.py | 173 |
1 files changed, 171 insertions, 2 deletions
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 735d304ad8..3b5b9f5bd9 100644 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py | |||
@@ -6,24 +6,36 @@ | |||
6 | # SPDX-License-Identifier: GPL-2.0-only | 6 | # SPDX-License-Identifier: GPL-2.0-only |
7 | # | 7 | # |
8 | 8 | ||
9 | import ast | ||
9 | import re | 10 | import re |
11 | import subprocess | ||
12 | import sys | ||
13 | |||
14 | import bb.cooker | ||
15 | from bb.ui import toasterui | ||
16 | from bb.ui import eventreplay | ||
10 | 17 | ||
11 | from django.db.models import F, Q, Sum | 18 | from django.db.models import F, Q, Sum |
12 | from django.db import IntegrityError | 19 | from django.db import IntegrityError |
13 | from django.shortcuts import render, redirect, get_object_or_404 | 20 | from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect |
14 | from django.utils.http import urlencode | 21 | from django.utils.http import urlencode |
15 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe | 22 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe |
16 | from orm.models import LogMessage, Variable, Package_Dependency, Package | 23 | from orm.models import LogMessage, Variable, Package_Dependency, Package |
17 | from orm.models import Task_Dependency, Package_File | 24 | from orm.models import Task_Dependency, Package_File |
18 | from orm.models import Target_Installed_Package, Target_File | 25 | from orm.models import Target_Installed_Package, Target_File |
19 | from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File | 26 | from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File |
20 | from orm.models import BitbakeVersion, CustomImageRecipe | 27 | from orm.models import BitbakeVersion, CustomImageRecipe, EventLogsImports |
21 | 28 | ||
22 | from django.urls import reverse, resolve | 29 | from django.urls import reverse, resolve |
30 | from django.contrib import messages | ||
31 | |||
23 | from django.core.exceptions import ObjectDoesNotExist | 32 | from django.core.exceptions import ObjectDoesNotExist |
33 | from django.core.files.storage import FileSystemStorage | ||
34 | from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile | ||
24 | from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger | 35 | from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger |
25 | from django.http import HttpResponseNotFound, JsonResponse | 36 | from django.http import HttpResponseNotFound, JsonResponse |
26 | from django.utils import timezone | 37 | from django.utils import timezone |
38 | from django.views.generic import TemplateView | ||
27 | from datetime import timedelta, datetime | 39 | from datetime import timedelta, datetime |
28 | from toastergui.templatetags.projecttags import json as jsonfilter | 40 | from toastergui.templatetags.projecttags import json as jsonfilter |
29 | from decimal import Decimal | 41 | from decimal import Decimal |
@@ -32,6 +44,10 @@ import os | |||
32 | from os.path import dirname | 44 | from os.path import dirname |
33 | import mimetypes | 45 | import mimetypes |
34 | 46 | ||
47 | from toastergui.forms import LoadFileForm | ||
48 | |||
49 | from collections import namedtuple | ||
50 | |||
35 | import logging | 51 | import logging |
36 | 52 | ||
37 | from toastermain.logs import log_view_mixin | 53 | from toastermain.logs import log_view_mixin |
@@ -41,6 +57,7 @@ logger = logging.getLogger("toaster") | |||
41 | # Project creation and managed build enable | 57 | # Project creation and managed build enable |
42 | project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER')) | 58 | project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER')) |
43 | is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC')) | 59 | is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC')) |
60 | import_page = False | ||
44 | 61 | ||
45 | class MimeTypeFinder(object): | 62 | class MimeTypeFinder(object): |
46 | # setting this to False enables additional non-standard mimetypes | 63 | # setting this to False enables additional non-standard mimetypes |
@@ -1940,3 +1957,155 @@ if True: | |||
1940 | except (ObjectDoesNotExist, IOError): | 1957 | except (ObjectDoesNotExist, IOError): |
1941 | return toaster_render(request, "unavailable_artifact.html") | 1958 | return toaster_render(request, "unavailable_artifact.html") |
1942 | 1959 | ||
1960 | |||
1961 | class CommandLineBuilds(TemplateView): | ||
1962 | model = EventLogsImports | ||
1963 | template_name = 'command_line_builds.html' | ||
1964 | |||
1965 | def get_context_data(self, **kwargs): | ||
1966 | context = super(CommandLineBuilds, self).get_context_data(**kwargs) | ||
1967 | #get value from BB_DEFAULT_EVENTLOG defined in bitbake.conf | ||
1968 | eventlog = subprocess.check_output(['bitbake-getvar', 'BB_DEFAULT_EVENTLOG', '--value']) | ||
1969 | if eventlog: | ||
1970 | logs_dir = os.path.dirname(eventlog.decode().strip('\n')) | ||
1971 | files = os.listdir(logs_dir) | ||
1972 | imported_files = EventLogsImports.objects.all() | ||
1973 | files_list = [] | ||
1974 | |||
1975 | # Filter files that end with ".json" | ||
1976 | event_files = [] | ||
1977 | for file in files: | ||
1978 | if file.endswith(".json"): | ||
1979 | # because BB_DEFAULT_EVENTLOG is a directory, we need to check if the file is a valid eventlog | ||
1980 | with open("{}/{}".format(logs_dir, file)) as efile: | ||
1981 | content = efile.read() | ||
1982 | if 'allvariables' in content: | ||
1983 | event_files.append(file) | ||
1984 | |||
1985 | #build dict for template using db data | ||
1986 | for event_file in event_files: | ||
1987 | if imported_files.filter(name=event_file): | ||
1988 | files_list.append({ | ||
1989 | 'name': event_file, | ||
1990 | 'imported': True, | ||
1991 | 'build_id': imported_files.filter(name=event_file)[0].build_id, | ||
1992 | 'size': os.path.getsize("{}/{}".format(logs_dir, event_file)) | ||
1993 | }) | ||
1994 | else: | ||
1995 | files_list.append({ | ||
1996 | 'name': event_file, | ||
1997 | 'imported': False, | ||
1998 | 'build_id': None, | ||
1999 | 'size': os.path.getsize("{}/{}".format(logs_dir, event_file)) | ||
2000 | }) | ||
2001 | context['import_all'] = True | ||
2002 | |||
2003 | context['files'] = files_list | ||
2004 | context['dir'] = logs_dir | ||
2005 | else: | ||
2006 | context['files'] = [] | ||
2007 | context['dir'] = '' | ||
2008 | |||
2009 | # enable session variable | ||
2010 | if not self.request.session.get('file'): | ||
2011 | self.request.session['file'] = "" | ||
2012 | |||
2013 | context['form'] = LoadFileForm() | ||
2014 | context['project_enable'] = project_enable | ||
2015 | return context | ||
2016 | |||
2017 | def post(self, request, **kwargs): | ||
2018 | logs_dir = request.POST.get('dir') | ||
2019 | all_files = request.POST.get('all') | ||
2020 | |||
2021 | imported_files = EventLogsImports.objects.all() | ||
2022 | try: | ||
2023 | if all_files == 'true': | ||
2024 | # use of session variable to deactivate icon for builds in progress | ||
2025 | request.session['all_builds'] = True | ||
2026 | request.session.modified = True | ||
2027 | request.session.save() | ||
2028 | |||
2029 | files = ast.literal_eval(request.POST.get('file')) | ||
2030 | for file in files: | ||
2031 | if imported_files.filter(name=file.get('name')).exists(): | ||
2032 | imported_files.filter(name=file.get('name'))[0].imported = True | ||
2033 | else: | ||
2034 | with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile: | ||
2035 | # load variables from the first line | ||
2036 | variables = None | ||
2037 | while line := eventfile.readline().strip(): | ||
2038 | try: | ||
2039 | variables = json.loads(line)['allvariables'] | ||
2040 | break | ||
2041 | except (KeyError, json.JSONDecodeError): | ||
2042 | continue | ||
2043 | if not variables: | ||
2044 | raise Exception("File content missing build variables") | ||
2045 | eventfile.seek(0) | ||
2046 | params = namedtuple('ConfigParams', ['observe_only'])(True) | ||
2047 | player = eventreplay.EventPlayer(eventfile, variables) | ||
2048 | |||
2049 | toasterui.main(player, player, params) | ||
2050 | event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True) | ||
2051 | event_log_import.build_id = Build.objects.last().id | ||
2052 | event_log_import.save() | ||
2053 | else: | ||
2054 | if self.request.FILES.get('eventlog_file'): | ||
2055 | file = self.request.FILES['eventlog_file'] | ||
2056 | else: | ||
2057 | file = request.POST.get('file') | ||
2058 | # use of session variable to deactivate icon for build in progress | ||
2059 | request.session['file'] = file | ||
2060 | request.session['all_builds'] = False | ||
2061 | request.session.modified = True | ||
2062 | request.session.save() | ||
2063 | |||
2064 | if imported_files.filter(name=file).exists(): | ||
2065 | imported_files.filter(name=file)[0].imported = True | ||
2066 | else: | ||
2067 | if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile): | ||
2068 | variables = None | ||
2069 | while line := file.readline().strip(): | ||
2070 | try: | ||
2071 | variables = json.loads(line)['allvariables'] | ||
2072 | break | ||
2073 | except (KeyError, json.JSONDecodeError): | ||
2074 | continue | ||
2075 | if not variables: | ||
2076 | raise Exception("File content missing build variables") | ||
2077 | file.seek(0) | ||
2078 | params = namedtuple('ConfigParams', ['observe_only'])(True) | ||
2079 | player = eventreplay.EventPlayer(file, variables) | ||
2080 | if not os.path.exists('{}/{}'.format(logs_dir, file.name)): | ||
2081 | fs = FileSystemStorage(location=logs_dir) | ||
2082 | fs.save(file.name, file) | ||
2083 | toasterui.main(player, player, params) | ||
2084 | else: | ||
2085 | with open("{}/{}".format(logs_dir, file)) as eventfile: | ||
2086 | # load variables from the first line | ||
2087 | variables = None | ||
2088 | while line := eventfile.readline().strip(): | ||
2089 | try: | ||
2090 | variables = json.loads(line)['allvariables'] | ||
2091 | break | ||
2092 | except (KeyError, json.JSONDecodeError): | ||
2093 | continue | ||
2094 | if not variables: | ||
2095 | raise Exception("File content missing build variables") | ||
2096 | eventfile.seek(0) | ||
2097 | params = namedtuple('ConfigParams', ['observe_only'])(True) | ||
2098 | player = eventreplay.EventPlayer(eventfile, variables) | ||
2099 | toasterui.main(player, player, params) | ||
2100 | event_log_import = EventLogsImports.objects.create(name=file, imported=True) | ||
2101 | event_log_import.build_id = Build.objects.last().id | ||
2102 | event_log_import.save() | ||
2103 | request.session['file'] = "" | ||
2104 | except Exception: | ||
2105 | messages.add_message( | ||
2106 | self.request, | ||
2107 | messages.ERROR, | ||
2108 | "The file content is not in the correct format. Update file content or upload a different file." | ||
2109 | ) | ||
2110 | return HttpResponseRedirect("/toastergui/cmdline/") | ||
2111 | return HttpResponseRedirect('/toastergui/builds/') | ||