diff options
Diffstat (limited to 'bitbake/lib/toaster/toastergui/views.py')
-rw-r--r-- | bitbake/lib/toaster/toastergui/views.py | 218 |
1 files changed, 201 insertions, 17 deletions
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 9a5e48e3bb..40aed265dc 100644 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py | |||
@@ -6,24 +6,36 @@ | |||
6 | # SPDX-License-Identifier: GPL-2.0-only | 6 | # SPDX-License-Identifier: GPL-2.0-only |
7 | # | 7 | # |
8 | 8 | ||
9 | import ast | ||
9 | import re | 10 | import re |
11 | import subprocess | ||
12 | import sys | ||
13 | |||
14 | import bb.cooker | ||
15 | from bb.ui import toasterui | ||
16 | from bb.ui import eventreplay | ||
10 | 17 | ||
11 | from django.db.models import F, Q, Sum | 18 | from django.db.models import F, Q, Sum |
12 | from django.db import IntegrityError | 19 | from django.db import IntegrityError |
13 | from django.shortcuts import render, redirect, get_object_or_404 | 20 | from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect |
14 | from django.utils.http import urlencode | 21 | from django.utils.http import urlencode |
15 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe | 22 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe |
16 | from orm.models import LogMessage, Variable, Package_Dependency, Package | 23 | from orm.models import LogMessage, Variable, Package_Dependency, Package |
17 | from orm.models import Task_Dependency, Package_File | 24 | from orm.models import Task_Dependency, Package_File |
18 | from orm.models import Target_Installed_Package, Target_File | 25 | from orm.models import Target_Installed_Package, Target_File |
19 | from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File | 26 | from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File |
20 | from orm.models import BitbakeVersion, CustomImageRecipe | 27 | from orm.models import BitbakeVersion, CustomImageRecipe, EventLogsImports |
21 | 28 | ||
22 | from django.urls import reverse, resolve | 29 | from django.urls import reverse, resolve |
30 | from django.contrib import messages | ||
31 | |||
23 | from django.core.exceptions import ObjectDoesNotExist | 32 | from django.core.exceptions import ObjectDoesNotExist |
33 | from django.core.files.storage import FileSystemStorage | ||
34 | from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile | ||
24 | from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger | 35 | from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger |
25 | from django.http import HttpResponseNotFound, JsonResponse | 36 | from django.http import HttpResponseNotFound, JsonResponse |
26 | from django.utils import timezone | 37 | from django.utils import timezone |
38 | from django.views.generic import TemplateView | ||
27 | from datetime import timedelta, datetime | 39 | from datetime import timedelta, datetime |
28 | from toastergui.templatetags.projecttags import json as jsonfilter | 40 | from toastergui.templatetags.projecttags import json as jsonfilter |
29 | from decimal import Decimal | 41 | from decimal import Decimal |
@@ -32,13 +44,20 @@ import os | |||
32 | from os.path import dirname | 44 | from os.path import dirname |
33 | import mimetypes | 45 | import mimetypes |
34 | 46 | ||
47 | from toastergui.forms import LoadFileForm | ||
48 | |||
49 | from collections import namedtuple | ||
50 | |||
35 | import logging | 51 | import logging |
36 | 52 | ||
53 | from toastermain.logs import log_view_mixin | ||
54 | |||
37 | logger = logging.getLogger("toaster") | 55 | logger = logging.getLogger("toaster") |
38 | 56 | ||
39 | # Project creation and managed build enable | 57 | # Project creation and managed build enable |
40 | project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER')) | 58 | project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER')) |
41 | is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC')) | 59 | is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC')) |
60 | import_page = False | ||
42 | 61 | ||
43 | class MimeTypeFinder(object): | 62 | class MimeTypeFinder(object): |
44 | # setting this to False enables additional non-standard mimetypes | 63 | # setting this to False enables additional non-standard mimetypes |
@@ -56,6 +75,7 @@ class MimeTypeFinder(object): | |||
56 | return guessed_type | 75 | return guessed_type |
57 | 76 | ||
58 | # single point to add global values into the context before rendering | 77 | # single point to add global values into the context before rendering |
78 | @log_view_mixin | ||
59 | def toaster_render(request, page, context): | 79 | def toaster_render(request, page, context): |
60 | context['project_enable'] = project_enable | 80 | context['project_enable'] = project_enable |
61 | context['project_specific'] = is_project_specific | 81 | context['project_specific'] = is_project_specific |
@@ -665,16 +685,17 @@ def recipe_packages(request, build_id, recipe_id): | |||
665 | return response | 685 | return response |
666 | 686 | ||
667 | from django.http import HttpResponse | 687 | from django.http import HttpResponse |
688 | @log_view_mixin | ||
668 | def xhr_dirinfo(request, build_id, target_id): | 689 | def xhr_dirinfo(request, build_id, target_id): |
669 | top = request.GET.get('start', '/') | 690 | top = request.GET.get('start', '/') |
670 | return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json") | 691 | return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json") |
671 | 692 | ||
672 | from django.utils.functional import Promise | 693 | from django.utils.functional import Promise |
673 | from django.utils.encoding import force_text | 694 | from django.utils.encoding import force_str |
674 | class LazyEncoder(json.JSONEncoder): | 695 | class LazyEncoder(json.JSONEncoder): |
675 | def default(self, obj): | 696 | def default(self, obj): |
676 | if isinstance(obj, Promise): | 697 | if isinstance(obj, Promise): |
677 | return force_text(obj) | 698 | return force_str(obj) |
678 | return super(LazyEncoder, self).default(obj) | 699 | return super(LazyEncoder, self).default(obj) |
679 | 700 | ||
680 | from toastergui.templatetags.projecttags import filtered_filesizeformat | 701 | from toastergui.templatetags.projecttags import filtered_filesizeformat |
@@ -1404,7 +1425,7 @@ if True: | |||
1404 | if not os.path.isdir('%s/conf' % request.POST['importdir']): | 1425 | if not os.path.isdir('%s/conf' % request.POST['importdir']): |
1405 | raise BadParameterException("Bad path or missing 'conf' directory (%s)" % request.POST['importdir']) | 1426 | raise BadParameterException("Bad path or missing 'conf' directory (%s)" % request.POST['importdir']) |
1406 | from django.core import management | 1427 | from django.core import management |
1407 | management.call_command('buildimport', '--command=import', '--name=%s' % request.POST['projectname'], '--path=%s' % request.POST['importdir'], interactive=False) | 1428 | management.call_command('buildimport', '--command=import', '--name=%s' % request.POST['projectname'], '--path=%s' % request.POST['importdir']) |
1408 | prj = Project.objects.get(name = request.POST['projectname']) | 1429 | prj = Project.objects.get(name = request.POST['projectname']) |
1409 | prj.merged_attr = True | 1430 | prj.merged_attr = True |
1410 | prj.save() | 1431 | prj.save() |
@@ -1606,12 +1627,13 @@ if True: | |||
1606 | # make sure we have a machine set for this project | 1627 | # make sure we have a machine set for this project |
1607 | ProjectVariable.objects.get_or_create(project=new_project, | 1628 | ProjectVariable.objects.get_or_create(project=new_project, |
1608 | name="MACHINE", | 1629 | name="MACHINE", |
1609 | value="qemux86") | 1630 | value="qemux86-64") |
1610 | context = {'project': new_project} | 1631 | context = {'project': new_project} |
1611 | return toaster_render(request, "js-unit-tests.html", context) | 1632 | return toaster_render(request, "js-unit-tests.html", context) |
1612 | 1633 | ||
1613 | from django.views.decorators.csrf import csrf_exempt | 1634 | from django.views.decorators.csrf import csrf_exempt |
1614 | @csrf_exempt | 1635 | @csrf_exempt |
1636 | @log_view_mixin | ||
1615 | def xhr_testreleasechange(request, pid): | 1637 | def xhr_testreleasechange(request, pid): |
1616 | def response(data): | 1638 | def response(data): |
1617 | return HttpResponse(jsonfilter(data), | 1639 | return HttpResponse(jsonfilter(data), |
@@ -1648,6 +1670,7 @@ if True: | |||
1648 | except Exception as e: | 1670 | except Exception as e: |
1649 | return response({"error": str(e) }) | 1671 | return response({"error": str(e) }) |
1650 | 1672 | ||
1673 | @log_view_mixin | ||
1651 | def xhr_configvaredit(request, pid): | 1674 | def xhr_configvaredit(request, pid): |
1652 | try: | 1675 | try: |
1653 | prj = Project.objects.get(id = pid) | 1676 | prj = Project.objects.get(id = pid) |
@@ -1683,12 +1706,12 @@ if True: | |||
1683 | t=request.POST['configvarDel'].strip() | 1706 | t=request.POST['configvarDel'].strip() |
1684 | pt = ProjectVariable.objects.get(pk = int(t)).delete() | 1707 | pt = ProjectVariable.objects.get(pk = int(t)).delete() |
1685 | 1708 | ||
1686 | # return all project settings, filter out blacklist and elsewhere-managed variables | 1709 | # return all project settings, filter out disallowed and elsewhere-managed variables |
1687 | vars_managed,vars_fstypes,vars_blacklist = get_project_configvars_context() | 1710 | vars_managed,vars_fstypes,vars_disallowed = get_project_configvars_context() |
1688 | configvars_query = ProjectVariable.objects.filter(project_id = pid).all() | 1711 | configvars_query = ProjectVariable.objects.filter(project_id = pid).all() |
1689 | for var in vars_managed: | 1712 | for var in vars_managed: |
1690 | configvars_query = configvars_query.exclude(name = var) | 1713 | configvars_query = configvars_query.exclude(name = var) |
1691 | for var in vars_blacklist: | 1714 | for var in vars_disallowed: |
1692 | configvars_query = configvars_query.exclude(name = var) | 1715 | configvars_query = configvars_query.exclude(name = var) |
1693 | 1716 | ||
1694 | return_data = { | 1717 | return_data = { |
@@ -1708,7 +1731,7 @@ if True: | |||
1708 | except ProjectVariable.DoesNotExist: | 1731 | except ProjectVariable.DoesNotExist: |
1709 | pass | 1732 | pass |
1710 | try: | 1733 | try: |
1711 | return_data['image_install_append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL_append").value, | 1734 | return_data['image_install:append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL:append").value, |
1712 | except ProjectVariable.DoesNotExist: | 1735 | except ProjectVariable.DoesNotExist: |
1713 | pass | 1736 | pass |
1714 | try: | 1737 | try: |
@@ -1726,6 +1749,7 @@ if True: | |||
1726 | return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json") | 1749 | return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json") |
1727 | 1750 | ||
1728 | 1751 | ||
1752 | @log_view_mixin | ||
1729 | def customrecipe_download(request, pid, recipe_id): | 1753 | def customrecipe_download(request, pid, recipe_id): |
1730 | recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) | 1754 | recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) |
1731 | 1755 | ||
@@ -1781,7 +1805,7 @@ if True: | |||
1781 | 'MACHINE', 'BBLAYERS' | 1805 | 'MACHINE', 'BBLAYERS' |
1782 | } | 1806 | } |
1783 | 1807 | ||
1784 | vars_blacklist = { | 1808 | vars_disallowed = { |
1785 | 'PARALLEL_MAKE','BB_NUMBER_THREADS', | 1809 | 'PARALLEL_MAKE','BB_NUMBER_THREADS', |
1786 | 'BB_DISKMON_DIRS','BB_NUMBER_THREADS','CVS_PROXY_HOST','CVS_PROXY_PORT', | 1810 | 'BB_DISKMON_DIRS','BB_NUMBER_THREADS','CVS_PROXY_HOST','CVS_PROXY_PORT', |
1787 | 'PARALLEL_MAKE','TMPDIR', | 1811 | 'PARALLEL_MAKE','TMPDIR', |
@@ -1790,7 +1814,7 @@ if True: | |||
1790 | 1814 | ||
1791 | vars_fstypes = Target_Image_File.SUFFIXES | 1815 | vars_fstypes = Target_Image_File.SUFFIXES |
1792 | 1816 | ||
1793 | return(vars_managed,sorted(vars_fstypes),vars_blacklist) | 1817 | return(vars_managed,sorted(vars_fstypes),vars_disallowed) |
1794 | 1818 | ||
1795 | def projectconf(request, pid): | 1819 | def projectconf(request, pid): |
1796 | 1820 | ||
@@ -1799,12 +1823,12 @@ if True: | |||
1799 | except Project.DoesNotExist: | 1823 | except Project.DoesNotExist: |
1800 | return HttpResponseNotFound("<h1>Project id " + pid + " is unavailable</h1>") | 1824 | return HttpResponseNotFound("<h1>Project id " + pid + " is unavailable</h1>") |
1801 | 1825 | ||
1802 | # remove blacklist and externally managed varaibles from this list | 1826 | # remove disallowed and externally managed varaibles from this list |
1803 | vars_managed,vars_fstypes,vars_blacklist = get_project_configvars_context() | 1827 | vars_managed,vars_fstypes,vars_disallowed = get_project_configvars_context() |
1804 | configvars = ProjectVariable.objects.filter(project_id = pid).all() | 1828 | configvars = ProjectVariable.objects.filter(project_id = pid).all() |
1805 | for var in vars_managed: | 1829 | for var in vars_managed: |
1806 | configvars = configvars.exclude(name = var) | 1830 | configvars = configvars.exclude(name = var) |
1807 | for var in vars_blacklist: | 1831 | for var in vars_disallowed: |
1808 | configvars = configvars.exclude(name = var) | 1832 | configvars = configvars.exclude(name = var) |
1809 | 1833 | ||
1810 | context = { | 1834 | context = { |
@@ -1812,7 +1836,7 @@ if True: | |||
1812 | 'configvars': configvars, | 1836 | 'configvars': configvars, |
1813 | 'vars_managed': vars_managed, | 1837 | 'vars_managed': vars_managed, |
1814 | 'vars_fstypes': vars_fstypes, | 1838 | 'vars_fstypes': vars_fstypes, |
1815 | 'vars_blacklist': vars_blacklist, | 1839 | 'vars_disallowed': vars_disallowed, |
1816 | } | 1840 | } |
1817 | 1841 | ||
1818 | try: | 1842 | try: |
@@ -1839,7 +1863,7 @@ if True: | |||
1839 | except ProjectVariable.DoesNotExist: | 1863 | except ProjectVariable.DoesNotExist: |
1840 | pass | 1864 | pass |
1841 | try: | 1865 | try: |
1842 | context['image_install_append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL_append").value | 1866 | context['image_install:append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL:append").value |
1843 | context['image_install_append_defined'] = "1" | 1867 | context['image_install_append_defined'] = "1" |
1844 | except ProjectVariable.DoesNotExist: | 1868 | except ProjectVariable.DoesNotExist: |
1845 | pass | 1869 | pass |
@@ -1933,3 +1957,163 @@ if True: | |||
1933 | except (ObjectDoesNotExist, IOError): | 1957 | except (ObjectDoesNotExist, IOError): |
1934 | return toaster_render(request, "unavailable_artifact.html") | 1958 | return toaster_render(request, "unavailable_artifact.html") |
1935 | 1959 | ||
1960 | |||
1961 | class CommandLineBuilds(TemplateView): | ||
1962 | model = EventLogsImports | ||
1963 | template_name = 'command_line_builds.html' | ||
1964 | |||
1965 | def get_context_data(self, **kwargs): | ||
1966 | context = super(CommandLineBuilds, self).get_context_data(**kwargs) | ||
1967 | #get value from BB_DEFAULT_EVENTLOG defined in bitbake.conf | ||
1968 | eventlog = subprocess.check_output(['bitbake-getvar', 'BB_DEFAULT_EVENTLOG', '--value']) | ||
1969 | if eventlog: | ||
1970 | logs_dir = os.path.dirname(eventlog.decode().strip('\n')) | ||
1971 | files = os.listdir(logs_dir) | ||
1972 | imported_files = EventLogsImports.objects.all() | ||
1973 | files_list = [] | ||
1974 | |||
1975 | # Filter files that end with ".json" | ||
1976 | event_files = [] | ||
1977 | for file in files: | ||
1978 | if file.endswith(".json"): | ||
1979 | # because BB_DEFAULT_EVENTLOG is a directory, we need to check if the file is a valid eventlog | ||
1980 | with open("{}/{}".format(logs_dir, file)) as efile: | ||
1981 | content = efile.read() | ||
1982 | if 'allvariables' in content: | ||
1983 | event_files.append(file) | ||
1984 | |||
1985 | #build dict for template using db data | ||
1986 | for event_file in event_files: | ||
1987 | if imported_files.filter(name=event_file): | ||
1988 | files_list.append({ | ||
1989 | 'name': event_file, | ||
1990 | 'imported': True, | ||
1991 | 'build_id': imported_files.filter(name=event_file)[0].build_id, | ||
1992 | 'size': os.path.getsize("{}/{}".format(logs_dir, event_file)) | ||
1993 | }) | ||
1994 | else: | ||
1995 | files_list.append({ | ||
1996 | 'name': event_file, | ||
1997 | 'imported': False, | ||
1998 | 'build_id': None, | ||
1999 | 'size': os.path.getsize("{}/{}".format(logs_dir, event_file)) | ||
2000 | }) | ||
2001 | context['import_all'] = True | ||
2002 | |||
2003 | context['files'] = files_list | ||
2004 | context['dir'] = logs_dir | ||
2005 | else: | ||
2006 | context['files'] = [] | ||
2007 | context['dir'] = '' | ||
2008 | |||
2009 | # enable session variable | ||
2010 | if not self.request.session.get('file'): | ||
2011 | self.request.session['file'] = "" | ||
2012 | |||
2013 | context['form'] = LoadFileForm() | ||
2014 | context['project_enable'] = project_enable | ||
2015 | return context | ||
2016 | |||
2017 | def post(self, request, **kwargs): | ||
2018 | logs_dir = request.POST.get('dir') | ||
2019 | all_files = request.POST.get('all') | ||
2020 | |||
2021 | # check if a build is already in progress | ||
2022 | if Build.objects.filter(outcome=Build.IN_PROGRESS): | ||
2023 | messages.add_message( | ||
2024 | self.request, | ||
2025 | messages.ERROR, | ||
2026 | "A build is already in progress. Please wait for it to complete before starting a new build." | ||
2027 | ) | ||
2028 | return JsonResponse({'response': 'building'}) | ||
2029 | imported_files = EventLogsImports.objects.all() | ||
2030 | try: | ||
2031 | if all_files == 'true': | ||
2032 | # use of session variable to deactivate icon for builds in progress | ||
2033 | request.session['all_builds'] = True | ||
2034 | request.session.modified = True | ||
2035 | request.session.save() | ||
2036 | |||
2037 | files = ast.literal_eval(request.POST.get('file')) | ||
2038 | for file in files: | ||
2039 | if imported_files.filter(name=file.get('name')).exists(): | ||
2040 | imported_files.filter(name=file.get('name'))[0].imported = True | ||
2041 | else: | ||
2042 | with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile: | ||
2043 | # load variables from the first line | ||
2044 | variables = None | ||
2045 | while line := eventfile.readline().strip(): | ||
2046 | try: | ||
2047 | variables = json.loads(line)['allvariables'] | ||
2048 | break | ||
2049 | except (KeyError, json.JSONDecodeError): | ||
2050 | continue | ||
2051 | if not variables: | ||
2052 | raise Exception("File content missing build variables") | ||
2053 | eventfile.seek(0) | ||
2054 | params = namedtuple('ConfigParams', ['observe_only'])(True) | ||
2055 | player = eventreplay.EventPlayer(eventfile, variables) | ||
2056 | |||
2057 | toasterui.main(player, player, params) | ||
2058 | event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True) | ||
2059 | event_log_import.build_id = Build.objects.last().id | ||
2060 | event_log_import.save() | ||
2061 | else: | ||
2062 | if self.request.FILES.get('eventlog_file'): | ||
2063 | file = self.request.FILES['eventlog_file'] | ||
2064 | else: | ||
2065 | file = request.POST.get('file') | ||
2066 | # use of session variable to deactivate icon for build in progress | ||
2067 | request.session['file'] = file | ||
2068 | request.session['all_builds'] = False | ||
2069 | request.session.modified = True | ||
2070 | request.session.save() | ||
2071 | |||
2072 | if imported_files.filter(name=file).exists(): | ||
2073 | imported_files.filter(name=file)[0].imported = True | ||
2074 | else: | ||
2075 | if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile): | ||
2076 | variables = None | ||
2077 | while line := file.readline().strip(): | ||
2078 | try: | ||
2079 | variables = json.loads(line)['allvariables'] | ||
2080 | break | ||
2081 | except (KeyError, json.JSONDecodeError): | ||
2082 | continue | ||
2083 | if not variables: | ||
2084 | raise Exception("File content missing build variables") | ||
2085 | file.seek(0) | ||
2086 | params = namedtuple('ConfigParams', ['observe_only'])(True) | ||
2087 | player = eventreplay.EventPlayer(file, variables) | ||
2088 | if not os.path.exists('{}/{}'.format(logs_dir, file.name)): | ||
2089 | fs = FileSystemStorage(location=logs_dir) | ||
2090 | fs.save(file.name, file) | ||
2091 | toasterui.main(player, player, params) | ||
2092 | else: | ||
2093 | with open("{}/{}".format(logs_dir, file)) as eventfile: | ||
2094 | # load variables from the first line | ||
2095 | variables = None | ||
2096 | while line := eventfile.readline().strip(): | ||
2097 | try: | ||
2098 | variables = json.loads(line)['allvariables'] | ||
2099 | break | ||
2100 | except (KeyError, json.JSONDecodeError): | ||
2101 | continue | ||
2102 | if not variables: | ||
2103 | raise Exception("File content missing build variables") | ||
2104 | eventfile.seek(0) | ||
2105 | params = namedtuple('ConfigParams', ['observe_only'])(True) | ||
2106 | player = eventreplay.EventPlayer(eventfile, variables) | ||
2107 | toasterui.main(player, player, params) | ||
2108 | event_log_import = EventLogsImports.objects.create(name=file, imported=True) | ||
2109 | event_log_import.build_id = Build.objects.last().id | ||
2110 | event_log_import.save() | ||
2111 | request.session['file'] = "" | ||
2112 | except Exception: | ||
2113 | messages.add_message( | ||
2114 | self.request, | ||
2115 | messages.ERROR, | ||
2116 | "The file content is not in the correct format. Update file content or upload a different file." | ||
2117 | ) | ||
2118 | return HttpResponseRedirect("/toastergui/cmdline/") | ||
2119 | return HttpResponseRedirect('/toastergui/builds/') | ||