6 files changed, 206 insertions, 53 deletions

diff --git a/bitbake/lib/toaster/toastermain/logs.py b/bitbake/lib/toaster/toastermain/logs.py
new file mode 100644
index 0000000000..62d871963a
--- /dev/null
+++ b/bitbake/lib/toaster/toastermain/logs.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import logging
+import json
+from pathlib import Path
+from django.http import HttpRequest
+
+BUILDDIR = Path(os.environ.get('BUILDDIR', '/tmp'))
+
+def log_api_request(request, response, view, logger_name='api'):
+    """Helper function for LogAPIMixin"""
+
+    repjson = {
+        'view': view,
+        'path': request.path,
+        'method': request.method,
+        'status': response.status_code
+    }
+
+    logger = logging.getLogger(logger_name)
+    logger.info(
+        json.dumps(repjson, indent=4, separators=(", ", " : "))
+    )
+
+
+def log_view_mixin(view):
+    def log_view_request(*args, **kwargs):
+        # get request from args else kwargs
+        request = None
+        if len(args) > 0:
+            for req in args:
+                if isinstance(req, HttpRequest):
+                    request = req
+                    break 
+        elif request is None:
+            request = kwargs.get('request')
+
+        response = view(*args, **kwargs)
+        view_name = 'unknown'
+        if hasattr(request, 'resolver_match'):
+            if hasattr(request.resolver_match, 'view_name'):
+                view_name = request.resolver_match.view_name
+
+        log_api_request(
+            request, response, view_name, 'toaster')
+        return response
+    return log_view_request
+
+
+
+class LogAPIMixin:
+    """Logs API requests
+
+    tested with:
+        - APIView
+        - ModelViewSet
+        - ReadOnlyModelViewSet
+        - GenericAPIView
+
+    Note: you can set `view_name` attribute in View to override get_view_name()
+    """
+
+    def get_view_name(self):
+        if hasattr(self, 'view_name'):
+            return self.view_name
+        return super().get_view_name()
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        log_api_request(request, response, self.get_view_name())
+        return super().finalize_response(request, response, *args, **kwargs)
+
+
+LOGGING_SETTINGS = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'formatters': {
+        'datetime': {
+            'format': '%(asctime)s %(levelname)s %(message)s'
+        },
+        'verbose': {
+            'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}',
+            'datefmt': "%d/%b/%Y %H:%M:%S",
+            'style': '{',
+        },
+        'api': {
+            'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}',
+            'style': '{'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'datetime',
+        },
+        'file_django': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BUILDDIR / 'toaster_logs/django.log',
+            'when': 'D',  # interval type
+            'interval': 1,  # defaults to 1
+            'backupCount': 10,  # how many files to keep
+            'formatter': 'verbose',
+        },
+        'file_api': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BUILDDIR / 'toaster_logs/api.log',
+            'when': 'D',
+            'interval': 1,
+            'backupCount': 10,
+            'formatter': 'verbose',
+        },
+        'file_toaster': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BUILDDIR / 'toaster_logs/web.log',
+            'when': 'D',
+            'interval': 1,
+            'backupCount': 10,
+            'formatter': 'verbose',
+        },
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['file_django', 'console'],
+            'level': 'WARN',
+            'propagate': True,
+        },
+        'django': {
+            'handlers': ['file_django', 'console'],
+            'level': 'WARNING',
+            'propogate': True,
+        },
+        'toaster': {
+            'handlers': ['file_toaster'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+        'api': {
+            'handlers': ['file_api'],
+            'level': 'INFO',
+            'propagate': False,
+        }
+    }
+}
diff --git a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
index 59da6ff7ac..f7139aa041 100644
--- a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
+++ b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
@@ -451,7 +451,7 @@ class Command(BaseCommand):
             # Catch vars relevant to Toaster (in case no Toaster section)
             self.update_project_vars(project,'DISTRO')
             self.update_project_vars(project,'MACHINE')
-            self.update_project_vars(project,'IMAGE_INSTALL_append')
+            self.update_project_vars(project,'IMAGE_INSTALL:append')
             self.update_project_vars(project,'IMAGE_FSTYPES')
             self.update_project_vars(project,'PACKAGE_CLASSES')
             # These vars are typically only assigned by Toaster
@@ -545,7 +545,7 @@ class Command(BaseCommand):
             # Find the directory's release, and promote to default_release if local paths
             release = self.find_import_release(layers_list,lv_dict,default_release)
             # create project, SANITY: reuse any project of same name
-            project = Project.objects.create_project(project_name,release,project)
+            project = Project.objects.create_project(project_name,release,project, imported=True)
             # Apply any new layers or variables
             self.apply_conf_variables(project,layers_list,lv_dict,release)
             # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
diff --git a/bitbake/lib/toaster/toastermain/management/commands/checksocket.py b/bitbake/lib/toaster/toastermain/management/commands/checksocket.py
index 811fd5d516..b2c002da7a 100644
--- a/bitbake/lib/toaster/toastermain/management/commands/checksocket.py
+++ b/bitbake/lib/toaster/toastermain/management/commands/checksocket.py
@@ -13,7 +13,7 @@ import errno
 import socket
 
 from django.core.management.base import BaseCommand, CommandError
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 
 DEFAULT_ADDRPORT = "0.0.0.0:8000"
 
@@ -51,7 +51,7 @@ class Command(BaseCommand):
             if hasattr(err, 'errno') and err.errno in errors:
                 errtext = errors[err.errno]
             else:
-                errtext = force_text(err)
+                errtext = force_str(err)
             raise CommandError(errtext)
 
         self.stdout.write("OK")
diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py
index a4b370c8d4..e06adc5a93 100644
--- a/bitbake/lib/toaster/toastermain/settings.py
+++ b/bitbake/lib/toaster/toastermain/settings.py
@@ -9,6 +9,8 @@
 # Django settings for Toaster project.
 
 import os
+from pathlib import Path
+from toastermain.logs import LOGGING_SETTINGS
 
 DEBUG = True
 
@@ -39,6 +41,9 @@ DATABASES = {
     }
 }
 
+# New in Django 3.2
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
 # Needed when Using sqlite especially to add a longer timeout for waiting
 # for the database lock to be  released
 # https://docs.djangoproject.com/en/1.6/ref/databases/#database-is-locked-errors
@@ -84,14 +89,17 @@ else:
                 from pytz.exceptions import UnknownTimeZoneError
                 try:
                     if pytz.timezone(zonename) is not None:
-                        zonefilelist[hashlib.md5(open(filepath, 'rb').read()).hexdigest()] = zonename
+                        with open(filepath, 'rb') as f:
+                            zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename
                 except UnknownTimeZoneError as ValueError:
                     # we expect timezone failures here, just move over
                     pass
             except ImportError:
-                zonefilelist[hashlib.md5(open(filepath, 'rb').read()).hexdigest()] = zonename
+                with open(filepath, 'rb') as f:
+                    zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename
 
-    TIME_ZONE = zonefilelist[hashlib.md5(open('/etc/localtime', 'rb').read()).hexdigest()]
+    with open('/etc/localtime', 'rb') as f:
+        TIME_ZONE = zonefilelist[hashlib.md5(f.read()).hexdigest()]
 
 # Language code for this installation. All choices can be found here:
 # http://www.i18nguy.com/unicode/language-identifiers.html
@@ -103,10 +111,6 @@ SITE_ID = 1
 # to load the internationalization machinery.
 USE_I18N = True
 
-# If you set this to False, Django will not format dates, numbers and
-# calendars according to the current locale.
-USE_L10N = True
-
 # If you set this to False, Django will not use timezone-aware datetimes.
 USE_TZ = True
 
@@ -147,6 +151,8 @@ STATICFILES_FINDERS = (
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT'
 
+TMPDIR = os.environ.get('TOASTER_DJANGO_TMPDIR', '/tmp')
+
 class InvalidString(str):
     def __mod__(self, other):
         from django.template.base import TemplateSyntaxError
@@ -183,7 +189,13 @@ TEMPLATES = [
                 'django.template.loaders.app_directories.Loader',
                 #'django.template.loaders.eggs.Loader',
             ],
-            'string_if_invalid': InvalidString("%s"),
+            # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled
+            # Generally, string_if_invalid should only be enabled in order to debug
+            # a specific template problem, then cleared once debugging is complete.
+            # If you assign a value other than '' to string_if_invalid,
+            # you will experience rendering problems with these templates and sites.
+            #  'string_if_invalid': InvalidString("%s"),
+            'string_if_invalid': "",
             'debug': DEBUG,
         },
     },
@@ -207,7 +219,7 @@ CACHES = {
     #        },
            'default': {
                'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-               'LOCATION': '/tmp/toaster_cache_%d' % os.getuid(),
+               'LOCATION': '%s/toaster_cache_%d' % (TMPDIR, os.getuid()),
                'TIMEOUT': 1,
             }
           }
@@ -239,6 +251,9 @@ INSTALLED_APPS = (
     'django.contrib.humanize',
     'bldcollector',
     'toastermain',
+
+    # 3rd-lib
+    "log_viewer",
 )
 
 
@@ -299,43 +314,21 @@ for t in os.walk(os.path.dirname(currentdir)):
 # the site admins on every HTTP 500 error when DEBUG=False.
 # See http://docs.djangoproject.com/en/dev/topics/logging for
 # more details on how to customize your logging configuration.
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'filters': {
-        'require_debug_false': {
-            '()': 'django.utils.log.RequireDebugFalse'
-        }
-    },
-    'formatters': {
-        'datetime': {
-            'format': '%(asctime)s %(levelname)s %(message)s'
-        }
-    },
-    'handlers': {
-        'mail_admins': {
-            'level': 'ERROR',
-            'filters': ['require_debug_false'],
-            'class': 'django.utils.log.AdminEmailHandler'
-        },
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'datetime',
-        }
-    },
-    'loggers': {
-        'toaster' : {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-        },
-        'django.request': {
-            'handlers': ['console'],
-            'level': 'WARN',
-            'propagate': True,
-        },
-    }
-}
+LOGGING = LOGGING_SETTINGS
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BUILDDIR = os.environ.get("BUILDDIR", TMPDIR)
+
+# LOG VIEWER
+# https://pypi.org/project/django-log-viewer/
+LOG_VIEWER_FILES_PATTERN = '*.log*'
+LOG_VIEWER_FILES_DIR = os.path.join(BUILDDIR, "toaster_logs/")
+LOG_VIEWER_PAGE_LENGTH = 25      # total log lines per-page
+LOG_VIEWER_MAX_READ_LINES = 100000  # total log lines will be read
+LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']
+
+# Optionally you can set the next variables in order to customize the admin:
+LOG_VIEWER_FILE_LIST_TITLE = "Logs list"
 
 if DEBUG and SQL_DEBUG:
     LOGGING['loggers']['django.db.backends'] = {
diff --git a/bitbake/lib/toaster/toastermain/settings_test.py b/bitbake/lib/toaster/toastermain/settings_test.py
index 6538d9e453..74def2d240 100644
--- a/bitbake/lib/toaster/toastermain/settings_test.py
+++ b/bitbake/lib/toaster/toastermain/settings_test.py
@@ -19,10 +19,10 @@ TEMPLATE_DEBUG = DEBUG
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': '/tmp/toaster-test-db.sqlite',
+        'NAME': '%s/toaster-test-db.sqlite' % TMPDIR,
         'TEST': {
             'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': '/tmp/toaster-test-db.sqlite',
+            'NAME': '%s/toaster-test-db.sqlite' % TMPDIR,
         }
     }
 }
diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py
index 5fb520b384..3be46fcf0c 100644
--- a/bitbake/lib/toaster/toastermain/urls.py
+++ b/bitbake/lib/toaster/toastermain/urls.py
@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from django.conf.urls import include, url
+from django.urls import re_path as url, include
 from django.views.generic import RedirectView, TemplateView
 from django.views.decorators.cache import never_cache
 import bldcollector.views
@@ -28,6 +28,8 @@ urlpatterns = [
     # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
 
 
+    url(r'^logs/', include('log_viewer.urls')),
+
     # This is here to maintain backward compatibility and will be deprecated
     # in the future.
     url(r'^orm/eventfile$', bldcollector.views.eventfile),