diff options
author | Alassane Yattara <alassane.yattara@savoirfairelinux.com> | 2023-10-04 14:44:15 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-10-06 11:42:46 +0100 |
commit | 78b02e1845c0b0ccf75b9de6801798ea4340addc (patch) | |
tree | 205fb4e1e373f4da6613c46375cd339cb606070d /bitbake/lib/toaster/toastermain | |
parent | 3ac4694fc3b85cf23909d7e2fcc4ae97004ae927 (diff) | |
download | poky-78b02e1845c0b0ccf75b9de6801798ea4340addc.tar.gz |
bitbake: toaster: Monitoring - implement Django logging system
(Bitbake rev: 2efb146480ee46c0463d9edb71bf1c03ce15bcf2)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster/toastermain')
-rw-r--r-- | bitbake/lib/toaster/toastermain/logs.py | 153 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastermain/settings.py | 66 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastermain/urls.py | 2 |
3 files changed, 183 insertions, 38 deletions
diff --git a/bitbake/lib/toaster/toastermain/logs.py b/bitbake/lib/toaster/toastermain/logs.py new file mode 100644 index 0000000000..f9953982b7 --- /dev/null +++ b/bitbake/lib/toaster/toastermain/logs.py | |||
@@ -0,0 +1,153 @@ | |||
1 | #!/usr/bin/env python3 | ||
2 | # -*- coding: utf-8 -*- | ||
3 | |||
4 | import logging | ||
5 | import json | ||
6 | from pathlib import Path | ||
7 | from django.http import HttpRequest | ||
8 | |||
9 | BASE_DIR = Path(__file__).resolve(strict=True).parent.parent | ||
10 | |||
11 | |||
12 | def log_api_request(request, response, view, logger_name='api'): | ||
13 | """Helper function for LogAPIMixin""" | ||
14 | |||
15 | repjson = { | ||
16 | 'view': view, | ||
17 | 'path': request.path, | ||
18 | 'method': request.method, | ||
19 | 'status': response.status_code | ||
20 | } | ||
21 | |||
22 | logger = logging.getLogger(logger_name) | ||
23 | logger.info( | ||
24 | json.dumps(repjson, indent=4, separators=(", ", " : ")) | ||
25 | ) | ||
26 | |||
27 | |||
28 | def log_view_mixin(view): | ||
29 | def log_view_request(*args, **kwargs): | ||
30 | # get request from args else kwargs | ||
31 | request = None | ||
32 | if len(args) > 0: | ||
33 | for req in args: | ||
34 | if isinstance(req, HttpRequest): | ||
35 | request = req | ||
36 | break | ||
37 | elif request is None: | ||
38 | request = kwargs.get('request') | ||
39 | |||
40 | response = view(*args, **kwargs) | ||
41 | log_api_request( | ||
42 | request, response, request.resolver_match.view_name, 'toaster') | ||
43 | return response | ||
44 | return log_view_request | ||
45 | |||
46 | |||
47 | |||
48 | class LogAPIMixin: | ||
49 | """Logs API requests | ||
50 | |||
51 | tested with: | ||
52 | - APIView | ||
53 | - ModelViewSet | ||
54 | - ReadOnlyModelViewSet | ||
55 | - GenericAPIView | ||
56 | |||
57 | Note: you can set `view_name` attribute in View to override get_view_name() | ||
58 | """ | ||
59 | |||
60 | def get_view_name(self): | ||
61 | if hasattr(self, 'view_name'): | ||
62 | return self.view_name | ||
63 | return super().get_view_name() | ||
64 | |||
65 | def finalize_response(self, request, response, *args, **kwargs): | ||
66 | log_api_request(request, response, self.get_view_name()) | ||
67 | return super().finalize_response(request, response, *args, **kwargs) | ||
68 | |||
69 | |||
70 | LOGGING_SETTINGS = { | ||
71 | 'version': 1, | ||
72 | 'disable_existing_loggers': False, | ||
73 | 'filters': { | ||
74 | 'require_debug_false': { | ||
75 | '()': 'django.utils.log.RequireDebugFalse' | ||
76 | } | ||
77 | }, | ||
78 | 'formatters': { | ||
79 | 'datetime': { | ||
80 | 'format': '%(asctime)s %(levelname)s %(message)s' | ||
81 | }, | ||
82 | 'verbose': { | ||
83 | 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}', | ||
84 | 'datefmt': "%d/%b/%Y %H:%M:%S", | ||
85 | 'style': '{', | ||
86 | }, | ||
87 | 'api': { | ||
88 | 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}', | ||
89 | 'style': '{' | ||
90 | } | ||
91 | }, | ||
92 | 'handlers': { | ||
93 | 'mail_admins': { | ||
94 | 'level': 'ERROR', | ||
95 | 'filters': ['require_debug_false'], | ||
96 | 'class': 'django.utils.log.AdminEmailHandler' | ||
97 | }, | ||
98 | 'console': { | ||
99 | 'level': 'DEBUG', | ||
100 | 'class': 'logging.StreamHandler', | ||
101 | 'formatter': 'datetime', | ||
102 | }, | ||
103 | 'file_django': { | ||
104 | 'level': 'INFO', | ||
105 | 'class': 'logging.handlers.TimedRotatingFileHandler', | ||
106 | 'filename': BASE_DIR / 'logs/django.log', | ||
107 | 'when': 'D', # interval type | ||
108 | 'interval': 1, # defaults to 1 | ||
109 | 'backupCount': 10, # how many files to keep | ||
110 | 'formatter': 'verbose', | ||
111 | }, | ||
112 | 'file_api': { | ||
113 | 'level': 'INFO', | ||
114 | 'class': 'logging.handlers.TimedRotatingFileHandler', | ||
115 | 'filename': BASE_DIR / 'logs/api.log', | ||
116 | 'when': 'D', | ||
117 | 'interval': 1, | ||
118 | 'backupCount': 10, | ||
119 | 'formatter': 'verbose', | ||
120 | }, | ||
121 | 'file_toaster': { | ||
122 | 'level': 'INFO', | ||
123 | 'class': 'logging.handlers.TimedRotatingFileHandler', | ||
124 | 'filename': BASE_DIR / 'logs/toaster.log', | ||
125 | 'when': 'D', | ||
126 | 'interval': 1, | ||
127 | 'backupCount': 10, | ||
128 | 'formatter': 'verbose', | ||
129 | }, | ||
130 | }, | ||
131 | 'loggers': { | ||
132 | 'django.request': { | ||
133 | 'handlers': ['file_django', 'console'], | ||
134 | 'level': 'WARN', | ||
135 | 'propagate': True, | ||
136 | }, | ||
137 | 'django': { | ||
138 | 'handlers': ['file_django', 'console'], | ||
139 | 'level': 'WARNING', | ||
140 | 'propogate': True, | ||
141 | }, | ||
142 | 'toaster': { | ||
143 | 'handlers': ['file_toaster'], | ||
144 | 'level': 'INFO', | ||
145 | 'propagate': False, | ||
146 | }, | ||
147 | 'api': { | ||
148 | 'handlers': ['file_api'], | ||
149 | 'level': 'INFO', | ||
150 | 'propagate': False, | ||
151 | } | ||
152 | } | ||
153 | } | ||
diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py index 609c85d9d8..b083cf5885 100644 --- a/bitbake/lib/toaster/toastermain/settings.py +++ b/bitbake/lib/toaster/toastermain/settings.py | |||
@@ -9,6 +9,8 @@ | |||
9 | # Django settings for Toaster project. | 9 | # Django settings for Toaster project. |
10 | 10 | ||
11 | import os | 11 | import os |
12 | from pathlib import Path | ||
13 | from toastermain.logs import LOGGING_SETTINGS | ||
12 | 14 | ||
13 | DEBUG = True | 15 | DEBUG = True |
14 | 16 | ||
@@ -186,7 +188,13 @@ TEMPLATES = [ | |||
186 | 'django.template.loaders.app_directories.Loader', | 188 | 'django.template.loaders.app_directories.Loader', |
187 | #'django.template.loaders.eggs.Loader', | 189 | #'django.template.loaders.eggs.Loader', |
188 | ], | 190 | ], |
189 | 'string_if_invalid': InvalidString("%s"), | 191 | # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled |
192 | # Generally, string_if_invalid should only be enabled in order to debug | ||
193 | # a specific template problem, then cleared once debugging is complete. | ||
194 | # If you assign a value other than '' to string_if_invalid, | ||
195 | # you will experience rendering problems with these templates and sites. | ||
196 | # 'string_if_invalid': InvalidString("%s"), | ||
197 | 'string_if_invalid': "", | ||
190 | 'debug': DEBUG, | 198 | 'debug': DEBUG, |
191 | }, | 199 | }, |
192 | }, | 200 | }, |
@@ -242,6 +250,9 @@ INSTALLED_APPS = ( | |||
242 | 'django.contrib.humanize', | 250 | 'django.contrib.humanize', |
243 | 'bldcollector', | 251 | 'bldcollector', |
244 | 'toastermain', | 252 | 'toastermain', |
253 | |||
254 | # 3rd-lib | ||
255 | "log_viewer", | ||
245 | ) | 256 | ) |
246 | 257 | ||
247 | 258 | ||
@@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)): | |||
302 | # the site admins on every HTTP 500 error when DEBUG=False. | 313 | # the site admins on every HTTP 500 error when DEBUG=False. |
303 | # See http://docs.djangoproject.com/en/dev/topics/logging for | 314 | # See http://docs.djangoproject.com/en/dev/topics/logging for |
304 | # more details on how to customize your logging configuration. | 315 | # more details on how to customize your logging configuration. |
305 | LOGGING = { | 316 | LOGGING = LOGGING_SETTINGS |
306 | 'version': 1, | 317 | |
307 | 'disable_existing_loggers': False, | 318 | # Build paths inside the project like this: BASE_DIR / 'subdir'. |
308 | 'filters': { | 319 | BASE_DIR = Path(__file__).resolve(strict=True).parent.parent |
309 | 'require_debug_false': { | 320 | |
310 | '()': 'django.utils.log.RequireDebugFalse' | 321 | # LOG VIEWER |
311 | } | 322 | # https://pypi.org/project/django-log-viewer/ |
312 | }, | 323 | LOG_VIEWER_FILES_PATTERN = '*.log*' |
313 | 'formatters': { | 324 | LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') |
314 | 'datetime': { | 325 | LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page |
315 | 'format': '%(asctime)s %(levelname)s %(message)s' | 326 | LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read |
316 | } | 327 | LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] |
317 | }, | 328 | |
318 | 'handlers': { | 329 | # Optionally you can set the next variables in order to customize the admin: |
319 | 'mail_admins': { | 330 | LOG_VIEWER_FILE_LIST_TITLE = "Logs list" |
320 | 'level': 'ERROR', | 331 | |
321 | 'filters': ['require_debug_false'], | ||
322 | 'class': 'django.utils.log.AdminEmailHandler' | ||
323 | }, | ||
324 | 'console': { | ||
325 | 'level': 'DEBUG', | ||
326 | 'class': 'logging.StreamHandler', | ||
327 | 'formatter': 'datetime', | ||
328 | } | ||
329 | }, | ||
330 | 'loggers': { | ||
331 | 'toaster' : { | ||
332 | 'handlers': ['console'], | ||
333 | 'level': 'DEBUG', | ||
334 | }, | ||
335 | 'django.request': { | ||
336 | 'handlers': ['console'], | ||
337 | 'level': 'WARN', | ||
338 | 'propagate': True, | ||
339 | }, | ||
340 | } | ||
341 | } | ||
342 | 332 | ||
343 | if DEBUG and SQL_DEBUG: | 333 | if DEBUG and SQL_DEBUG: |
344 | LOGGING['loggers']['django.db.backends'] = { | 334 | LOGGING['loggers']['django.db.backends'] = { |
diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py index 0360302668..3be46fcf0c 100644 --- a/bitbake/lib/toaster/toastermain/urls.py +++ b/bitbake/lib/toaster/toastermain/urls.py | |||
@@ -28,6 +28,8 @@ urlpatterns = [ | |||
28 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), | 28 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), |
29 | 29 | ||
30 | 30 | ||
31 | url(r'^logs/', include('log_viewer.urls')), | ||
32 | |||
31 | # This is here to maintain backward compatibility and will be deprecated | 33 | # This is here to maintain backward compatibility and will be deprecated |
32 | # in the future. | 34 | # in the future. |
33 | url(r'^orm/eventfile$', bldcollector.views.eventfile), | 35 | url(r'^orm/eventfile$', bldcollector.views.eventfile), |