diff options
author | Alassane Yattara <alassane.yattara@savoirfairelinux.com> | 2023-10-04 14:44:15 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-10-06 11:42:46 +0100 |
commit | 78b02e1845c0b0ccf75b9de6801798ea4340addc (patch) | |
tree | 205fb4e1e373f4da6613c46375cd339cb606070d | |
parent | 3ac4694fc3b85cf23909d7e2fcc4ae97004ae927 (diff) | |
download | poky-78b02e1845c0b0ccf75b9de6801798ea4340addc.tar.gz |
bitbake: toaster: Monitoring - implement Django logging system
(Bitbake rev: 2efb146480ee46c0463d9edb71bf1c03ce15bcf2)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | bitbake/lib/toaster/bldcollector/views.py | 3 | ||||
-rw-r--r-- | bitbake/lib/toaster/logs/.gitignore | 1 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastergui/views.py | 7 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastergui/widgets.py | 4 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastermain/logs.py | 153 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastermain/settings.py | 66 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastermain/urls.py | 2 |
7 files changed, 198 insertions, 38 deletions
diff --git a/bitbake/lib/toaster/bldcollector/views.py b/bitbake/lib/toaster/bldcollector/views.py index 04cd8b3dd4..bdf38ae6e8 100644 --- a/bitbake/lib/toaster/bldcollector/views.py +++ b/bitbake/lib/toaster/bldcollector/views.py | |||
@@ -14,8 +14,11 @@ import subprocess | |||
14 | import toastermain | 14 | import toastermain |
15 | from django.views.decorators.csrf import csrf_exempt | 15 | from django.views.decorators.csrf import csrf_exempt |
16 | 16 | ||
17 | from toastermain.logs import log_view_mixin | ||
18 | |||
17 | 19 | ||
18 | @csrf_exempt | 20 | @csrf_exempt |
21 | @log_view_mixin | ||
19 | def eventfile(request): | 22 | def eventfile(request): |
20 | """ Receives a file by POST, and runs toaster-eventreply on this file """ | 23 | """ Receives a file by POST, and runs toaster-eventreply on this file """ |
21 | if request.method != "POST": | 24 | if request.method != "POST": |
diff --git a/bitbake/lib/toaster/logs/.gitignore b/bitbake/lib/toaster/logs/.gitignore new file mode 100644 index 0000000000..e5ebf25a49 --- /dev/null +++ b/bitbake/lib/toaster/logs/.gitignore | |||
@@ -0,0 +1 @@ | |||
*.log* | |||
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 552ff1649b..cc8517ba6c 100644 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py | |||
@@ -34,6 +34,8 @@ import mimetypes | |||
34 | 34 | ||
35 | import logging | 35 | import logging |
36 | 36 | ||
37 | from toastermain.logs import log_view_mixin | ||
38 | |||
37 | logger = logging.getLogger("toaster") | 39 | logger = logging.getLogger("toaster") |
38 | 40 | ||
39 | # Project creation and managed build enable | 41 | # Project creation and managed build enable |
@@ -56,6 +58,7 @@ class MimeTypeFinder(object): | |||
56 | return guessed_type | 58 | return guessed_type |
57 | 59 | ||
58 | # single point to add global values into the context before rendering | 60 | # single point to add global values into the context before rendering |
61 | @log_view_mixin | ||
59 | def toaster_render(request, page, context): | 62 | def toaster_render(request, page, context): |
60 | context['project_enable'] = project_enable | 63 | context['project_enable'] = project_enable |
61 | context['project_specific'] = is_project_specific | 64 | context['project_specific'] = is_project_specific |
@@ -665,6 +668,7 @@ def recipe_packages(request, build_id, recipe_id): | |||
665 | return response | 668 | return response |
666 | 669 | ||
667 | from django.http import HttpResponse | 670 | from django.http import HttpResponse |
671 | @log_view_mixin | ||
668 | def xhr_dirinfo(request, build_id, target_id): | 672 | def xhr_dirinfo(request, build_id, target_id): |
669 | top = request.GET.get('start', '/') | 673 | top = request.GET.get('start', '/') |
670 | return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json") | 674 | return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json") |
@@ -1612,6 +1616,7 @@ if True: | |||
1612 | 1616 | ||
1613 | from django.views.decorators.csrf import csrf_exempt | 1617 | from django.views.decorators.csrf import csrf_exempt |
1614 | @csrf_exempt | 1618 | @csrf_exempt |
1619 | @log_view_mixin | ||
1615 | def xhr_testreleasechange(request, pid): | 1620 | def xhr_testreleasechange(request, pid): |
1616 | def response(data): | 1621 | def response(data): |
1617 | return HttpResponse(jsonfilter(data), | 1622 | return HttpResponse(jsonfilter(data), |
@@ -1648,6 +1653,7 @@ if True: | |||
1648 | except Exception as e: | 1653 | except Exception as e: |
1649 | return response({"error": str(e) }) | 1654 | return response({"error": str(e) }) |
1650 | 1655 | ||
1656 | @log_view_mixin | ||
1651 | def xhr_configvaredit(request, pid): | 1657 | def xhr_configvaredit(request, pid): |
1652 | try: | 1658 | try: |
1653 | prj = Project.objects.get(id = pid) | 1659 | prj = Project.objects.get(id = pid) |
@@ -1726,6 +1732,7 @@ if True: | |||
1726 | return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json") | 1732 | return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json") |
1727 | 1733 | ||
1728 | 1734 | ||
1735 | @log_view_mixin | ||
1729 | def customrecipe_download(request, pid, recipe_id): | 1736 | def customrecipe_download(request, pid, recipe_id): |
1730 | recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) | 1737 | recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) |
1731 | 1738 | ||
diff --git a/bitbake/lib/toaster/toastergui/widgets.py b/bitbake/lib/toaster/toastergui/widgets.py index 6e87285c8c..b32abf40b3 100644 --- a/bitbake/lib/toaster/toastergui/widgets.py +++ b/bitbake/lib/toaster/toastergui/widgets.py | |||
@@ -32,6 +32,7 @@ import re | |||
32 | import os | 32 | import os |
33 | 33 | ||
34 | from toastergui.tablefilter import TableFilterMap | 34 | from toastergui.tablefilter import TableFilterMap |
35 | from toastermain.logs import log_view_mixin | ||
35 | 36 | ||
36 | try: | 37 | try: |
37 | from urllib import unquote_plus | 38 | from urllib import unquote_plus |
@@ -84,6 +85,7 @@ class ToasterTable(TemplateView): | |||
84 | 85 | ||
85 | return context | 86 | return context |
86 | 87 | ||
88 | @log_view_mixin | ||
87 | def get(self, request, *args, **kwargs): | 89 | def get(self, request, *args, **kwargs): |
88 | if request.GET.get('format', None) == 'json': | 90 | if request.GET.get('format', None) == 'json': |
89 | 91 | ||
@@ -415,6 +417,7 @@ class ToasterTypeAhead(View): | |||
415 | def __init__(self, *args, **kwargs): | 417 | def __init__(self, *args, **kwargs): |
416 | super(ToasterTypeAhead, self).__init__() | 418 | super(ToasterTypeAhead, self).__init__() |
417 | 419 | ||
420 | @log_view_mixin | ||
418 | def get(self, request, *args, **kwargs): | 421 | def get(self, request, *args, **kwargs): |
419 | def response(data): | 422 | def response(data): |
420 | return HttpResponse(json.dumps(data, | 423 | return HttpResponse(json.dumps(data, |
@@ -470,6 +473,7 @@ class MostRecentBuildsView(View): | |||
470 | 473 | ||
471 | return False | 474 | return False |
472 | 475 | ||
476 | @log_view_mixin | ||
473 | def get(self, request, *args, **kwargs): | 477 | def get(self, request, *args, **kwargs): |
474 | """ | 478 | """ |
475 | Returns a list of builds in JSON format. | 479 | Returns a list of builds in JSON format. |
diff --git a/bitbake/lib/toaster/toastermain/logs.py b/bitbake/lib/toaster/toastermain/logs.py new file mode 100644 index 0000000000..f9953982b7 --- /dev/null +++ b/bitbake/lib/toaster/toastermain/logs.py | |||
@@ -0,0 +1,153 @@ | |||
1 | #!/usr/bin/env python3 | ||
2 | # -*- coding: utf-8 -*- | ||
3 | |||
4 | import logging | ||
5 | import json | ||
6 | from pathlib import Path | ||
7 | from django.http import HttpRequest | ||
8 | |||
9 | BASE_DIR = Path(__file__).resolve(strict=True).parent.parent | ||
10 | |||
11 | |||
12 | def log_api_request(request, response, view, logger_name='api'): | ||
13 | """Helper function for LogAPIMixin""" | ||
14 | |||
15 | repjson = { | ||
16 | 'view': view, | ||
17 | 'path': request.path, | ||
18 | 'method': request.method, | ||
19 | 'status': response.status_code | ||
20 | } | ||
21 | |||
22 | logger = logging.getLogger(logger_name) | ||
23 | logger.info( | ||
24 | json.dumps(repjson, indent=4, separators=(", ", " : ")) | ||
25 | ) | ||
26 | |||
27 | |||
28 | def log_view_mixin(view): | ||
29 | def log_view_request(*args, **kwargs): | ||
30 | # get request from args else kwargs | ||
31 | request = None | ||
32 | if len(args) > 0: | ||
33 | for req in args: | ||
34 | if isinstance(req, HttpRequest): | ||
35 | request = req | ||
36 | break | ||
37 | elif request is None: | ||
38 | request = kwargs.get('request') | ||
39 | |||
40 | response = view(*args, **kwargs) | ||
41 | log_api_request( | ||
42 | request, response, request.resolver_match.view_name, 'toaster') | ||
43 | return response | ||
44 | return log_view_request | ||
45 | |||
46 | |||
47 | |||
48 | class LogAPIMixin: | ||
49 | """Logs API requests | ||
50 | |||
51 | tested with: | ||
52 | - APIView | ||
53 | - ModelViewSet | ||
54 | - ReadOnlyModelViewSet | ||
55 | - GenericAPIView | ||
56 | |||
57 | Note: you can set `view_name` attribute in View to override get_view_name() | ||
58 | """ | ||
59 | |||
60 | def get_view_name(self): | ||
61 | if hasattr(self, 'view_name'): | ||
62 | return self.view_name | ||
63 | return super().get_view_name() | ||
64 | |||
65 | def finalize_response(self, request, response, *args, **kwargs): | ||
66 | log_api_request(request, response, self.get_view_name()) | ||
67 | return super().finalize_response(request, response, *args, **kwargs) | ||
68 | |||
69 | |||
70 | LOGGING_SETTINGS = { | ||
71 | 'version': 1, | ||
72 | 'disable_existing_loggers': False, | ||
73 | 'filters': { | ||
74 | 'require_debug_false': { | ||
75 | '()': 'django.utils.log.RequireDebugFalse' | ||
76 | } | ||
77 | }, | ||
78 | 'formatters': { | ||
79 | 'datetime': { | ||
80 | 'format': '%(asctime)s %(levelname)s %(message)s' | ||
81 | }, | ||
82 | 'verbose': { | ||
83 | 'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}', | ||
84 | 'datefmt': "%d/%b/%Y %H:%M:%S", | ||
85 | 'style': '{', | ||
86 | }, | ||
87 | 'api': { | ||
88 | 'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}', | ||
89 | 'style': '{' | ||
90 | } | ||
91 | }, | ||
92 | 'handlers': { | ||
93 | 'mail_admins': { | ||
94 | 'level': 'ERROR', | ||
95 | 'filters': ['require_debug_false'], | ||
96 | 'class': 'django.utils.log.AdminEmailHandler' | ||
97 | }, | ||
98 | 'console': { | ||
99 | 'level': 'DEBUG', | ||
100 | 'class': 'logging.StreamHandler', | ||
101 | 'formatter': 'datetime', | ||
102 | }, | ||
103 | 'file_django': { | ||
104 | 'level': 'INFO', | ||
105 | 'class': 'logging.handlers.TimedRotatingFileHandler', | ||
106 | 'filename': BASE_DIR / 'logs/django.log', | ||
107 | 'when': 'D', # interval type | ||
108 | 'interval': 1, # defaults to 1 | ||
109 | 'backupCount': 10, # how many files to keep | ||
110 | 'formatter': 'verbose', | ||
111 | }, | ||
112 | 'file_api': { | ||
113 | 'level': 'INFO', | ||
114 | 'class': 'logging.handlers.TimedRotatingFileHandler', | ||
115 | 'filename': BASE_DIR / 'logs/api.log', | ||
116 | 'when': 'D', | ||
117 | 'interval': 1, | ||
118 | 'backupCount': 10, | ||
119 | 'formatter': 'verbose', | ||
120 | }, | ||
121 | 'file_toaster': { | ||
122 | 'level': 'INFO', | ||
123 | 'class': 'logging.handlers.TimedRotatingFileHandler', | ||
124 | 'filename': BASE_DIR / 'logs/toaster.log', | ||
125 | 'when': 'D', | ||
126 | 'interval': 1, | ||
127 | 'backupCount': 10, | ||
128 | 'formatter': 'verbose', | ||
129 | }, | ||
130 | }, | ||
131 | 'loggers': { | ||
132 | 'django.request': { | ||
133 | 'handlers': ['file_django', 'console'], | ||
134 | 'level': 'WARN', | ||
135 | 'propagate': True, | ||
136 | }, | ||
137 | 'django': { | ||
138 | 'handlers': ['file_django', 'console'], | ||
139 | 'level': 'WARNING', | ||
140 | 'propogate': True, | ||
141 | }, | ||
142 | 'toaster': { | ||
143 | 'handlers': ['file_toaster'], | ||
144 | 'level': 'INFO', | ||
145 | 'propagate': False, | ||
146 | }, | ||
147 | 'api': { | ||
148 | 'handlers': ['file_api'], | ||
149 | 'level': 'INFO', | ||
150 | 'propagate': False, | ||
151 | } | ||
152 | } | ||
153 | } | ||
diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py index 609c85d9d8..b083cf5885 100644 --- a/bitbake/lib/toaster/toastermain/settings.py +++ b/bitbake/lib/toaster/toastermain/settings.py | |||
@@ -9,6 +9,8 @@ | |||
9 | # Django settings for Toaster project. | 9 | # Django settings for Toaster project. |
10 | 10 | ||
11 | import os | 11 | import os |
12 | from pathlib import Path | ||
13 | from toastermain.logs import LOGGING_SETTINGS | ||
12 | 14 | ||
13 | DEBUG = True | 15 | DEBUG = True |
14 | 16 | ||
@@ -186,7 +188,13 @@ TEMPLATES = [ | |||
186 | 'django.template.loaders.app_directories.Loader', | 188 | 'django.template.loaders.app_directories.Loader', |
187 | #'django.template.loaders.eggs.Loader', | 189 | #'django.template.loaders.eggs.Loader', |
188 | ], | 190 | ], |
189 | 'string_if_invalid': InvalidString("%s"), | 191 | # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled |
192 | # Generally, string_if_invalid should only be enabled in order to debug | ||
193 | # a specific template problem, then cleared once debugging is complete. | ||
194 | # If you assign a value other than '' to string_if_invalid, | ||
195 | # you will experience rendering problems with these templates and sites. | ||
196 | # 'string_if_invalid': InvalidString("%s"), | ||
197 | 'string_if_invalid': "", | ||
190 | 'debug': DEBUG, | 198 | 'debug': DEBUG, |
191 | }, | 199 | }, |
192 | }, | 200 | }, |
@@ -242,6 +250,9 @@ INSTALLED_APPS = ( | |||
242 | 'django.contrib.humanize', | 250 | 'django.contrib.humanize', |
243 | 'bldcollector', | 251 | 'bldcollector', |
244 | 'toastermain', | 252 | 'toastermain', |
253 | |||
254 | # 3rd-lib | ||
255 | "log_viewer", | ||
245 | ) | 256 | ) |
246 | 257 | ||
247 | 258 | ||
@@ -302,43 +313,22 @@ for t in os.walk(os.path.dirname(currentdir)): | |||
302 | # the site admins on every HTTP 500 error when DEBUG=False. | 313 | # the site admins on every HTTP 500 error when DEBUG=False. |
303 | # See http://docs.djangoproject.com/en/dev/topics/logging for | 314 | # See http://docs.djangoproject.com/en/dev/topics/logging for |
304 | # more details on how to customize your logging configuration. | 315 | # more details on how to customize your logging configuration. |
305 | LOGGING = { | 316 | LOGGING = LOGGING_SETTINGS |
306 | 'version': 1, | 317 | |
307 | 'disable_existing_loggers': False, | 318 | # Build paths inside the project like this: BASE_DIR / 'subdir'. |
308 | 'filters': { | 319 | BASE_DIR = Path(__file__).resolve(strict=True).parent.parent |
309 | 'require_debug_false': { | 320 | |
310 | '()': 'django.utils.log.RequireDebugFalse' | 321 | # LOG VIEWER |
311 | } | 322 | # https://pypi.org/project/django-log-viewer/ |
312 | }, | 323 | LOG_VIEWER_FILES_PATTERN = '*.log*' |
313 | 'formatters': { | 324 | LOG_VIEWER_FILES_DIR = os.path.join(BASE_DIR, 'logs') |
314 | 'datetime': { | 325 | LOG_VIEWER_PAGE_LENGTH = 25 # total log lines per-page |
315 | 'format': '%(asctime)s %(levelname)s %(message)s' | 326 | LOG_VIEWER_MAX_READ_LINES = 100000 # total log lines will be read |
316 | } | 327 | LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL'] |
317 | }, | 328 | |
318 | 'handlers': { | 329 | # Optionally you can set the next variables in order to customize the admin: |
319 | 'mail_admins': { | 330 | LOG_VIEWER_FILE_LIST_TITLE = "Logs list" |
320 | 'level': 'ERROR', | 331 | |
321 | 'filters': ['require_debug_false'], | ||
322 | 'class': 'django.utils.log.AdminEmailHandler' | ||
323 | }, | ||
324 | 'console': { | ||
325 | 'level': 'DEBUG', | ||
326 | 'class': 'logging.StreamHandler', | ||
327 | 'formatter': 'datetime', | ||
328 | } | ||
329 | }, | ||
330 | 'loggers': { | ||
331 | 'toaster' : { | ||
332 | 'handlers': ['console'], | ||
333 | 'level': 'DEBUG', | ||
334 | }, | ||
335 | 'django.request': { | ||
336 | 'handlers': ['console'], | ||
337 | 'level': 'WARN', | ||
338 | 'propagate': True, | ||
339 | }, | ||
340 | } | ||
341 | } | ||
342 | 332 | ||
343 | if DEBUG and SQL_DEBUG: | 333 | if DEBUG and SQL_DEBUG: |
344 | LOGGING['loggers']['django.db.backends'] = { | 334 | LOGGING['loggers']['django.db.backends'] = { |
diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py index 0360302668..3be46fcf0c 100644 --- a/bitbake/lib/toaster/toastermain/urls.py +++ b/bitbake/lib/toaster/toastermain/urls.py | |||
@@ -28,6 +28,8 @@ urlpatterns = [ | |||
28 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), | 28 | # url(r'^admin/doc/', include('django.contrib.admindocs.urls')), |
29 | 29 | ||
30 | 30 | ||
31 | url(r'^logs/', include('log_viewer.urls')), | ||
32 | |||
31 | # This is here to maintain backward compatibility and will be deprecated | 33 | # This is here to maintain backward compatibility and will be deprecated |
32 | # in the future. | 34 | # in the future. |
33 | url(r'^orm/eventfile$', bldcollector.views.eventfile), | 35 | url(r'^orm/eventfile$', bldcollector.views.eventfile), |