bitbake: toaster: enable remote HTTP API for status aggregation

Add support for Toaster aggregators with a set of api links that return JSON data for (a) builds in progress, (b) builds completed, (c) specific build data, and (d) an is-alive health ping link. [YOCTO #11794] (Bitbake rev: d8e79661c69671dd424dca5cc3f7f2f855b0afed) Signed-off-by: David Reyna <David.Reyna@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
author: David Reyna <David.Reyna@windriver.com> 2017-07-28 17:14:13 -0700
committer: Richard Purdie <richard.purdie@linuxfoundation.org> 2017-07-31 15:13:53 +0100
commit: 3bc3d26b465b5af3cebc1d4c0d1fa382965c32cf (patch)
tree: 6723a54029a4a3c5eaa75d19eae07aec792c792d /bitbake
parent: 3f71378fdeed0d5f18eb7c58776fd313a0ab8fe3 (diff)
download: poky-3bc3d26b465b5af3cebc1d4c0d1fa382965c32cf.tar.gz
4 files changed, 100 insertions, 4 deletions
diff --git a/bitbake/lib/toaster/toastergui/templates/health.html b/bitbake/lib/toaster/toastergui/templates/health.html
new file mode 100644
index 0000000000..f17fdbc439
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/templates/health.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head><title>Toaster Health</title></head>
+  <body>Ok</body>
+</html>
diff --git a/bitbake/lib/toaster/toastergui/urls.py b/bitbake/lib/toaster/toastergui/urls.py
index 6aebc3f833..3ad5566b1d 100644
--- a/bitbake/lib/toaster/toastergui/urls.py
+++ b/bitbake/lib/toaster/toastergui/urls.py
@@ -244,6 +244,11 @@ urlpatterns = [
        url(r'^mostrecentbuilds$', widgets.MostRecentBuildsView.as_view(),
            name='most_recent_builds'),
-          # default redirection
+        # JSON data for aggregators
+        url(r'^api/builds$', views.json_builds, name='json_builds'),
+        url(r'^api/building$', views.json_building, name='json_building'),
+        url(r'^api/build/(?P<build_id>\d+)$', views.json_build, name='json_build'),
+        # default redirection
        url(r'^$', RedirectView.as_view(url='landing', permanent=True)),
 ]
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py
index 5720b9d5e4..334bb4a2e9 100755
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -35,7 +35,7 @@ from orm.models import BitbakeVersion, CustomImageRecipe
 from django.core.urlresolvers import reverse, resolve
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.http import HttpResponseNotFound
+from django.http import HttpResponseNotFound, JsonResponse
 from django.utils import timezone
 from datetime import timedelta, datetime
 from toastergui.templatetags.projecttags import json as jsonfilter
@@ -1256,6 +1256,89 @@ def managedcontextprocessor(request):
    }
    return ret
+# REST-based API calls to return build/building status to external Toaster
+# managers and aggregators via JSON
+def _json_build_status(build_id,extend):
+    build_stat = None
+    try:
+        build = Build.objects.get( pk = build_id )
+        build_stat = {}
+        build_stat['id'] = build.id
+        build_stat['name'] = build.build_name
+        build_stat['machine'] = build.machine
+        build_stat['distro'] = build.distro
+        build_stat['start'] = build.started_on
+        # look up target name
+        target= Target.objects.get( build = build )
+        if target:
+            if target.task:
+                build_stat['target'] = '%s:%s' % (target.target,target.task)
+            else:
+                build_stat['target'] = '%s' % (target.target)
+        else:
+            build_stat['target'] = ''
+        # look up project name
+        project = Project.objects.get( build = build )
+        if project:
+            build_stat['project'] = project.name
+        else:
+            build_stat['project'] = ''
+        if Build.IN_PROGRESS == build.outcome:
+            now = timezone.now()
+            timediff = now - build.started_on
+            build_stat['seconds']='%.3f' % timediff.total_seconds()
+            build_stat['clone']='%d:%d' % (build.repos_cloned,build.repos_to_clone)
+            build_stat['parse']='%d:%d' % (build.recipes_parsed,build.recipes_to_parse)
+            tf = Task.objects.filter(build = build)
+            tfc = tf.count()
+            if tfc > 0:
+                tfd = tf.exclude(order__isnull=True).count()
+            else:
+                tfd = 0
+            build_stat['task']='%d:%d' % (tfd,tfc)
+        else:
+            build_stat['outcome'] = build.get_outcome_text()
+            timediff = build.completed_on - build.started_on
+            build_stat['seconds']='%.3f' % timediff.total_seconds()
+            build_stat['stop'] = build.completed_on
+            messages = LogMessage.objects.all().filter(build = build)
+            errors = len(messages.filter(level=LogMessage.ERROR) |
+                 messages.filter(level=LogMessage.EXCEPTION) |
+                 messages.filter(level=LogMessage.CRITICAL))
+            build_stat['errors'] = errors
+            warnings = len(messages.filter(level=LogMessage.WARNING))
+            build_stat['warnings'] = warnings
+        if extend:
+            build_stat['cooker_log'] = build.cooker_log_path
+    except Exception as e:
+        build_state = str(e)
+    return build_stat
+def json_builds(request):
+    build_table = []
+    builds = []
+    try:
+        builds = Build.objects.exclude(outcome=Build.IN_PROGRESS).order_by("-started_on")
+        for build in builds:
+            build_table.append(_json_build_status(build.id,False))
+    except Exception as e:
+        build_table = str(e)
+    return JsonResponse({'builds' : build_table, 'count' : len(builds)})
+def json_building(request):
+    build_table = []
+    builds = []
+    try:
+        builds = Build.objects.filter(outcome=Build.IN_PROGRESS).order_by("-started_on")
+        for build in builds:
+            build_table.append(_json_build_status(build.id,False))
+    except Exception as e:
+        build_table = str(e)
+    return JsonResponse({'building' : build_table, 'count' : len(builds)})
+def json_build(request,build_id):
+    return JsonResponse({'build' : _json_build_status(build_id,True)})
 import toastermain.settings
@@ -1694,3 +1777,4 @@ if True:
                return render(request, "unavailable_artifact.html")
        except (ObjectDoesNotExist, IOError):
            return render(request, "unavailable_artifact.html")
diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py
index bb325596bb..6750bdf3aa 100644
--- a/bitbake/lib/toaster/toastermain/urls.py
+++ b/bitbake/lib/toaster/toastermain/urls.py
@@ -20,9 +20,8 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 from django.conf.urls import patterns, include, url
-from django.views.generic import RedirectView
+from django.views.generic import RedirectView, TemplateView
 from django.views.decorators.cache import never_cache
 import bldcollector.views
 import logging
@@ -46,6 +45,8 @@ urlpatterns = [
    # in the future.
    url(r'^orm/eventfile$', bldcollector.views.eventfile),
+    url(r'^health$', TemplateView.as_view(template_name="health.html"), name='Toaster Health'),
    # if no application is selected, we have the magic toastergui app here
    url(r'^$', never_cache(RedirectView.as_view(url='/toastergui/', permanent=True))),
 ]
author	David Reyna <David.Reyna@windriver.com>	2017-07-28 17:14:13 -0700
committer	Richard Purdie <richard.purdie@linuxfoundation.org>	2017-07-31 15:13:53 +0100
commit	3bc3d26b465b5af3cebc1d4c0d1fa382965c32cf (patch)
tree	6723a54029a4a3c5eaa75d19eae07aec792c792d /bitbake
parent	3f71378fdeed0d5f18eb7c58776fd313a0ab8fe3 (diff)
download	poky-3bc3d26b465b5af3cebc1d4c0d1fa382965c32cf.tar.gz

diff --git a/bitbake/lib/toaster/toastergui/templates/health.html b/bitbake/lib/toaster/toastergui/templates/health.html new file mode 100644 index 0000000000..f17fdbc439 --- /dev/null +++ b/bitbake/lib/toaster/toastergui/templates/health.html
@@ -0,0 +1,6 @@
		1	<!DOCTYPE html>
		2	<html lang="en">
		3	<head><title>Toaster Health</title></head>
		4	<body>Ok</body>
		5	</html>
		6


diff --git a/bitbake/lib/toaster/toastergui/urls.py b/bitbake/lib/toaster/toastergui/urls.py index 6aebc3f833..3ad5566b1d 100644 --- a/bitbake/lib/toaster/toastergui/urls.py +++ b/bitbake/lib/toaster/toastergui/urls.py
@@ -244,6 +244,11 @@ urlpatterns = [
244	url(r'^mostrecentbuilds$', widgets.MostRecentBuildsView.as_view(),	244	url(r'^mostrecentbuilds$', widgets.MostRecentBuildsView.as_view(),
245	name='most_recent_builds'),	245	name='most_recent_builds'),
246		246
247	# default redirection	247	# JSON data for aggregators
		248	url(r'^api/builds$', views.json_builds, name='json_builds'),
		249	url(r'^api/building$', views.json_building, name='json_building'),
		250	url(r'^api/build/(?P<build_id>\d+)$', views.json_build, name='json_build'),
		251
		252	# default redirection
248	url(r'^$', RedirectView.as_view(url='landing', permanent=True)),	253	url(r'^$', RedirectView.as_view(url='landing', permanent=True)),
249	]	254	]


diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 5720b9d5e4..334bb4a2e9 100755 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py
@@ -35,7 +35,7 @@ from orm.models import BitbakeVersion, CustomImageRecipe
35	from django.core.urlresolvers import reverse, resolve	35	from django.core.urlresolvers import reverse, resolve
36	from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist	36	from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
37	from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger	37	from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
38	from django.http import HttpResponseNotFound	38	from django.http import HttpResponseNotFound, JsonResponse
39	from django.utils import timezone	39	from django.utils import timezone
40	from datetime import timedelta, datetime	40	from datetime import timedelta, datetime
41	from toastergui.templatetags.projecttags import json as jsonfilter	41	from toastergui.templatetags.projecttags import json as jsonfilter
@@ -1256,6 +1256,89 @@ def managedcontextprocessor(request):
1256	}	1256	}
1257	return ret	1257	return ret
1258		1258
		1259	# REST-based API calls to return build/building status to external Toaster
		1260	# managers and aggregators via JSON
		1261
		1262	def _json_build_status(build_id,extend):
		1263	build_stat = None
		1264	try:
		1265	build = Build.objects.get( pk = build_id )
		1266	build_stat = {}
		1267	build_stat['id'] = build.id
		1268	build_stat['name'] = build.build_name
		1269	build_stat['machine'] = build.machine
		1270	build_stat['distro'] = build.distro
		1271	build_stat['start'] = build.started_on
		1272	# look up target name
		1273	target= Target.objects.get( build = build )
		1274	if target:
		1275	if target.task:
		1276	build_stat['target'] = '%s:%s' % (target.target,target.task)
		1277	else:
		1278	build_stat['target'] = '%s' % (target.target)
		1279	else:
		1280	build_stat['target'] = ''
		1281	# look up project name
		1282	project = Project.objects.get( build = build )
		1283	if project:
		1284	build_stat['project'] = project.name
		1285	else:
		1286	build_stat['project'] = ''
		1287	if Build.IN_PROGRESS == build.outcome:
		1288	now = timezone.now()
		1289	timediff = now - build.started_on
		1290	build_stat['seconds']='%.3f' % timediff.total_seconds()
		1291	build_stat['clone']='%d:%d' % (build.repos_cloned,build.repos_to_clone)
		1292	build_stat['parse']='%d:%d' % (build.recipes_parsed,build.recipes_to_parse)
		1293	tf = Task.objects.filter(build = build)
		1294	tfc = tf.count()
		1295	if tfc > 0:
		1296	tfd = tf.exclude(order__isnull=True).count()
		1297	else:
		1298	tfd = 0
		1299	build_stat['task']='%d:%d' % (tfd,tfc)
		1300	else:
		1301	build_stat['outcome'] = build.get_outcome_text()
		1302	timediff = build.completed_on - build.started_on
		1303	build_stat['seconds']='%.3f' % timediff.total_seconds()
		1304	build_stat['stop'] = build.completed_on
		1305	messages = LogMessage.objects.all().filter(build = build)
		1306	errors = len(messages.filter(level=LogMessage.ERROR) \|
		1307	messages.filter(level=LogMessage.EXCEPTION) \|
		1308	messages.filter(level=LogMessage.CRITICAL))
		1309	build_stat['errors'] = errors
		1310	warnings = len(messages.filter(level=LogMessage.WARNING))
		1311	build_stat['warnings'] = warnings
		1312	if extend:
		1313	build_stat['cooker_log'] = build.cooker_log_path
		1314	except Exception as e:
		1315	build_state = str(e)
		1316	return build_stat
		1317
		1318	def json_builds(request):
		1319	build_table = []
		1320	builds = []
		1321	try:
		1322	builds = Build.objects.exclude(outcome=Build.IN_PROGRESS).order_by("-started_on")
		1323	for build in builds:
		1324	build_table.append(_json_build_status(build.id,False))
		1325	except Exception as e:
		1326	build_table = str(e)
		1327	return JsonResponse({'builds' : build_table, 'count' : len(builds)})
		1328
		1329	def json_building(request):
		1330	build_table = []
		1331	builds = []
		1332	try:
		1333	builds = Build.objects.filter(outcome=Build.IN_PROGRESS).order_by("-started_on")
		1334	for build in builds:
		1335	build_table.append(_json_build_status(build.id,False))
		1336	except Exception as e:
		1337	build_table = str(e)
		1338	return JsonResponse({'building' : build_table, 'count' : len(builds)})
		1339
		1340	def json_build(request,build_id):
		1341	return JsonResponse({'build' : _json_build_status(build_id,True)})
1259		1342
1260		1343
1261	import toastermain.settings	1344	import toastermain.settings
@@ -1694,3 +1777,4 @@ if True:
1694	return render(request, "unavailable_artifact.html")	1777	return render(request, "unavailable_artifact.html")
1695	except (ObjectDoesNotExist, IOError):	1778	except (ObjectDoesNotExist, IOError):
1696	return render(request, "unavailable_artifact.html")	1779	return render(request, "unavailable_artifact.html")
		1780


diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py index bb325596bb..6750bdf3aa 100644 --- a/bitbake/lib/toaster/toastermain/urls.py +++ b/bitbake/lib/toaster/toastermain/urls.py
@@ -20,9 +20,8 @@
20	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.	20	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21		21
22	from django.conf.urls import patterns, include, url	22	from django.conf.urls import patterns, include, url
23	from django.views.generic import RedirectView	23	from django.views.generic import RedirectView, TemplateView
24	from django.views.decorators.cache import never_cache	24	from django.views.decorators.cache import never_cache
25
26	import bldcollector.views	25	import bldcollector.views
27		26
28	import logging	27	import logging
@@ -46,6 +45,8 @@ urlpatterns = [
46	# in the future.	45	# in the future.
47	url(r'^orm/eventfile$', bldcollector.views.eventfile),	46	url(r'^orm/eventfile$', bldcollector.views.eventfile),
48		47
		48	url(r'^health$', TemplateView.as_view(template_name="health.html"), name='Toaster Health'),
		49
49	# if no application is selected, we have the magic toastergui app here	50	# if no application is selected, we have the magic toastergui app here
50	url(r'^$', never_cache(RedirectView.as_view(url='/toastergui/', permanent=True))),	51	url(r'^$', never_cache(RedirectView.as_view(url='/toastergui/', permanent=True))),
51	]	52	]