summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/toaster
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-01-20 09:39:34 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-12-18 10:24:06 +0000
commitd086fa3aed34a05d52e73c255ca22379149a64a1 (patch)
treec55f38b6b86b513c0d4b47a01fed47c9db1cb163 /bitbake/lib/toaster
parentf99f2cdd69f1015953a7e9ab07af46dd63e50ad4 (diff)
downloadpoky-d086fa3aed34a05d52e73c255ca22379149a64a1.tar.gz
bitbake: toasterui: add extra debug and development infos
We update and add logs throughout the code in order to help with development. The extra logging is turned off by default, but it can be enabled by using environment variables. All logging happens through the Python logging facilities. The toaster UI will save a log of all incoming events if the TOASTER_EVENTLOG variable is set. If TOASTER_SQLDEBUG is set all DB queries will be logged. If TOASTER_DEVEL is set and the django-fresh module is available, the module is enabled to allow auto-reload of pages when the source is changed. (Bitbake rev: 10c27450601b4d24bbb273bd0e053498807d1060) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster')
-rw-r--r--bitbake/lib/toaster/bldcontrol/localhostbecontroller.py27
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py2
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py13
-rw-r--r--bitbake/lib/toaster/toastermain/settings.py27
-rw-r--r--bitbake/lib/toaster/toastermain/urls.py10
5 files changed, 59 insertions, 20 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index ebf2b4f3bb..22d31e33f2 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -32,6 +32,10 @@ from toastermain import settings
32 32
33from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname 33from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname
34 34
35import logging
36logger = logging.getLogger("toaster")
37
38
35class LocalhostBEController(BuildEnvironmentController): 39class LocalhostBEController(BuildEnvironmentController):
36 """ Implementation of the BuildEnvironmentController for the localhost; 40 """ Implementation of the BuildEnvironmentController for the localhost;
37 this controller manages the default build directory, 41 this controller manages the default build directory,
@@ -56,8 +60,10 @@ class LocalhostBEController(BuildEnvironmentController):
56 err = "command: %s \n%s" % (command, out) 60 err = "command: %s \n%s" % (command, out)
57 else: 61 else:
58 err = "command: %s \n%s" % (command, err) 62 err = "command: %s \n%s" % (command, err)
63 logger.debug("localhostbecontroller: shellcmd error %s" % err)
59 raise ShellCmdException(err) 64 raise ShellCmdException(err)
60 else: 65 else:
66 logger.debug("localhostbecontroller: shellcmd success")
61 return out 67 return out
62 68
63 def _createdirpath(self, path): 69 def _createdirpath(self, path):
@@ -90,7 +96,7 @@ class LocalhostBEController(BuildEnvironmentController):
90 for i in self._shellcmd(cmd).split("\n"): 96 for i in self._shellcmd(cmd).split("\n"):
91 if i.startswith("Bitbake server address"): 97 if i.startswith("Bitbake server address"):
92 port = i.split(" ")[-1] 98 port = i.split(" ")[-1]
93 print "Found bitbake server port ", port 99 logger.debug("localhostbecontroller: Found bitbake server port %s" % port)
94 100
95 def _toaster_ui_started(filepath): 101 def _toaster_ui_started(filepath):
96 if not os.path.exists(filepath): 102 if not os.path.exists(filepath):
@@ -103,10 +109,10 @@ class LocalhostBEController(BuildEnvironmentController):
103 109
104 while not _toaster_ui_started(os.path.join(self.be.builddir, "toaster_ui.log")): 110 while not _toaster_ui_started(os.path.join(self.be.builddir, "toaster_ui.log")):
105 import time 111 import time
106 print "DEBUG: Waiting server to start" 112 logger.debug("localhostbecontroller: Waiting bitbake server to start")
107 time.sleep(0.5) 113 time.sleep(0.5)
108 114
109 print("DEBUG: Started server") 115 logger.debug("localhostbecontroller: Started bitbake server")
110 assert self.be.sourcedir and os.path.exists(self.be.builddir) 116 assert self.be.sourcedir and os.path.exists(self.be.builddir)
111 self.be.bbaddress = "localhost" 117 self.be.bbaddress = "localhost"
112 self.be.bbport = port 118 self.be.bbport = port
@@ -116,11 +122,11 @@ class LocalhostBEController(BuildEnvironmentController):
116 def stopBBServer(self): 122 def stopBBServer(self):
117 assert self.pokydirname and os.path.exists(self.pokydirname) 123 assert self.pokydirname and os.path.exists(self.pokydirname)
118 assert self.islayerset 124 assert self.islayerset
119 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % 125 self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
120 (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)())) 126 (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
121 self.be.bbstate = BuildEnvironment.SERVER_STOPPED 127 self.be.bbstate = BuildEnvironment.SERVER_STOPPED
122 self.be.save() 128 self.be.save()
123 print "Stopped server" 129 logger.debug("localhostbecontroller: Stopped bitbake server")
124 130
125 def setLayers(self, bitbakes, layers): 131 def setLayers(self, bitbakes, layers):
126 """ a word of attention: by convention, the first layer for any build will be poky! """ 132 """ a word of attention: by convention, the first layer for any build will be poky! """
@@ -149,12 +155,13 @@ class LocalhostBEController(BuildEnvironmentController):
149 raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) 155 raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
150 156
151 157
158 logger.debug("localhostbecontroller, our git repos are %s" % gitrepos)
152 layerlist = [] 159 layerlist = []
153 160
154 # 2. checkout the repositories 161 # 2. checkout the repositories
155 for giturl in gitrepos.keys(): 162 for giturl in gitrepos.keys():
156 localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) 163 localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
157 print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname 164 logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname))
158 165
159 # make sure our directory is a git repository 166 # make sure our directory is a git repository
160 if os.path.exists(localdirname): 167 if os.path.exists(localdirname):
@@ -167,17 +174,17 @@ class LocalhostBEController(BuildEnvironmentController):
167 174
168 # branch magic name "HEAD" will inhibit checkout 175 # branch magic name "HEAD" will inhibit checkout
169 if commit != "HEAD": 176 if commit != "HEAD":
170 print "DEBUG: checking out commit ", commit, "to", localdirname 177 logger.debug("localhostbecontroller: checking out commit %s to %s " % (commit, localdirname))
171 self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) 178 self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
172 179
173 # take the localdirname as poky dir if we can find the oe-init-build-env 180 # take the localdirname as poky dir if we can find the oe-init-build-env
174 if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): 181 if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
175 print "DEBUG: selected poky dir name", localdirname 182 logger.debug("localhostbecontroller: selected poky dir name %s" % localdirname)
176 self.pokydirname = localdirname 183 self.pokydirname = localdirname
177 184
178 # make sure we have a working bitbake 185 # make sure we have a working bitbake
179 if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): 186 if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
180 print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname 187 logger.debug("localhostbecontroller: checking bitbake into the poky dirname %s " % self.pokydirname)
181 self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) 188 self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
182 189
183 # verify our repositories 190 # verify our repositories
@@ -189,7 +196,7 @@ class LocalhostBEController(BuildEnvironmentController):
189 if name != "bitbake": 196 if name != "bitbake":
190 layerlist.append(localdirpath.rstrip("/")) 197 layerlist.append(localdirpath.rstrip("/"))
191 198
192 print "DEBUG: current layer list ", layerlist 199 logger.debug("localhostbecontroller: current layer list %s " % layerlist)
193 200
194 # 3. configure the build environment, so we have a conf/bblayers.conf 201 # 3. configure the build environment, so we have a conf/bblayers.conf
195 assert self.pokydirname is not None 202 assert self.pokydirname is not None
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
index 96d2d51691..55f118cf77 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
@@ -1,7 +1,7 @@
1from django.core.management.base import NoArgsCommand, CommandError 1from django.core.management.base import NoArgsCommand, CommandError
2from django.db import transaction 2from django.db import transaction
3from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException 3from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
4from bldcontrol.models import BuildRequest, BuildEnvironment 4from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
5from orm.models import ToasterSetting 5from orm.models import ToasterSetting
6import os 6import os
7 7
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
index 5e253e0efc..56c989c9c9 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -4,6 +4,9 @@ from orm.models import Build
4from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException 4from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
5from bldcontrol.models import BuildRequest, BuildEnvironment, BRError 5from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
6import os 6import os
7import logging
8
9logger = logging.getLogger("toaster")
7 10
8class Command(NoArgsCommand): 11class Command(NoArgsCommand):
9 args = "" 12 args = ""
@@ -32,6 +35,7 @@ class Command(NoArgsCommand):
32 # select the build environment and the request to build 35 # select the build environment and the request to build
33 br = self._selectBuildRequest() 36 br = self._selectBuildRequest()
34 except IndexError as e: 37 except IndexError as e:
38 # logger.debug("runbuilds: No build request")
35 return 39 return
36 try: 40 try:
37 bec = self._selectBuildEnvironment() 41 bec = self._selectBuildEnvironment()
@@ -39,10 +43,10 @@ class Command(NoArgsCommand):
39 # we could not find a BEC; postpone the BR 43 # we could not find a BEC; postpone the BR
40 br.state = BuildRequest.REQ_QUEUED 44 br.state = BuildRequest.REQ_QUEUED
41 br.save() 45 br.save()
42 print "No build env" 46 logger.debug("runbuilds: No build env")
43 return 47 return
44 48
45 print "Build %s, Environment %s" % (br, bec.be) 49 logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be))
46 # let the build request know where it is being executed 50 # let the build request know where it is being executed
47 br.environment = bec.be 51 br.environment = bec.be
48 br.save() 52 br.save()
@@ -63,7 +67,7 @@ class Command(NoArgsCommand):
63 task = None 67 task = None
64 bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())), task) 68 bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())), task)
65 69
66 print "Build launched, exiting" 70 logger.debug("runbuilds: Build launched, exiting")
67 # disconnect from the server 71 # disconnect from the server
68 bbctrl.disconnect() 72 bbctrl.disconnect()
69 73
@@ -71,7 +75,7 @@ class Command(NoArgsCommand):
71 75
72 76
73 except Exception as e: 77 except Exception as e:
74 print " EE Error executing shell command\n", e 78 logger.error("runbuilds: Error executing shell command %s" % e)
75 traceback.print_exc(e) 79 traceback.print_exc(e)
76 BRError.objects.create(req = br, 80 BRError.objects.create(req = br,
77 errtype = str(type(e)), 81 errtype = str(type(e)),
@@ -83,6 +87,7 @@ class Command(NoArgsCommand):
83 bec.be.save() 87 bec.be.save()
84 88
85 89
90
86 def cleanup(self): 91 def cleanup(self):
87 from django.utils import timezone 92 from django.utils import timezone
88 from datetime import timedelta 93 from datetime import timedelta
diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py
index 0974b90525..acc20cc11d 100644
--- a/bitbake/lib/toaster/toastermain/settings.py
+++ b/bitbake/lib/toaster/toastermain/settings.py
@@ -21,11 +21,15 @@
21 21
22# Django settings for Toaster project. 22# Django settings for Toaster project.
23 23
24import os, re
25
24DEBUG = True 26DEBUG = True
25TEMPLATE_DEBUG = DEBUG 27TEMPLATE_DEBUG = DEBUG
26 28
27# Set to True to see the SQL queries in console 29# Set to True to see the SQL queries in console
28SQL_DEBUG = False 30SQL_DEBUG = False
31if os.environ.get("TOASTER_SQLDEBUG", None) is not None:
32 SQL_DEBUG = True
29 33
30 34
31ADMINS = ( 35ADMINS = (
@@ -46,7 +50,6 @@ DATABASES = {
46} 50}
47 51
48# Reinterpret database settings if we have DATABASE_URL environment variable defined 52# Reinterpret database settings if we have DATABASE_URL environment variable defined
49import os, re
50 53
51if 'DATABASE_URL' in os.environ: 54if 'DATABASE_URL' in os.environ:
52 dburl = os.environ['DATABASE_URL'] 55 dburl = os.environ['DATABASE_URL']
@@ -212,6 +215,9 @@ MIDDLEWARE_CLASSES = (
212 # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 215 # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
213) 216)
214 217
218from os.path import dirname as DN
219SITE_ROOT=DN(DN(os.path.abspath(__file__)))
220
215ROOT_URLCONF = 'toastermain.urls' 221ROOT_URLCONF = 'toastermain.urls'
216 222
217# Python dotted path to the WSGI application used by Django's runserver. 223# Python dotted path to the WSGI application used by Django's runserver.
@@ -245,6 +251,19 @@ INSTALLED_APPS = (
245 'south', 251 'south',
246) 252)
247 253
254
255# Load django-fresh is TOASTER_DEVEL is set, and the module is available
256FRESH_ENABLED = False
257if os.environ.get('TOASTER_DEVEL', None) is not None:
258 try:
259 import fresh
260 MIDDLEWARE_CLASSES = MIDDLEWARE_CLASSES + ("fresh.middleware.FreshMiddleware",)
261 INSTALLED_APPS = INSTALLED_APPS + ('fresh',)
262 FRESH_ENABLED = True
263 except:
264 pass
265
266
248SOUTH_TESTS_MIGRATE = False 267SOUTH_TESTS_MIGRATE = False
249 268
250# if we run in managed mode, we need user support 269# if we run in managed mode, we need user support
@@ -286,7 +305,7 @@ LOGGING = {
286 }, 305 },
287 'formatters': { 306 'formatters': {
288 'datetime': { 307 'datetime': {
289 'format': 'DB %(asctime)s %(message)s' 308 'format': '%(levelname)s %(asctime)s %(message)s'
290 } 309 }
291 }, 310 },
292 'handlers': { 311 'handlers': {
@@ -302,6 +321,10 @@ LOGGING = {
302 } 321 }
303 }, 322 },
304 'loggers': { 323 'loggers': {
324 'toaster' : {
325 'handlers': ['console'],
326 'level': 'DEBUG',
327 },
305 'django.request': { 328 'django.request': {
306 'handlers': ['mail_admins'], 329 'handlers': ['mail_admins'],
307 'level': 'ERROR', 330 'level': 'ERROR',
diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py
index 549fda12d6..a2916e2dd7 100644
--- a/bitbake/lib/toaster/toastermain/urls.py
+++ b/bitbake/lib/toaster/toastermain/urls.py
@@ -44,11 +44,15 @@ urlpatterns = patterns('',
44) 44)
45 45
46import toastermain.settings 46import toastermain.settings
47
48if toastermain.settings.FRESH_ENABLED:
49 urlpatterns.insert(1, url(r'', include('fresh.urls')))
50
47if toastermain.settings.MANAGED: 51if toastermain.settings.MANAGED:
48 urlpatterns = urlpatterns + [ 52 urlpatterns = [
49 # Uncomment the next line to enable the admin: 53 # Uncomment the next line to enable the admin:
50 url(r'^admin/', include(admin.site.urls)), 54 url(r'^admin/', include(admin.site.urls)),
51 ] 55 ] + urlpatterns
52# Automatically discover urls.py in various apps, beside our own 56# Automatically discover urls.py in various apps, beside our own
53# and map module directories to the patterns 57# and map module directories to the patterns
54 58
@@ -61,4 +65,4 @@ for t in os.walk(os.path.dirname(currentdir)):
61 65
62 if "urls.py" in t[2] and t[0] != currentdir: 66 if "urls.py" in t[2] and t[0] != currentdir:
63 modulename = os.path.basename(t[0]) 67 modulename = os.path.basename(t[0])
64 urlpatterns.append( url(r'^' + modulename + '/', include ( modulename + '.urls'))) 68 urlpatterns.insert(0, url(r'^' + modulename + '/', include ( modulename + '.urls')))