summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-11-03 13:47:16 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-11-12 17:04:48 +0000
commit0ca70ce37aa8cec6a74ec874a7b11597b608c403 (patch)
tree65e74aa95de35de101286746cd5b9f6acca80468
parentbaa1082c64beb0385aca2807ae6da26bee5dbcbf (diff)
downloadpoky-0ca70ce37aa8cec6a74ec874a7b11597b608c403.tar.gz
bitbake: toaster script: webport option and other improvements
We add the "webport=" command line option as to allow starting the web server on a custom port. The bitbake server port is now auto-allocated. This is needed to be able to run multiple toaster environments on a single machine. We tackle bug 6023 (toaster refusing to start when lock file is present) by using more specific checks, and automatically recover from bitbake server down / webserver up error mode. Command line parameters are now read on both interactive and managed modes. The localhost and ssh controllers are updated to work with the modified toaster launcher script. [YOCTO #6023] (Bitbake rev: cd3eb5b051743463cfe51dba97cae4da75420048) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xbitbake/bin/toaster96
-rw-r--r--bitbake/lib/toaster/bldcontrol/bbcontroller.py19
-rw-r--r--bitbake/lib/toaster/bldcontrol/localhostbecontroller.py38
-rw-r--r--bitbake/lib/toaster/bldcontrol/sshbecontroller.py46
4 files changed, 120 insertions, 79 deletions
diff --git a/bitbake/bin/toaster b/bitbake/bin/toaster
index 34f65e4721..4c6ac5ac40 100755
--- a/bitbake/bin/toaster
+++ b/bitbake/bin/toaster
@@ -37,7 +37,7 @@ function webserverKillAll()
37 kill -SIGTERM -$(< ${pidfile}) 2>/dev/null 37 kill -SIGTERM -$(< ${pidfile}) 2>/dev/null
38 sleep 1; 38 sleep 1;
39 # Kill processes if they are still running - may happen in interactive shells 39 # Kill processes if they are still running - may happen in interactive shells
40 ps fux | grep "python.*manage.py" | awk '{print $2}' | xargs kill 40 ps fux | grep "python.*manage.py runserver" | awk '{print $2}' | xargs kill
41 done; 41 done;
42 rm ${pidfile} 42 rm ${pidfile}
43 fi 43 fi
@@ -69,13 +69,13 @@ function webserverStartAll()
69 fi 69 fi
70 if [ $retval -eq 0 ]; then 70 if [ $retval -eq 0 ]; then
71 echo "Starting webserver" 71 echo "Starting webserver"
72 python $BBBASEDIR/lib/toaster/manage.py runserver 0.0.0.0:8000 </dev/null >${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid 72 python $BBBASEDIR/lib/toaster/manage.py runserver "0.0.0.0:$WEB_PORT" </dev/null >${BUILDDIR}/toaster_web.log 2>&1 & echo $! >${BUILDDIR}/.toastermain.pid
73 sleep 1 73 sleep 1
74 if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then 74 if ! cat "${BUILDDIR}/.toastermain.pid" | xargs -I{} kill -0 {} ; then
75 retval=1 75 retval=1
76 rm "${BUILDDIR}/.toastermain.pid" 76 rm "${BUILDDIR}/.toastermain.pid"
77 else 77 else
78 echo "Webserver address: 0.0.0.0:8000" 78 echo "Webserver address: http://0.0.0.0:$WEB_PORT/"
79 fi 79 fi
80 fi 80 fi
81 return $retval 81 return $retval
@@ -103,7 +103,7 @@ function stop_system()
103 kill $(< ${BUILDDIR}/.toasterui.pid ) 2>/dev/null 103 kill $(< ${BUILDDIR}/.toasterui.pid ) 2>/dev/null
104 rm ${BUILDDIR}/.toasterui.pid 104 rm ${BUILDDIR}/.toasterui.pid
105 fi 105 fi
106 BBSERVER=0.0.0.0:8200 bitbake -m 106 BBSERVER=0.0.0.0:-1 bitbake -m
107 unset BBSERVER 107 unset BBSERVER
108 webserverKillAll 108 webserverKillAll
109 # force stop any misbehaving bitbake server 109 # force stop any misbehaving bitbake server
@@ -126,9 +126,46 @@ function notify_chldexit() {
126} 126}
127 127
128 128
129
130# Verify prerequisites
131
132if ! echo "import django; print (1,) == django.VERSION[0:1] and django.VERSION[1:2][0] in (5,6)" | python 2>/dev/null | grep True >/dev/null; then
133 echo -e "This program needs Django 1.5 or 1.6. Please install with\n\npip install django==1.6"
134 return 2
135fi
136
137if ! echo "import south; print [0,8,4] == map(int,south.__version__.split(\".\"))" | python 2>/dev/null | grep True >/dev/null; then
138 echo -e "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4"
139 return 2
140fi
141
142# read command line parameters
143
129BBBASEDIR=`dirname ${BASH_SOURCE}`/.. 144BBBASEDIR=`dirname ${BASH_SOURCE}`/..
130RUNNING=0 145RUNNING=0
131 146
147NOTOASTERUI=0
148WEBSERVER=1
149TOASTER_BRBE=""
150WEB_PORT="8000"
151
152for param in $*; do
153 case $param in
154 noui )
155 NOTOASTERUI=1
156 ;;
157 noweb )
158 WEBSERVER=0
159 ;;
160 brbe=* )
161 TOASTER_BRBE=$'\n'"TOASTER_BRBE=\""${param#*=}"\""
162 ;;
163 webport=*)
164 WEB_PORT="${param#*=}"
165 esac
166done
167
168
132if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; then 169if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; then
133 # We are called as standalone. We refuse to run in a build environment - we need the interactive mode for that. 170 # We are called as standalone. We refuse to run in a build environment - we need the interactive mode for that.
134 # Start just the web server, point the web browser to the interface, and start any Django services. 171 # Start just the web server, point the web browser to the interface, and start any Django services.
@@ -152,7 +189,7 @@ if [ -z "$ZSH_NAME" ] && [ `basename \"$0\"` = `basename \"$BASH_SOURCE\"` ]; th
152 echo "Failed to start the web server, stopping" 1>&2; 189 echo "Failed to start the web server, stopping" 1>&2;
153 exit 1; 190 exit 1;
154 fi 191 fi
155 xdg-open http://0.0.0.0:8000/ >/dev/null 2>&1 & 192 xdg-open http://0.0.0.0:$WEB_PORT/ >/dev/null 2>&1 &
156 trap trap_ctrlc SIGINT 193 trap trap_ctrlc SIGINT
157 echo "Running. Stop with Ctrl-C" 194 echo "Running. Stop with Ctrl-C"
158 while [ $RUNNING -gt 0 ]; do 195 while [ $RUNNING -gt 0 ]; do
@@ -170,23 +207,6 @@ if [ -z "$BUILDDIR" ] || [ -z `which bitbake` ]; then
170fi 207fi
171 208
172 209
173
174# Verify prerequisites
175
176if ! echo "import django; print (1,) == django.VERSION[0:1] and django.VERSION[1:2][0] in (5,6)" | python 2>/dev/null | grep True >/dev/null; then
177 echo -e "This program needs Django 1.5 or 1.6. Please install with\n\npip install django==1.6"
178 return 2
179fi
180
181if ! echo "import south; print [0,8,4] == map(int,south.__version__.split(\".\"))" | python 2>/dev/null | grep True >/dev/null; then
182 echo -e "This program needs South 0.8.4. Please install with\n\npip install south==0.8.4"
183 return 2
184fi
185
186
187
188
189
190# Determine the action. If specified by arguments, fine, if not, toggle it 210# Determine the action. If specified by arguments, fine, if not, toggle it
191if [ "x$1" == "xstart" ] || [ "x$1" == "xstop" ]; then 211if [ "x$1" == "xstart" ] || [ "x$1" == "xstop" ]; then
192 CMD="$1" 212 CMD="$1"
@@ -198,22 +218,6 @@ else
198 fi; 218 fi;
199fi 219fi
200 220
201NOTOASTERUI=0
202WEBSERVER=1
203TOASTER_BRBE=""
204for param in $*; do
205 case $param in
206 noui )
207 NOTOASTERUI=1
208 ;;
209 noweb )
210 WEBSERVER=0
211 ;;
212 brbe=* )
213 TOASTER_BRBE=$'\n'"TOASTER_BRBE=\""${param#*=}"\""
214 esac
215done
216
217echo "The system will $CMD." 221echo "The system will $CMD."
218 222
219# Make sure it's safe to run by checking bitbake lock 223# Make sure it's safe to run by checking bitbake lock
@@ -223,12 +227,18 @@ if [ -e $BUILDDIR/bitbake.lock ]; then
223 (flock -n 200 ) 200<$BUILDDIR/bitbake.lock || lock=0 227 (flock -n 200 ) 200<$BUILDDIR/bitbake.lock || lock=0
224fi 228fi
225 229
226if [ ${CMD} == "start" ] && ( [ $lock -eq 0 ] || [ -e $BUILDDIR/.toastermain.pid ] ); then 230if [ ${CMD} == "start" ] && [ $lock -eq 0 ]; then
227 echo "Error: bitbake lock state error. File locks show that the system is on." 2>&1 231 echo "Error: bitbake lock state error. File locks show that the system is on." 1>&2
228 echo "If you see problems, stop and then start the system again." 2>&1 232 echo "Please wait for the current build to finish, stop and then start the system again." 1>&2
229 return 3 233 return 3
230fi 234fi
231 235
236if [ ${CMD} == "start" ] && [ -e $BUILDDIR/.toastermain.pid ] && kill -0 `cat $BUILDDIR/.toastermain.pid`; then
237 echo "Error: bitbake appears to be dead, but the webserver is alive. Something fishy is going on." 1>&2
238 echo "Cleaning up the web server at to start a clean slate."
239 webserverKillAll
240fi
241
232 242
233# Execute the commands 243# Execute the commands
234 244
@@ -241,12 +251,12 @@ case $CMD in
241 return 4 251 return 4
242 fi 252 fi
243 unset BBSERVER 253 unset BBSERVER
244 bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:8200 254 bitbake --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0
245 if [ $? -ne 0 ]; then 255 if [ $? -ne 0 ]; then
246 start_success=0 256 start_success=0
247 echo "Bitbake server start failed" 257 echo "Bitbake server start failed"
248 else 258 else
249 export BBSERVER=0.0.0.0:8200 259 export BBSERVER=0.0.0.0:-1
250 if [ $NOTOASTERUI == 0 ]; then # we start the TOASTERUI only if not inhibited 260 if [ $NOTOASTERUI == 0 ]; then # we start the TOASTERUI only if not inhibited
251 bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid 261 bitbake --observe-only -u toasterui >${BUILDDIR}/toaster_ui.log 2>&1 & echo $! >${BUILDDIR}/.toasterui.pid
252 fi 262 fi
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
index 6bb45d6388..7c27fe110e 100644
--- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
@@ -117,6 +117,25 @@ class BuildEnvironmentController(object):
117 self.be = be 117 self.be = be
118 self.connection = None 118 self.connection = None
119 119
120 @staticmethod
121 def _updateBBLayers(bblayerconf, layerlist):
122 conflines = open(bblayerconf, "r").readlines()
123
124 bblayerconffile = open(bblayerconf, "w")
125 skip = 0
126 for i in xrange(len(conflines)):
127 if skip > 0:
128 skip =- 1
129 continue
130 if conflines[i].startswith("# line added by toaster"):
131 skip = 1
132 else:
133 bblayerconffile.write(conflines[i])
134
135 bblayerconffile.write("# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
136 bblayerconffile.close()
137
138
120 139
121 def startBBServer(self, brbe): 140 def startBBServer(self, brbe):
122 """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI. 141 """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI.
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index a272860ec0..ebf2b4f3bb 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -86,8 +86,12 @@ class LocalhostBEController(BuildEnvironmentController):
86 raise 86 raise
87 87
88 cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe) 88 cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)
89 print("DEBUG: executing ", cmd) 89 port = "-1"
90 print self._shellcmd(cmd) 90 for i in self._shellcmd(cmd).split("\n"):
91 if i.startswith("Bitbake server address"):
92 port = i.split(" ")[-1]
93 print "Found bitbake server port ", port
94
91 def _toaster_ui_started(filepath): 95 def _toaster_ui_started(filepath):
92 if not os.path.exists(filepath): 96 if not os.path.exists(filepath):
93 return False 97 return False
@@ -105,7 +109,7 @@ class LocalhostBEController(BuildEnvironmentController):
105 print("DEBUG: Started server") 109 print("DEBUG: Started server")
106 assert self.be.sourcedir and os.path.exists(self.be.builddir) 110 assert self.be.sourcedir and os.path.exists(self.be.builddir)
107 self.be.bbaddress = "localhost" 111 self.be.bbaddress = "localhost"
108 self.be.bbport = "8200" 112 self.be.bbport = port
109 self.be.bbstate = BuildEnvironment.SERVER_STARTED 113 self.be.bbstate = BuildEnvironment.SERVER_STARTED
110 self.be.save() 114 self.be.save()
111 115
@@ -129,7 +133,7 @@ class LocalhostBEController(BuildEnvironmentController):
129 gitrepos = {} 133 gitrepos = {}
130 gitrepos[bitbakes[0].giturl] = [] 134 gitrepos[bitbakes[0].giturl] = []
131 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) 135 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
132 136
133 for layer in layers: 137 for layer in layers:
134 # we don't process local URLs 138 # we don't process local URLs
135 if layer.giturl.startswith("file://"): 139 if layer.giturl.startswith("file://"):
@@ -141,7 +145,8 @@ class LocalhostBEController(BuildEnvironmentController):
141 commitid = gitrepos[giturl][0][2] 145 commitid = gitrepos[giturl][0][2]
142 for e in gitrepos[giturl]: 146 for e in gitrepos[giturl]:
143 if commitid != e[2]: 147 if commitid != e[2]:
144 raise BuildSetupException("More than one commit per git url, unsupported configuration") 148 import pprint
149 raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
145 150
146 151
147 layerlist = [] 152 layerlist = []
@@ -170,6 +175,11 @@ class LocalhostBEController(BuildEnvironmentController):
170 print "DEBUG: selected poky dir name", localdirname 175 print "DEBUG: selected poky dir name", localdirname
171 self.pokydirname = localdirname 176 self.pokydirname = localdirname
172 177
178 # make sure we have a working bitbake
179 if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
180 print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname
181 self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
182
173 # verify our repositories 183 # verify our repositories
174 for name, dirpath, commit in gitrepos[giturl]: 184 for name, dirpath, commit in gitrepos[giturl]:
175 localdirpath = os.path.join(localdirname, dirpath) 185 localdirpath = os.path.join(localdirname, dirpath)
@@ -177,7 +187,7 @@ class LocalhostBEController(BuildEnvironmentController):
177 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) 187 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
178 188
179 if name != "bitbake": 189 if name != "bitbake":
180 layerlist.append(localdirpath) 190 layerlist.append(localdirpath.rstrip("/"))
181 191
182 print "DEBUG: current layer list ", layerlist 192 print "DEBUG: current layer list ", layerlist
183 193
@@ -190,21 +200,7 @@ class LocalhostBEController(BuildEnvironmentController):
190 if not os.path.exists(bblayerconf): 200 if not os.path.exists(bblayerconf):
191 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) 201 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
192 202
193 conflines = open(bblayerconf, "r").readlines() 203 BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
194
195 bblayerconffile = open(bblayerconf, "w")
196 skip = 0
197 for i in xrange(len(conflines)):
198 if skip > 0:
199 skip =- 1
200 continue
201 if conflines[i].startswith("# line added by toaster"):
202 skip = 1
203 else:
204 bblayerconffile.write(conflines[i])
205
206 bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
207 bblayerconffile.close()
208 204
209 self.islayerset = True 205 self.islayerset = True
210 return True 206 return True
diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
index f9fd51e5f8..45e15392e5 100644
--- a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
@@ -80,13 +80,18 @@ class SSHBEController(BuildEnvironmentController):
80 def startBBServer(self, brbe): 80 def startBBServer(self, brbe):
81 assert self.pokydirname and self._pathexists(self.pokydirname) 81 assert self.pokydirname and self._pathexists(self.pokydirname)
82 assert self.islayerset 82 assert self.islayerset
83 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)) 83 cmd = self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe))
84 # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected 84
85 # but since they start async without any return, we just wait a bit 85 port = "-1"
86 print "Started server" 86 for i in cmd.split("\n"):
87 if i.startswith("Bitbake server address"):
88 port = i.split(" ")[-1]
89 print "Found bitbake server port ", port
90
91
87 assert self.be.sourcedir and self._pathexists(self.be.builddir) 92 assert self.be.sourcedir and self._pathexists(self.be.builddir)
88 self.be.bbaddress = self.be.address.split("@")[-1] 93 self.be.bbaddress = self.be.address.split("@")[-1]
89 self.be.bbport = "8200" 94 self.be.bbport = port
90 self.be.bbstate = BuildEnvironment.SERVER_STARTED 95 self.be.bbstate = BuildEnvironment.SERVER_STARTED
91 self.be.save() 96 self.be.save()
92 97
@@ -99,6 +104,19 @@ class SSHBEController(BuildEnvironmentController):
99 self.be.save() 104 self.be.save()
100 print "Stopped server" 105 print "Stopped server"
101 106
107
108 def _copyFile(self, filepath1, filepath2):
109 p = subprocess.Popen("scp '%s' '%s'" % (filepath1, filepath2), stdout=subprocess.PIPE, stderr = subprocess.PIPE, shell=True)
110 (out, err) = p.communicate()
111 if p.returncode:
112 raise ShellCmdException(err)
113
114 def pullFile(self, local_filename, remote_filename):
115 _copyFile(local_filename, "%s:%s" % (self.be.address, remote_filename))
116
117 def pushFile(self, local_filename, remote_filename):
118 _copyFile("%s:%s" % (self.be.address, remote_filename), local_filename)
119
102 def setLayers(self, bitbakes, layers): 120 def setLayers(self, bitbakes, layers):
103 """ a word of attention: by convention, the first layer for any build will be poky! """ 121 """ a word of attention: by convention, the first layer for any build will be poky! """
104 122
@@ -110,7 +128,7 @@ class SSHBEController(BuildEnvironmentController):
110 gitrepos = {} 128 gitrepos = {}
111 gitrepos[bitbakes[0].giturl] = [] 129 gitrepos[bitbakes[0].giturl] = []
112 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) 130 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
113 131
114 for layer in layers: 132 for layer in layers:
115 # we don't process local URLs 133 # we don't process local URLs
116 if layer.giturl.startswith("file://"): 134 if layer.giturl.startswith("file://"):
@@ -171,17 +189,15 @@ class SSHBEController(BuildEnvironmentController):
171 if not self._pathexists(bblayerconf): 189 if not self._pathexists(bblayerconf):
172 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) 190 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
173 191
174 conflines = open(bblayerconf, "r").readlines() 192 import uuid
193 local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf"
175 194
176 bblayerconffile = open(bblayerconf, "w") 195 self.pullFile(bblayerconf, local_bblayerconf)
177 for i in xrange(len(conflines)): 196
178 if conflines[i].startswith("# line added by toaster"): 197 BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist)
179 i += 2 198 self.pushFile(local_bblayerconf, bblayerconf)
180 else:
181 bblayerconffile.write(conflines[i])
182 199
183 bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") 200 os.unlink(local_bblayerconf)
184 bblayerconffile.close()
185 201
186 self.islayerset = True 202 self.islayerset = True
187 return True 203 return True