summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-11-03 13:47:16 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-11-12 17:04:48 +0000
commit0ca70ce37aa8cec6a74ec874a7b11597b608c403 (patch)
tree65e74aa95de35de101286746cd5b9f6acca80468 /bitbake/lib
parentbaa1082c64beb0385aca2807ae6da26bee5dbcbf (diff)
downloadpoky-0ca70ce37aa8cec6a74ec874a7b11597b608c403.tar.gz
bitbake: toaster script: webport option and other improvements
We add the "webport=" command line option as to allow starting the web server on a custom port. The bitbake server port is now auto-allocated. This is needed to be able to run multiple toaster environments on a single machine. We tackle bug 6023 (toaster refusing to start when lock file is present) by using more specific checks, and automatically recover from bitbake server down / webserver up error mode. Command line parameters are now read on both interactive and managed modes. The localhost and ssh controllers are updated to work with the modified toaster launcher script. [YOCTO #6023] (Bitbake rev: cd3eb5b051743463cfe51dba97cae4da75420048) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/toaster/bldcontrol/bbcontroller.py19
-rw-r--r--bitbake/lib/toaster/bldcontrol/localhostbecontroller.py38
-rw-r--r--bitbake/lib/toaster/bldcontrol/sshbecontroller.py46
3 files changed, 67 insertions, 36 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
index 6bb45d6388..7c27fe110e 100644
--- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
@@ -117,6 +117,25 @@ class BuildEnvironmentController(object):
117 self.be = be 117 self.be = be
118 self.connection = None 118 self.connection = None
119 119
120 @staticmethod
121 def _updateBBLayers(bblayerconf, layerlist):
122 conflines = open(bblayerconf, "r").readlines()
123
124 bblayerconffile = open(bblayerconf, "w")
125 skip = 0
126 for i in xrange(len(conflines)):
127 if skip > 0:
128 skip =- 1
129 continue
130 if conflines[i].startswith("# line added by toaster"):
131 skip = 1
132 else:
133 bblayerconffile.write(conflines[i])
134
135 bblayerconffile.write("# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
136 bblayerconffile.close()
137
138
120 139
121 def startBBServer(self, brbe): 140 def startBBServer(self, brbe):
122 """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI. 141 """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI.
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index a272860ec0..ebf2b4f3bb 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -86,8 +86,12 @@ class LocalhostBEController(BuildEnvironmentController):
86 raise 86 raise
87 87
88 cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe) 88 cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)
89 print("DEBUG: executing ", cmd) 89 port = "-1"
90 print self._shellcmd(cmd) 90 for i in self._shellcmd(cmd).split("\n"):
91 if i.startswith("Bitbake server address"):
92 port = i.split(" ")[-1]
93 print "Found bitbake server port ", port
94
91 def _toaster_ui_started(filepath): 95 def _toaster_ui_started(filepath):
92 if not os.path.exists(filepath): 96 if not os.path.exists(filepath):
93 return False 97 return False
@@ -105,7 +109,7 @@ class LocalhostBEController(BuildEnvironmentController):
105 print("DEBUG: Started server") 109 print("DEBUG: Started server")
106 assert self.be.sourcedir and os.path.exists(self.be.builddir) 110 assert self.be.sourcedir and os.path.exists(self.be.builddir)
107 self.be.bbaddress = "localhost" 111 self.be.bbaddress = "localhost"
108 self.be.bbport = "8200" 112 self.be.bbport = port
109 self.be.bbstate = BuildEnvironment.SERVER_STARTED 113 self.be.bbstate = BuildEnvironment.SERVER_STARTED
110 self.be.save() 114 self.be.save()
111 115
@@ -129,7 +133,7 @@ class LocalhostBEController(BuildEnvironmentController):
129 gitrepos = {} 133 gitrepos = {}
130 gitrepos[bitbakes[0].giturl] = [] 134 gitrepos[bitbakes[0].giturl] = []
131 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) 135 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
132 136
133 for layer in layers: 137 for layer in layers:
134 # we don't process local URLs 138 # we don't process local URLs
135 if layer.giturl.startswith("file://"): 139 if layer.giturl.startswith("file://"):
@@ -141,7 +145,8 @@ class LocalhostBEController(BuildEnvironmentController):
141 commitid = gitrepos[giturl][0][2] 145 commitid = gitrepos[giturl][0][2]
142 for e in gitrepos[giturl]: 146 for e in gitrepos[giturl]:
143 if commitid != e[2]: 147 if commitid != e[2]:
144 raise BuildSetupException("More than one commit per git url, unsupported configuration") 148 import pprint
149 raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos))
145 150
146 151
147 layerlist = [] 152 layerlist = []
@@ -170,6 +175,11 @@ class LocalhostBEController(BuildEnvironmentController):
170 print "DEBUG: selected poky dir name", localdirname 175 print "DEBUG: selected poky dir name", localdirname
171 self.pokydirname = localdirname 176 self.pokydirname = localdirname
172 177
178 # make sure we have a working bitbake
179 if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')):
180 print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname
181 self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
182
173 # verify our repositories 183 # verify our repositories
174 for name, dirpath, commit in gitrepos[giturl]: 184 for name, dirpath, commit in gitrepos[giturl]:
175 localdirpath = os.path.join(localdirname, dirpath) 185 localdirpath = os.path.join(localdirname, dirpath)
@@ -177,7 +187,7 @@ class LocalhostBEController(BuildEnvironmentController):
177 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) 187 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
178 188
179 if name != "bitbake": 189 if name != "bitbake":
180 layerlist.append(localdirpath) 190 layerlist.append(localdirpath.rstrip("/"))
181 191
182 print "DEBUG: current layer list ", layerlist 192 print "DEBUG: current layer list ", layerlist
183 193
@@ -190,21 +200,7 @@ class LocalhostBEController(BuildEnvironmentController):
190 if not os.path.exists(bblayerconf): 200 if not os.path.exists(bblayerconf):
191 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) 201 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
192 202
193 conflines = open(bblayerconf, "r").readlines() 203 BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist)
194
195 bblayerconffile = open(bblayerconf, "w")
196 skip = 0
197 for i in xrange(len(conflines)):
198 if skip > 0:
199 skip =- 1
200 continue
201 if conflines[i].startswith("# line added by toaster"):
202 skip = 1
203 else:
204 bblayerconffile.write(conflines[i])
205
206 bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
207 bblayerconffile.close()
208 204
209 self.islayerset = True 205 self.islayerset = True
210 return True 206 return True
diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
index f9fd51e5f8..45e15392e5 100644
--- a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
@@ -80,13 +80,18 @@ class SSHBEController(BuildEnvironmentController):
80 def startBBServer(self, brbe): 80 def startBBServer(self, brbe):
81 assert self.pokydirname and self._pathexists(self.pokydirname) 81 assert self.pokydirname and self._pathexists(self.pokydirname)
82 assert self.islayerset 82 assert self.islayerset
83 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)) 83 cmd = self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe))
84 # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected 84
85 # but since they start async without any return, we just wait a bit 85 port = "-1"
86 print "Started server" 86 for i in cmd.split("\n"):
87 if i.startswith("Bitbake server address"):
88 port = i.split(" ")[-1]
89 print "Found bitbake server port ", port
90
91
87 assert self.be.sourcedir and self._pathexists(self.be.builddir) 92 assert self.be.sourcedir and self._pathexists(self.be.builddir)
88 self.be.bbaddress = self.be.address.split("@")[-1] 93 self.be.bbaddress = self.be.address.split("@")[-1]
89 self.be.bbport = "8200" 94 self.be.bbport = port
90 self.be.bbstate = BuildEnvironment.SERVER_STARTED 95 self.be.bbstate = BuildEnvironment.SERVER_STARTED
91 self.be.save() 96 self.be.save()
92 97
@@ -99,6 +104,19 @@ class SSHBEController(BuildEnvironmentController):
99 self.be.save() 104 self.be.save()
100 print "Stopped server" 105 print "Stopped server"
101 106
107
108 def _copyFile(self, filepath1, filepath2):
109 p = subprocess.Popen("scp '%s' '%s'" % (filepath1, filepath2), stdout=subprocess.PIPE, stderr = subprocess.PIPE, shell=True)
110 (out, err) = p.communicate()
111 if p.returncode:
112 raise ShellCmdException(err)
113
114 def pullFile(self, local_filename, remote_filename):
115 _copyFile(local_filename, "%s:%s" % (self.be.address, remote_filename))
116
117 def pushFile(self, local_filename, remote_filename):
118 _copyFile("%s:%s" % (self.be.address, remote_filename), local_filename)
119
102 def setLayers(self, bitbakes, layers): 120 def setLayers(self, bitbakes, layers):
103 """ a word of attention: by convention, the first layer for any build will be poky! """ 121 """ a word of attention: by convention, the first layer for any build will be poky! """
104 122
@@ -110,7 +128,7 @@ class SSHBEController(BuildEnvironmentController):
110 gitrepos = {} 128 gitrepos = {}
111 gitrepos[bitbakes[0].giturl] = [] 129 gitrepos[bitbakes[0].giturl] = []
112 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) 130 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
113 131
114 for layer in layers: 132 for layer in layers:
115 # we don't process local URLs 133 # we don't process local URLs
116 if layer.giturl.startswith("file://"): 134 if layer.giturl.startswith("file://"):
@@ -171,17 +189,15 @@ class SSHBEController(BuildEnvironmentController):
171 if not self._pathexists(bblayerconf): 189 if not self._pathexists(bblayerconf):
172 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) 190 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
173 191
174 conflines = open(bblayerconf, "r").readlines() 192 import uuid
193 local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf"
175 194
176 bblayerconffile = open(bblayerconf, "w") 195 self.pullFile(bblayerconf, local_bblayerconf)
177 for i in xrange(len(conflines)): 196
178 if conflines[i].startswith("# line added by toaster"): 197 BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist)
179 i += 2 198 self.pushFile(local_bblayerconf, bblayerconf)
180 else:
181 bblayerconffile.write(conflines[i])
182 199
183 bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") 200 os.unlink(local_bblayerconf)
184 bblayerconffile.close()
185 201
186 self.islayerset = True 202 self.islayerset = True
187 return True 203 return True