From 0ca70ce37aa8cec6a74ec874a7b11597b608c403 Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Mon, 3 Nov 2014 13:47:16 +0000 Subject: bitbake: toaster script: webport option and other improvements We add the "webport=" command line option as to allow starting the web server on a custom port. The bitbake server port is now auto-allocated. This is needed to be able to run multiple toaster environments on a single machine. We tackle bug 6023 (toaster refusing to start when lock file is present) by using more specific checks, and automatically recover from bitbake server down / webserver up error mode. Command line parameters are now read on both interactive and managed modes. The localhost and ssh controllers are updated to work with the modified toaster launcher script. [YOCTO #6023] (Bitbake rev: cd3eb5b051743463cfe51dba97cae4da75420048) Signed-off-by: Alexandru DAMIAN Signed-off-by: Richard Purdie --- bitbake/lib/toaster/bldcontrol/bbcontroller.py | 19 +++++++++ .../toaster/bldcontrol/localhostbecontroller.py | 38 ++++++++---------- bitbake/lib/toaster/bldcontrol/sshbecontroller.py | 46 +++++++++++++++------- 3 files changed, 67 insertions(+), 36 deletions(-) (limited to 'bitbake/lib/toaster/bldcontrol') diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py index 6bb45d6388..7c27fe110e 100644 --- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -117,6 +117,25 @@ class BuildEnvironmentController(object): self.be = be self.connection = None + @staticmethod + def _updateBBLayers(bblayerconf, layerlist): + conflines = open(bblayerconf, "r").readlines() + + bblayerconffile = open(bblayerconf, "w") + skip = 0 + for i in xrange(len(conflines)): + if skip > 0: + skip =- 1 + continue + if conflines[i].startswith("# line added by toaster"): + skip = 1 + else: + bblayerconffile.write(conflines[i]) + + bblayerconffile.write("# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") + bblayerconffile.close() + + def startBBServer(self, brbe): """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI. diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index a272860ec0..ebf2b4f3bb 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -86,8 +86,12 @@ class LocalhostBEController(BuildEnvironmentController): raise cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe) - print("DEBUG: executing ", cmd) - print self._shellcmd(cmd) + port = "-1" + for i in self._shellcmd(cmd).split("\n"): + if i.startswith("Bitbake server address"): + port = i.split(" ")[-1] + print "Found bitbake server port ", port + def _toaster_ui_started(filepath): if not os.path.exists(filepath): return False @@ -105,7 +109,7 @@ class LocalhostBEController(BuildEnvironmentController): print("DEBUG: Started server") assert self.be.sourcedir and os.path.exists(self.be.builddir) self.be.bbaddress = "localhost" - self.be.bbport = "8200" + self.be.bbport = port self.be.bbstate = BuildEnvironment.SERVER_STARTED self.be.save() @@ -129,7 +133,7 @@ class LocalhostBEController(BuildEnvironmentController): gitrepos = {} gitrepos[bitbakes[0].giturl] = [] gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - + for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): @@ -141,7 +145,8 @@ class LocalhostBEController(BuildEnvironmentController): commitid = gitrepos[giturl][0][2] for e in gitrepos[giturl]: if commitid != e[2]: - raise BuildSetupException("More than one commit per git url, unsupported configuration") + import pprint + raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) layerlist = [] @@ -170,6 +175,11 @@ class LocalhostBEController(BuildEnvironmentController): print "DEBUG: selected poky dir name", localdirname self.pokydirname = localdirname + # make sure we have a working bitbake + if not os.path.exists(os.path.join(self.pokydirname, 'bitbake')): + print "DEBUG: checking bitbake into the poky dirname %s " % self.pokydirname + self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) + # verify our repositories for name, dirpath, commit in gitrepos[giturl]: localdirpath = os.path.join(localdirname, dirpath) @@ -177,7 +187,7 @@ class LocalhostBEController(BuildEnvironmentController): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": - layerlist.append(localdirpath) + layerlist.append(localdirpath.rstrip("/")) print "DEBUG: current layer list ", layerlist @@ -190,21 +200,7 @@ class LocalhostBEController(BuildEnvironmentController): if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - conflines = open(bblayerconf, "r").readlines() - - bblayerconffile = open(bblayerconf, "w") - skip = 0 - for i in xrange(len(conflines)): - if skip > 0: - skip =- 1 - continue - if conflines[i].startswith("# line added by toaster"): - skip = 1 - else: - bblayerconffile.write(conflines[i]) - - bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") - bblayerconffile.close() + BuildEnvironmentController._updateBBLayers(bblayerconf, layerlist) self.islayerset = True return True diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py index f9fd51e5f8..45e15392e5 100644 --- a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py @@ -80,13 +80,18 @@ class SSHBEController(BuildEnvironmentController): def startBBServer(self, brbe): assert self.pokydirname and self._pathexists(self.pokydirname) assert self.islayerset - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)) - # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected - # but since they start async without any return, we just wait a bit - print "Started server" + cmd = self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe)) + + port = "-1" + for i in cmd.split("\n"): + if i.startswith("Bitbake server address"): + port = i.split(" ")[-1] + print "Found bitbake server port ", port + + assert self.be.sourcedir and self._pathexists(self.be.builddir) self.be.bbaddress = self.be.address.split("@")[-1] - self.be.bbport = "8200" + self.be.bbport = port self.be.bbstate = BuildEnvironment.SERVER_STARTED self.be.save() @@ -99,6 +104,19 @@ class SSHBEController(BuildEnvironmentController): self.be.save() print "Stopped server" + + def _copyFile(self, filepath1, filepath2): + p = subprocess.Popen("scp '%s' '%s'" % (filepath1, filepath2), stdout=subprocess.PIPE, stderr = subprocess.PIPE, shell=True) + (out, err) = p.communicate() + if p.returncode: + raise ShellCmdException(err) + + def pullFile(self, local_filename, remote_filename): + _copyFile(local_filename, "%s:%s" % (self.be.address, remote_filename)) + + def pushFile(self, local_filename, remote_filename): + _copyFile("%s:%s" % (self.be.address, remote_filename), local_filename) + def setLayers(self, bitbakes, layers): """ a word of attention: by convention, the first layer for any build will be poky! """ @@ -110,7 +128,7 @@ class SSHBEController(BuildEnvironmentController): gitrepos = {} gitrepos[bitbakes[0].giturl] = [] gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - + for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): @@ -171,17 +189,15 @@ class SSHBEController(BuildEnvironmentController): if not self._pathexists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - conflines = open(bblayerconf, "r").readlines() + import uuid + local_bblayerconf = "/tmp/" + uuid.uuid4() + "-bblayer.conf" - bblayerconffile = open(bblayerconf, "w") - for i in xrange(len(conflines)): - if conflines[i].startswith("# line added by toaster"): - i += 2 - else: - bblayerconffile.write(conflines[i]) + self.pullFile(bblayerconf, local_bblayerconf) + + BuildEnvironmentController._updateBBLayers(local_bblayerconf, layerlist) + self.pushFile(local_bblayerconf, bblayerconf) - bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") - bblayerconffile.close() + os.unlink(local_bblayerconf) self.islayerset = True return True -- cgit v1.2.3-54-g00ecf