From 32a27931db7dc29c528a516bf666ce96806887fe Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Thu, 4 Sep 2014 15:27:32 +0100 Subject: bitbake: toaster: enable SSH-based remote build support We enable support for starting builds on remote machines through SSH. The support is limited to poky-based distributions. We refactor localhost build support and we update bldcontrol application tests to uniformely test the APIs of localhost and SSH build controllers. [YOCTO #6240] (Bitbake rev: c2ad9c9bb83f61c171434324df8c4d5ee655a556) Signed-off-by: Alexandru DAMIAN Signed-off-by: Richard Purdie --- bitbake/lib/toaster/bldcontrol/bbcontroller.py | 170 ++----------------------- 1 file changed, 12 insertions(+), 158 deletions(-) (limited to 'bitbake/lib/toaster/bldcontrol/bbcontroller.py') diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py index bf9cdf9f67..6812ae3e6e 100644 --- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -26,10 +26,6 @@ import re from django.db import transaction from django.db.models import Q from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake -import subprocess - -from toastermain import settings - # load Bitbake components path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) @@ -72,6 +68,10 @@ def getBuildEnvironmentController(**kwargs): The return object MUST always be a BuildEnvironmentController. """ + + from localhostbecontroller import LocalhostBEController + from sshbecontroller import SSHBEController + be = BuildEnvironment.objects.filter(Q(**kwargs))[0] if be.betype == BuildEnvironment.TYPE_LOCAL: return LocalhostBEController(be) @@ -81,6 +81,13 @@ def getBuildEnvironmentController(**kwargs): raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) +def _getgitcheckoutdirectoryname(url): + """ Utility that returns the last component of a git path as directory + """ + import re + components = re.split(r'[:\.\/]', url) + return components[-2] if components[-1] == "git" else components[-1] + class BuildEnvironmentController(object): """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST @@ -110,6 +117,7 @@ class BuildEnvironmentController(object): self.be = be self.connection = None + def startBBServer(self): """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI. After this method executes, self.be bbaddress/bbport MUST point to a running and free server, @@ -173,157 +181,3 @@ class ShellCmdException(Exception): class BuildSetupException(Exception): pass -class LocalhostBEController(BuildEnvironmentController): - """ Implementation of the BuildEnvironmentController for the localhost; - this controller manages the default build directory, - the server setup and system start and stop for the localhost-type build environment - - """ - - def __init__(self, be): - super(LocalhostBEController, self).__init__(be) - self.dburl = settings.getDATABASE_URL() - self.pokydirname = None - - def _shellcmd(self, command, cwd = None): - if cwd is None: - cwd = self.be.sourcedir - - p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (out,err) = p.communicate() - if p.returncode: - if len(err) == 0: - err = "command: %s \n%s" % (command, out) - else: - err = "command: %s \n%s" % (command, err) - raise ShellCmdException(err) - else: - return out - - def _createdirpath(self, path): - from os.path import dirname as DN - if not os.path.exists(DN(path)): - self._createdirpath(DN(path)) - if not os.path.exists(path): - os.mkdir(path, 0755) - - def _startBE(self): - assert self.pokydirname and os.path.exists(self.pokydirname) - self._createdirpath(self.be.builddir) - self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) - - def startBBServer(self): - assert self.pokydirname and os.path.exists(self.pokydirname) - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl)) - # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected - # but since they start async without any return, we just wait a bit - print "Started server" - assert self.be.sourcedir and os.path.exists(self.be.builddir) - self.be.bbaddress = "localhost" - self.be.bbport = "8200" - self.be.bbstate = BuildEnvironment.SERVER_STARTED - self.be.save() - - def stopBBServer(self): - assert self.be.sourcedir - print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" % - (self.be.sourcedir, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)())) - self.be.bbstate = BuildEnvironment.SERVER_STOPPED - self.be.save() - print "Stopped server" - - def setLayers(self, bitbakes, layers): - """ a word of attention: by convention, the first layer for any build will be poky! """ - - assert self.be.sourcedir is not None - assert len(bitbakes) == 1 - # set layers in the layersource - - # 1. get a list of repos, and map dirpaths for each layer - gitrepos = {} - gitrepos[bitbakes[0].giturl] = [] - gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - - for layer in layers: - # we don't process local URLs - if layer.giturl.startswith("file://"): - continue - if not layer.giturl in gitrepos: - gitrepos[layer.giturl] = [] - gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) - for giturl in gitrepos.keys(): - commitid = gitrepos[giturl][0][2] - for e in gitrepos[giturl]: - if commitid != e[2]: - raise BuildSetupException("More than one commit per git url, unsupported configuration") - - def _getgitdirectoryname(url): - import re - components = re.split(r'[:\.\/]', url) - return components[-2] if components[-1] == "git" else components[-1] - - layerlist = [] - - # 2. checkout the repositories - for giturl in gitrepos.keys(): - localdirname = os.path.join(self.be.sourcedir, _getgitdirectoryname(giturl)) - print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname - - # make sure our directory is a git repository - if os.path.exists(localdirname): - if not giturl in self._shellcmd("git remote -v", localdirname): - raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) - else: - self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) - # checkout the needed commit - commit = gitrepos[giturl][0][2] - - # branch magic name "HEAD" will inhibit checkout - if commit != "HEAD": - print "DEBUG: checking out commit ", commit, "to", localdirname - self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) - - # take the localdirname as poky dir if we can find the oe-init-build-env - if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")): - print "DEBUG: selected poky dir name", localdirname - self.pokydirname = localdirname - - # verify our repositories - for name, dirpath, commit in gitrepos[giturl]: - localdirpath = os.path.join(localdirname, dirpath) - if not os.path.exists(localdirpath): - raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) - - if name != "bitbake": - layerlist.append(localdirpath) - - print "DEBUG: current layer list ", layerlist - - # 3. configure the build environment, so we have a conf/bblayers.conf - assert self.pokydirname is not None - self._startBE() - - # 4. update the bblayers.conf - bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") - if not os.path.exists(bblayerconf): - raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) - - conflines = open(bblayerconf, "r").readlines() - - bblayerconffile = open(bblayerconf, "w") - for i in xrange(len(conflines)): - if conflines[i].startswith("# line added by toaster"): - i += 2 - else: - bblayerconffile.write(conflines[i]) - - bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"") - bblayerconffile.close() - - return True - - def release(self): - assert self.be.sourcedir and os.path.exists(self.be.builddir) - import shutil - shutil.rmtree(os.path.join(self.be.sourcedir, "build")) - assert not os.path.exists(self.be.builddir) -- cgit v1.2.3-54-g00ecf