bitbake: toaster: enable SSH-based remote build support

We enable support for starting builds on remote machines
through SSH. The support is limited to poky-based distributions.

We refactor localhost build support and we update
bldcontrol application tests to uniformely test the APIs
of localhost and SSH build controllers.

[YOCTO #6240]

(Bitbake rev: c2ad9c9bb83f61c171434324df8c4d5ee655a556)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 92 insertions, 24 deletions

diff --git a/bitbake/lib/toaster/bldcontrol/tests.py b/bitbake/lib/toaster/bldcontrol/tests.py
index ebe477d8a8..4577c3f03b 100644
--- a/bitbake/lib/toaster/bldcontrol/tests.py
+++ b/bitbake/lib/toaster/bldcontrol/tests.py
@@ -7,46 +7,114 @@ Replace this with more appropriate tests for your application.
 
 from django.test import TestCase
 
-from bldcontrol.bbcontroller import LocalhostBEController, BitbakeController
+from bldcontrol.bbcontroller import BitbakeController
+from bldcontrol.localhostbecontroller import LocalhostBEController
+from bldcontrol.sshbecontroller import SSHBEController
 from bldcontrol.models import BuildEnvironment, BuildRequest
 from bldcontrol.management.commands.runbuilds import Command
 
 import socket
 import subprocess
 
-class LocalhostBEControllerTests(TestCase):
-    def test_StartAndStopServer(self):
-        obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL)
-        lbc = LocalhostBEController(obe)
+# standard poky data hardcoded for testing
+BITBAKE_LAYERS = [type('bitbake_info', (object,), { "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "", "commit": "HEAD"})]
+POKY_LAYERS = [
+    type('poky_info', (object,), { "name": "meta", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta", "commit": "HEAD"}),
+    type('poky_info', (object,), { "name": "meta-yocto", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto", "commit": "HEAD"}),
+    type('poky_info', (object,), { "name": "meta-yocto-bsp", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto-bsp", "commit": "HEAD"}),
+    ]
+
 
-        # test start server and stop
-        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Port already occupied")
-        lbc.startBBServer()
-        self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Server not answering")
 
-        lbc.stopBBServer()
-        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Server not stopped")
+# we have an abstract test class designed to ensure that the controllers use a single interface
+# specific controller tests only need to override the _getBuildEnvironment() method
 
-        # clean up
-        import subprocess
-        out, err = subprocess.Popen("netstat  -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+class BEControllerTests(object):
 
+    def _serverForceStop(self, bc):
+        err = bc._shellcmd("netstat  -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill")
         self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
 
-        obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL)
-        lbc = LocalhostBEController(obe)
+    def test_serverStartAndStop(self):
+        obe =  self._getBuildEnvironment()
+        bc = self._getBEController(obe)
+        bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
+
+        hostname = self.test_address.split("@")[-1]
 
-        bbc = lbc.getBBController()
+        # test start server and stop
+        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Port already occupied")
+        bc.startBBServer()
+        self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not answering")
+
+        bc.stopBBServer()
+        self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not stopped")
+
+        self._serverForceStop(bc)
+
+    def test_getBBController(self):
+        obe = self._getBuildEnvironment()
+        bc = self._getBEController(obe)
+        bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
+
+        bbc = bc.getBBController()
         self.assertTrue(isinstance(bbc, BitbakeController))
-        # test set variable
+        # test set variable, use no build marker -1 for BR value
         try:
-            bbc.setVariable
+            bbc.setVariable("TOASTER_BRBE", "%d:%d" % (-1, obe.pk))
         except Exception as e :
             self.fail("setVariable raised %s", e)
 
-        lbc.stopBBServer()
-        out, err = subprocess.Popen("netstat  -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-        self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
+        bc.stopBBServer()
+
+        self._serverForceStop(bc)
+
+class LocalhostBEControllerTests(TestCase, BEControllerTests):
+    def __init__(self, *args):
+        super(LocalhostBEControllerTests, self).__init__(*args)
+        # hardcoded for Alex's machine; since the localhost BE is machine-dependent,
+        # I found no good way to abstractize this
+        self.test_sourcedir = "/home/ddalex/ssd/yocto"
+        self.test_builddir = "/home/ddalex/ssd/yocto/build"
+        self.test_address = "localhost"
+
+    def _getBuildEnvironment(self):
+        return BuildEnvironment.objects.create(
+                lock = BuildEnvironment.LOCK_FREE,
+                betype = BuildEnvironment.TYPE_LOCAL,
+                address = self.test_address,
+                sourcedir = self.test_sourcedir,
+                builddir = self.test_builddir )
+
+    def _getBEController(self, obe):
+        return LocalhostBEController(obe)
+
+class SSHBEControllerTests(TestCase, BEControllerTests):
+    def __init__(self, *args):
+        super(SSHBEControllerTests, self).__init__(*args)
+        self.test_address = "ddalex-desktop.local"
+        # hardcoded for ddalex-desktop.local machine; since the localhost BE is machine-dependent,
+        # I found no good way to abstractize this
+        self.test_sourcedir = "/home/ddalex/ssd/yocto"
+        self.test_builddir = "/home/ddalex/ssd/yocto/build"
+
+    def _getBuildEnvironment(self):
+        return BuildEnvironment.objects.create(
+                lock = BuildEnvironment.LOCK_FREE,
+                betype = BuildEnvironment.TYPE_SSH,
+                address = self.test_address,
+                sourcedir = self.test_sourcedir,
+                builddir = self.test_builddir )
+
+    def _getBEController(self, obe):
+        return SSHBEController(obe)
+
+    def test_pathExists(self):
+        obe = BuildEnvironment.objects.create(betype = BuildEnvironment.TYPE_SSH, address= self.test_address)
+        sbc = SSHBEController(obe)
+        self.assertTrue(sbc._pathexists("/"))
+        self.assertFalse(sbc._pathexists("/.deadbeef"))
+        self.assertTrue(sbc._pathexists(sbc._shellcmd("pwd")))
 
 
 class RunBuildsCommandTests(TestCase):
@@ -67,8 +135,8 @@ class RunBuildsCommandTests(TestCase):
         self.assertRaises(IndexError, command._selectBuildEnvironment)
 
     def test_br_select(self):
-        from orm.models import Project
-        p, created = Project.objects.get_or_create(pk=1)
+        from orm.models import Project, Release, BitbakeVersion
+        p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch="HEAD")[0])[0])
         obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p)
         command = Command()
         br = command._selectBuildRequest()