summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-06-12 12:57:22 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-06-13 11:55:34 +0100
commite16352220572835ff2185cf992518fb4f3b2de0e (patch)
tree34ed801e6883d1ebd8e00d4431785dd5e7255039 /bitbake/lib
parent87b99274e9c90101ec9d4c49ce0874bcb85f7746 (diff)
downloadpoky-e16352220572835ff2185cf992518fb4f3b2de0e.tar.gz
bitbake: toaster: build control functionality
We add the build control functionality to toaster. * The bldcontrol application gains bbcontroller classes that know how to manage a localhost build environment. * The toaster UI now detects it is running under build environment controller, and update the build controller database and will shut down the bitbake server once the build is complete. * The toaster script can now run in standalone mode, launching the build controller and the web interface instead of just monitoring the build, as in the interactive mode. * A fixture with the default build controller entry for localhost is provided. [YOCTO #5490] [YOCTO #5491] [YOCTO #5492] [YOCTO #5493] [YOCTO #5494] [YOCTO #5537] (Bitbake rev: 10988bd77c8c7cefad3b88744bc5d8a7e3c1f4cf) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py10
-rw-r--r--bitbake/lib/bb/ui/toasterui.py3
-rw-r--r--bitbake/lib/toaster/bldcontrol/bbcontroller.py239
-rw-r--r--bitbake/lib/toaster/bldcontrol/fixtures/initial_data.json2
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/__init__.py0
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/__init__.py0
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py85
-rw-r--r--bitbake/lib/toaster/bldcontrol/models.py2
-rw-r--r--bitbake/lib/toaster/bldcontrol/tests.py73
9 files changed, 409 insertions, 5 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 58df687ca1..77be7c744c 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -916,6 +916,16 @@ class BuildInfoHelper(object):
916 self.internal_state['recipes'], 916 self.internal_state['recipes'],
917 ) 917 )
918 918
919 def store_build_done(self, br_id, be_id):
920 from bldcontrol.models import BuildEnvironment, BuildRequest
921 be = BuildEnvironment.objects.get(pk = be_id)
922 be.lock = BuildEnvironment.LOCK_LOCK
923 be.save()
924 br = BuildRequest.objects.get(pk = br_id)
925 br.state = BuildRequest.REQ_COMPLETED
926 br.build = self.internal_state['build']
927 br.save()
928
919 def _store_log_information(self, level, text): 929 def _store_log_information(self, level, text):
920 log_information = {} 930 log_information = {}
921 log_information['build'] = self.internal_state['build'] 931 log_information['build'] = self.internal_state['build']
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index c1696b2a31..b1c80cc822 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -228,8 +228,11 @@ def main(server, eventHandler, params ):
228 228
229 229
230 brbe = server.runCommand(["getVariable", "TOASTER_BRBE"])[0] 230 brbe = server.runCommand(["getVariable", "TOASTER_BRBE"])[0]
231 br_id, be_id = brbe.split(":")
231 # we start a new build info 232 # we start a new build info
232 if brbe is not None: 233 if brbe is not None:
234 buildinfohelper.store_build_done(br_id, be_id)
235
233 print "we are under BuildEnvironment management - after the build, we exit" 236 print "we are under BuildEnvironment management - after the build, we exit"
234 server.terminateServer() 237 server.terminateServer()
235 else: 238 else:
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
new file mode 100644
index 0000000000..d2b2a236bd
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
@@ -0,0 +1,239 @@
1#
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# BitBake Toaster Implementation
6#
7# Copyright (C) 2014 Intel Corporation
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
23import os
24import sys
25import re
26from django.db import transaction
27from django.db.models import Q
28from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget
29import subprocess
30
31from toastermain import settings
32
33
34# load Bitbake components
35path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
36sys.path.insert(0, path)
37import bb.server.xmlrpc
38
39class BitbakeController(object):
40 """ This is the basic class that controlls a bitbake server.
41 It is outside the scope of this class on how the server is started and aquired
42 """
43
44 def __init__(self, connection):
45 self.connection = connection
46
47 def _runCommand(self, command):
48 result, error = self.connection.connection.runCommand(command)
49 if error:
50 raise Exception(error)
51 return result
52
53 def disconnect(self):
54 return self.connection.removeClient()
55
56 def setVariable(self, name, value):
57 return self._runCommand(["setVariable", name, value])
58
59 def build(self, targets, task = None):
60 if task is None:
61 task = "build"
62 return self._runCommand(["buildTargets", targets, task])
63
64
65
66def getBuildEnvironmentController(**kwargs):
67 """ Gets you a BuildEnvironmentController that encapsulates a build environment,
68 based on the query dictionary sent in.
69
70 This is used to retrieve, for example, the currently running BE from inside
71 the toaster UI, or find a new BE to start a new build in it.
72
73 The return object MUST always be a BuildEnvironmentController.
74 """
75 be = BuildEnvironment.objects.filter(Q(**kwargs))[0]
76 if be.betype == BuildEnvironment.TYPE_LOCAL:
77 return LocalhostBEController(be)
78 elif be.betype == BuildEnvironment.TYPE_SSH:
79 return SSHBEController(be)
80 else:
81 raise Exception("FIXME: Implement BEC for type %s" % str(be.betype))
82
83
84
85class BuildEnvironmentController(object):
86 """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST
87 or SHOULD be supported by a Build Environment. It is used to establish the framework, and must
88 not be instantiated directly by the user.
89
90 Use the "getBuildEnvironmentController()" function to get a working BEC for your remote.
91
92 How the BuildEnvironments are discovered is outside the scope of this class.
93
94 You must derive this class to teach Toaster how to operate in your own infrastructure.
95 We provide some specific BuildEnvironmentController classes that can be used either to
96 directly set-up Toaster infrastructure, or as a model for your own infrastructure set:
97
98 * Localhost controller will run the Toaster BE on the same account as the web server
99 (current user if you are using the the Django development web server)
100 on the local machine, with the "build/" directory under the "poky/" source checkout directory.
101 Bash is expected to be available.
102
103 * SSH controller will run the Toaster BE on a remote machine, where the current user
104 can connect without raise Exception("FIXME: implement")word (set up with either ssh-agent or raise Exception("FIXME: implement")phrase-less key authentication)
105
106 """
107 def __init__(self, be):
108 """ Takes a BuildEnvironment object as parameter that points to the settings of the BE.
109 """
110 self.be = be
111 self.connection = None
112
113 def startBBServer(self):
114 """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI.
115 After this method executes, self.be bbaddress/bbport MUST point to a running and free server,
116 and the bbstate MUST be updated to "started".
117 """
118 raise Exception("Must override in order to actually start the BB server")
119
120 def stopBBServer(self):
121 """ Stops the currently running BB server.
122 The bbstate MUST be updated to "stopped".
123 self.connection must be none.
124 """
125
126 def setLayers(self,ls):
127 """ Sets the layer variables in the config file, after validating local layer paths.
128 The layer paths must be in a list of BRLayer object
129 """
130 raise Exception("Must override setLayers")
131
132
133 def getBBController(self):
134 """ returns a BitbakeController to an already started server; this is the point where the server
135 starts if needed; or reconnects to the server if we can
136 """
137 if not self.connection:
138 self.startBBServer()
139 self.be.lock = BuildEnvironment.LOCK_RUNNING
140 self.be.save()
141
142 server = bb.server.xmlrpc.BitBakeXMLRPCClient()
143 server.initServer()
144 server.saveConnectionDetails("%s:%s" % (self.be.bbaddress, self.be.bbport))
145 self.connection = server.establishConnection([])
146
147 self.be.bbtoken = self.connection.transport.connection_token
148 self.be.save()
149
150 return BitbakeController(self.connection)
151
152 def getArtifact(path):
153 """ This call returns an artifact identified by the 'path'. How 'path' is interpreted as
154 up to the implementing BEC. The return MUST be a REST URL where a GET will actually return
155 the content of the artifact, e.g. for use as a "download link" in a web UI.
156 """
157 raise Exception("Must return the REST URL of the artifact")
158
159 def release(self):
160 """ This stops the server and releases any resources. After this point, all resources
161 are un-available for further reference
162 """
163 raise Exception("Must override BE release")
164
165class ShellCmdException(Exception):
166 pass
167
168class LocalhostBEController(BuildEnvironmentController):
169 """ Implementation of the BuildEnvironmentController for the localhost;
170 this controller manages the default build directory,
171 the server setup and system start and stop for the localhost-type build environment
172
173 The address field is used as working directory; if not set, the build/ directory
174 is created
175 """
176
177 def __init__(self, be):
178 super(LocalhostBEController, self).__init__(be)
179 from os.path import dirname as DN
180 self.cwd = DN(DN(DN(DN(DN(os.path.realpath(__file__))))))
181 if self.be.address is None or len(self.be.address) == 0:
182 self.be.address = "build"
183 self.be.save()
184 self.bwd = self.be.address
185 self.dburl = settings.getDATABASE_URL()
186
187 # transform relative paths to absolute ones
188 if not self.bwd.startswith("/"):
189 self.bwd = os.path.join(self.cwd, self.bwd)
190 self._createBE()
191
192 def _shellcmd(self, command):
193 p = subprocess.Popen(command, cwd=self.cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
194 (out,err) = p.communicate()
195 if p.returncode:
196 if len(err) == 0:
197 err = "command: %s" % command
198 else:
199 err = "command: %s \n%s" % (command, err)
200 raise ShellCmdException(err)
201 else:
202 return out
203
204 def _createBE(self):
205 assert self.cwd and os.path.exists(self.cwd)
206 self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.cwd, self.bwd))
207
208 def startBBServer(self):
209 assert self.cwd and os.path.exists(self.cwd)
210 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.cwd, self.bwd, self.dburl))
211 # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
212 # but since they start async without any return, we just wait a bit
213 print "Started server"
214 assert self.cwd and os.path.exists(self.bwd)
215 self.be.bbaddress = "localhost"
216 self.be.bbport = "8200"
217 self.be.bbstate = BuildEnvironment.SERVER_STARTED
218 self.be.save()
219
220 def stopBBServer(self):
221 assert self.cwd
222 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
223 (self.cwd, self.bwd, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
224 self.be.bbstate = BuildEnvironment.SERVER_STOPPED
225 self.be.save()
226 print "Stopped server"
227
228 def setLayers(self, layers):
229 assert self.cwd is not None
230 layerconf = os.path.join(self.bwd, "conf/bblayers.conf")
231 if not os.path.exists(layerconf):
232 raise Exception("BE is not consistent: bblayers.conf file missing at ", layerconf)
233 return True
234
235 def release(self):
236 assert self.cwd and os.path.exists(self.bwd)
237 import shutil
238 shutil.rmtree(os.path.join(self.cwd, "build"))
239 assert not os.path.exists(self.bwd)
diff --git a/bitbake/lib/toaster/bldcontrol/fixtures/initial_data.json b/bitbake/lib/toaster/bldcontrol/fixtures/initial_data.json
index 21883abd5c..b2f12b38ae 100644
--- a/bitbake/lib/toaster/bldcontrol/fixtures/initial_data.json
+++ b/bitbake/lib/toaster/bldcontrol/fixtures/initial_data.json
@@ -1 +1 @@
[{"pk": 1, "model": "bldcontrol.buildenvironment", "fields": {"updated": "2014-05-20T12:17:30Z", "created": "2014-05-20T12:17:30Z", "lock": 0, "bbstate": 0, "bbaddress": "", "betype": 0, "bbtoken": "", "address": "localhost", "bbport": -1}}] [{"pk": 1, "model": "bldcontrol.buildenvironment", "fields": {"updated": "2014-05-20T12:17:30Z", "created": "2014-05-20T12:17:30Z", "lock": 0, "bbstate": 0, "bbaddress": "", "betype": 0, "bbtoken": "", "address": "", "bbport": -1}}]
diff --git a/bitbake/lib/toaster/bldcontrol/management/__init__.py b/bitbake/lib/toaster/bldcontrol/management/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/management/__init__.py
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py b/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/__init__.py
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
new file mode 100644
index 0000000000..dd8f84b07a
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -0,0 +1,85 @@
1from django.core.management.base import NoArgsCommand, CommandError
2from django.db import transaction
3from orm.models import Build
4from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException
5from bldcontrol.models import BuildRequest, BuildEnvironment
6import os
7
8class Command(NoArgsCommand):
9 args = ""
10 help = "Schedules and executes build requests as possible. Does not return (interrupt with Ctrl-C)"
11
12
13 @transaction.commit_on_success
14 def _selectBuildEnvironment(self):
15 bec = getBuildEnvironmentController(lock = BuildEnvironment.LOCK_FREE)
16 bec.be.lock = BuildEnvironment.LOCK_LOCK
17 bec.be.save()
18 return bec
19
20 @transaction.commit_on_success
21 def _selectBuildRequest(self):
22 br = BuildRequest.objects.filter(state = BuildRequest.REQ_QUEUED).order_by('pk')[0]
23 br.state = BuildRequest.REQ_INPROGRESS
24 br.save()
25 return br
26
27 def schedule(self):
28 try:
29 br = None
30 try:
31 # select the build environment and the request to build
32 br = self._selectBuildRequest()
33 except IndexError as e:
34 return
35 try:
36 bec = self._selectBuildEnvironment()
37 except IndexError as e:
38 # we could not find a BEC; postpone the BR
39 br.state = BuildRequest.REQ_QUEUED
40 br.save()
41 return
42
43 # set up the buid environment with the needed layers
44 print "Build %s, Environment %s" % (br, bec.be)
45 bec.setLayers(br.brlayer_set.all())
46
47 # get the bb server running
48 bbctrl = bec.getBBController()
49
50 # let toasterui that this is a managed build
51 bbctrl.setVariable("TOASTER_BRBE", "%d:%d" % (br.pk, bec.be.pk))
52
53 # set the build configuration
54 for variable in br.brvariable_set.all():
55 bbctrl.setVariable(variable.name, variable.value)
56
57 # trigger the build command
58 bbctrl.build(list(map(lambda x:x.target, br.brtarget_set.all())))
59
60 print "Build launched, exiting"
61 # disconnect from the server
62 bbctrl.disconnect()
63
64 # cleanup to be performed by toaster when the deed is done
65
66 except ShellCmdException as e:
67 import traceback
68 print " EE Error executing shell command\n", e
69 traceback.format_exc(e)
70
71 except Exception as e:
72 import traceback
73 traceback.print_exc()
74 raise e
75
76 def cleanup(self):
77 from django.utils import timezone
78 from datetime import timedelta
79 # environments locked for more than 30 seconds - they should be unlocked
80 BuildEnvironment.objects.filter(lock=BuildEnvironment.LOCK_LOCK).filter(updated__lt = timezone.now() - timedelta(seconds = 30)).update(lock = BuildEnvironment.LOCK_FREE)
81
82
83 def handle_noargs(self, **options):
84 self.cleanup()
85 self.schedule()
diff --git a/bitbake/lib/toaster/bldcontrol/models.py b/bitbake/lib/toaster/bldcontrol/models.py
index 11f487c9b6..158874f393 100644
--- a/bitbake/lib/toaster/bldcontrol/models.py
+++ b/bitbake/lib/toaster/bldcontrol/models.py
@@ -20,9 +20,11 @@ class BuildEnvironment(models.Model):
20 20
21 LOCK_FREE = 0 21 LOCK_FREE = 0
22 LOCK_LOCK = 1 22 LOCK_LOCK = 1
23 LOCK_RUNNING = 2
23 LOCK_STATE = ( 24 LOCK_STATE = (
24 (LOCK_FREE, "free"), 25 (LOCK_FREE, "free"),
25 (LOCK_LOCK, "lock"), 26 (LOCK_LOCK, "lock"),
27 (LOCK_RUNNING, "running"),
26 ) 28 )
27 29
28 address = models.CharField(max_length = 254) 30 address = models.CharField(max_length = 254)
diff --git a/bitbake/lib/toaster/bldcontrol/tests.py b/bitbake/lib/toaster/bldcontrol/tests.py
index 501deb776c..ebe477d8a8 100644
--- a/bitbake/lib/toaster/bldcontrol/tests.py
+++ b/bitbake/lib/toaster/bldcontrol/tests.py
@@ -7,10 +7,75 @@ Replace this with more appropriate tests for your application.
7 7
8from django.test import TestCase 8from django.test import TestCase
9 9
10from bldcontrol.bbcontroller import LocalhostBEController, BitbakeController
11from bldcontrol.models import BuildEnvironment, BuildRequest
12from bldcontrol.management.commands.runbuilds import Command
10 13
11class SimpleTest(TestCase): 14import socket
12 def test_basic_addition(self): 15import subprocess
16
17class LocalhostBEControllerTests(TestCase):
18 def test_StartAndStopServer(self):
19 obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL)
20 lbc = LocalhostBEController(obe)
21
22 # test start server and stop
23 self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Port already occupied")
24 lbc.startBBServer()
25 self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Server not answering")
26
27 lbc.stopBBServer()
28 self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Server not stopped")
29
30 # clean up
31 import subprocess
32 out, err = subprocess.Popen("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
33
34 self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
35
36 obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL)
37 lbc = LocalhostBEController(obe)
38
39 bbc = lbc.getBBController()
40 self.assertTrue(isinstance(bbc, BitbakeController))
41 # test set variable
42 try:
43 bbc.setVariable
44 except Exception as e :
45 self.fail("setVariable raised %s", e)
46
47 lbc.stopBBServer()
48 out, err = subprocess.Popen("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
49 self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
50
51
52class RunBuildsCommandTests(TestCase):
53 def test_bec_select(self):
13 """ 54 """
14 Tests that 1 + 1 always equals 2. 55 Tests that we can find and lock a build environment
15 """ 56 """
16 self.assertEqual(1 + 1, 2) 57
58 obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL)
59 command = Command()
60 bec = command._selectBuildEnvironment()
61
62 # make sure we select the object we've just built
63 self.assertTrue(bec.be.id == obe.id, "Environment is not properly selected")
64 # we have a locked environment
65 self.assertTrue(bec.be.lock == BuildEnvironment.LOCK_LOCK, "Environment is not locked")
66 # no more selections possible here
67 self.assertRaises(IndexError, command._selectBuildEnvironment)
68
69 def test_br_select(self):
70 from orm.models import Project
71 p, created = Project.objects.get_or_create(pk=1)
72 obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p)
73 command = Command()
74 br = command._selectBuildRequest()
75
76 # make sure we select the object we've just built
77 self.assertTrue(obr.id == br.id, "Request is not properly selected")
78 # we have a locked environment
79 self.assertTrue(br.state == BuildRequest.REQ_INPROGRESS, "Request is not updated")
80 # no more selections possible here
81 self.assertRaises(IndexError, command._selectBuildRequest)