summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/toaster
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-09-04 15:27:32 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-09-05 10:14:25 +0100
commit32a27931db7dc29c528a516bf666ce96806887fe (patch)
tree331c64af1d50059ef0db81c157a9e79ee4c363b5 /bitbake/lib/toaster
parent5bd2b3f9a6ad85a6c5d1fe90e91aed64ef658962 (diff)
downloadpoky-32a27931db7dc29c528a516bf666ce96806887fe.tar.gz
bitbake: toaster: enable SSH-based remote build support
We enable support for starting builds on remote machines through SSH. The support is limited to poky-based distributions. We refactor localhost build support and we update bldcontrol application tests to uniformely test the APIs of localhost and SSH build controllers. [YOCTO #6240] (Bitbake rev: c2ad9c9bb83f61c171434324df8c4d5ee655a556) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster')
-rw-r--r--bitbake/lib/toaster/bldcontrol/bbcontroller.py170
-rw-r--r--bitbake/lib/toaster/bldcontrol/localhostbecontroller.py191
-rw-r--r--bitbake/lib/toaster/bldcontrol/sshbecontroller.py193
-rw-r--r--bitbake/lib/toaster/bldcontrol/tests.py116
4 files changed, 488 insertions, 182 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
index bf9cdf9f67..6812ae3e6e 100644
--- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
@@ -26,10 +26,6 @@ import re
26from django.db import transaction 26from django.db import transaction
27from django.db.models import Q 27from django.db.models import Q
28from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake 28from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
29import subprocess
30
31from toastermain import settings
32
33 29
34# load Bitbake components 30# load Bitbake components
35path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 31path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
@@ -72,6 +68,10 @@ def getBuildEnvironmentController(**kwargs):
72 68
73 The return object MUST always be a BuildEnvironmentController. 69 The return object MUST always be a BuildEnvironmentController.
74 """ 70 """
71
72 from localhostbecontroller import LocalhostBEController
73 from sshbecontroller import SSHBEController
74
75 be = BuildEnvironment.objects.filter(Q(**kwargs))[0] 75 be = BuildEnvironment.objects.filter(Q(**kwargs))[0]
76 if be.betype == BuildEnvironment.TYPE_LOCAL: 76 if be.betype == BuildEnvironment.TYPE_LOCAL:
77 return LocalhostBEController(be) 77 return LocalhostBEController(be)
@@ -81,6 +81,13 @@ def getBuildEnvironmentController(**kwargs):
81 raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) 81 raise Exception("FIXME: Implement BEC for type %s" % str(be.betype))
82 82
83 83
84def _getgitcheckoutdirectoryname(url):
85 """ Utility that returns the last component of a git path as directory
86 """
87 import re
88 components = re.split(r'[:\.\/]', url)
89 return components[-2] if components[-1] == "git" else components[-1]
90
84 91
85class BuildEnvironmentController(object): 92class BuildEnvironmentController(object):
86 """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST 93 """ BuildEnvironmentController (BEC) is the abstract class that defines the operations that MUST
@@ -110,6 +117,7 @@ class BuildEnvironmentController(object):
110 self.be = be 117 self.be = be
111 self.connection = None 118 self.connection = None
112 119
120
113 def startBBServer(self): 121 def startBBServer(self):
114 """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI. 122 """ Starts a BB server with Toaster toasterui set up to record the builds, an no controlling UI.
115 After this method executes, self.be bbaddress/bbport MUST point to a running and free server, 123 After this method executes, self.be bbaddress/bbport MUST point to a running and free server,
@@ -173,157 +181,3 @@ class ShellCmdException(Exception):
173class BuildSetupException(Exception): 181class BuildSetupException(Exception):
174 pass 182 pass
175 183
176class LocalhostBEController(BuildEnvironmentController):
177 """ Implementation of the BuildEnvironmentController for the localhost;
178 this controller manages the default build directory,
179 the server setup and system start and stop for the localhost-type build environment
180
181 """
182
183 def __init__(self, be):
184 super(LocalhostBEController, self).__init__(be)
185 self.dburl = settings.getDATABASE_URL()
186 self.pokydirname = None
187
188 def _shellcmd(self, command, cwd = None):
189 if cwd is None:
190 cwd = self.be.sourcedir
191
192 p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
193 (out,err) = p.communicate()
194 if p.returncode:
195 if len(err) == 0:
196 err = "command: %s \n%s" % (command, out)
197 else:
198 err = "command: %s \n%s" % (command, err)
199 raise ShellCmdException(err)
200 else:
201 return out
202
203 def _createdirpath(self, path):
204 from os.path import dirname as DN
205 if not os.path.exists(DN(path)):
206 self._createdirpath(DN(path))
207 if not os.path.exists(path):
208 os.mkdir(path, 0755)
209
210 def _startBE(self):
211 assert self.pokydirname and os.path.exists(self.pokydirname)
212 self._createdirpath(self.be.builddir)
213 self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
214
215 def startBBServer(self):
216 assert self.pokydirname and os.path.exists(self.pokydirname)
217 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
218 # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
219 # but since they start async without any return, we just wait a bit
220 print "Started server"
221 assert self.be.sourcedir and os.path.exists(self.be.builddir)
222 self.be.bbaddress = "localhost"
223 self.be.bbport = "8200"
224 self.be.bbstate = BuildEnvironment.SERVER_STARTED
225 self.be.save()
226
227 def stopBBServer(self):
228 assert self.be.sourcedir
229 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
230 (self.be.sourcedir, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
231 self.be.bbstate = BuildEnvironment.SERVER_STOPPED
232 self.be.save()
233 print "Stopped server"
234
235 def setLayers(self, bitbakes, layers):
236 """ a word of attention: by convention, the first layer for any build will be poky! """
237
238 assert self.be.sourcedir is not None
239 assert len(bitbakes) == 1
240 # set layers in the layersource
241
242 # 1. get a list of repos, and map dirpaths for each layer
243 gitrepos = {}
244 gitrepos[bitbakes[0].giturl] = []
245 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
246
247 for layer in layers:
248 # we don't process local URLs
249 if layer.giturl.startswith("file://"):
250 continue
251 if not layer.giturl in gitrepos:
252 gitrepos[layer.giturl] = []
253 gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
254 for giturl in gitrepos.keys():
255 commitid = gitrepos[giturl][0][2]
256 for e in gitrepos[giturl]:
257 if commitid != e[2]:
258 raise BuildSetupException("More than one commit per git url, unsupported configuration")
259
260 def _getgitdirectoryname(url):
261 import re
262 components = re.split(r'[:\.\/]', url)
263 return components[-2] if components[-1] == "git" else components[-1]
264
265 layerlist = []
266
267 # 2. checkout the repositories
268 for giturl in gitrepos.keys():
269 localdirname = os.path.join(self.be.sourcedir, _getgitdirectoryname(giturl))
270 print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
271
272 # make sure our directory is a git repository
273 if os.path.exists(localdirname):
274 if not giturl in self._shellcmd("git remote -v", localdirname):
275 raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
276 else:
277 self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
278 # checkout the needed commit
279 commit = gitrepos[giturl][0][2]
280
281 # branch magic name "HEAD" will inhibit checkout
282 if commit != "HEAD":
283 print "DEBUG: checking out commit ", commit, "to", localdirname
284 self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
285
286 # take the localdirname as poky dir if we can find the oe-init-build-env
287 if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
288 print "DEBUG: selected poky dir name", localdirname
289 self.pokydirname = localdirname
290
291 # verify our repositories
292 for name, dirpath, commit in gitrepos[giturl]:
293 localdirpath = os.path.join(localdirname, dirpath)
294 if not os.path.exists(localdirpath):
295 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
296
297 if name != "bitbake":
298 layerlist.append(localdirpath)
299
300 print "DEBUG: current layer list ", layerlist
301
302 # 3. configure the build environment, so we have a conf/bblayers.conf
303 assert self.pokydirname is not None
304 self._startBE()
305
306 # 4. update the bblayers.conf
307 bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
308 if not os.path.exists(bblayerconf):
309 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
310
311 conflines = open(bblayerconf, "r").readlines()
312
313 bblayerconffile = open(bblayerconf, "w")
314 for i in xrange(len(conflines)):
315 if conflines[i].startswith("# line added by toaster"):
316 i += 2
317 else:
318 bblayerconffile.write(conflines[i])
319
320 bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
321 bblayerconffile.close()
322
323 return True
324
325 def release(self):
326 assert self.be.sourcedir and os.path.exists(self.be.builddir)
327 import shutil
328 shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
329 assert not os.path.exists(self.be.builddir)
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
new file mode 100644
index 0000000000..fe7fd81fb9
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -0,0 +1,191 @@
1#
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# BitBake Toaster Implementation
6#
7# Copyright (C) 2014 Intel Corporation
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
23import os
24import sys
25import re
26from django.db import transaction
27from django.db.models import Q
28from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
29import subprocess
30
31from toastermain import settings
32
33from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname
34
35class LocalhostBEController(BuildEnvironmentController):
36 """ Implementation of the BuildEnvironmentController for the localhost;
37 this controller manages the default build directory,
38 the server setup and system start and stop for the localhost-type build environment
39
40 """
41
42 def __init__(self, be):
43 super(LocalhostBEController, self).__init__(be)
44 self.dburl = settings.getDATABASE_URL()
45 self.pokydirname = None
46 self.islayerset = False
47
48 def _shellcmd(self, command, cwd = None):
49 if cwd is None:
50 cwd = self.be.sourcedir
51
52 p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
53 (out,err) = p.communicate()
54 if p.returncode:
55 if len(err) == 0:
56 err = "command: %s \n%s" % (command, out)
57 else:
58 err = "command: %s \n%s" % (command, err)
59 raise ShellCmdException(err)
60 else:
61 return out
62
63 def _createdirpath(self, path):
64 from os.path import dirname as DN
65 if path == "":
66 raise Exception("Invalid path creation specified.")
67 if not os.path.exists(DN(path)):
68 self._createdirpath(DN(path))
69 if not os.path.exists(path):
70 os.mkdir(path, 0755)
71
72 def _setupBE(self):
73 assert self.pokydirname and os.path.exists(self.pokydirname)
74 self._createdirpath(self.be.builddir)
75 self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
76
77 def startBBServer(self):
78 assert self.pokydirname and os.path.exists(self.pokydirname)
79 assert self.islayerset
80 print("DEBUG: executing ", "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
81 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
82 # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
83 # but since they start async without any return, we just wait a bit
84 print "Started server"
85 assert self.be.sourcedir and os.path.exists(self.be.builddir)
86 self.be.bbaddress = "localhost"
87 self.be.bbport = "8200"
88 self.be.bbstate = BuildEnvironment.SERVER_STARTED
89 self.be.save()
90
91 def stopBBServer(self):
92 assert self.pokydirname and os.path.exists(self.pokydirname)
93 assert self.islayerset
94 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
95 (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
96 self.be.bbstate = BuildEnvironment.SERVER_STOPPED
97 self.be.save()
98 print "Stopped server"
99
100 def setLayers(self, bitbakes, layers):
101 """ a word of attention: by convention, the first layer for any build will be poky! """
102
103 assert self.be.sourcedir is not None
104 assert len(bitbakes) == 1
105 # set layers in the layersource
106
107 # 1. get a list of repos, and map dirpaths for each layer
108 gitrepos = {}
109 gitrepos[bitbakes[0].giturl] = []
110 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
111
112 for layer in layers:
113 # we don't process local URLs
114 if layer.giturl.startswith("file://"):
115 continue
116 if not layer.giturl in gitrepos:
117 gitrepos[layer.giturl] = []
118 gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
119 for giturl in gitrepos.keys():
120 commitid = gitrepos[giturl][0][2]
121 for e in gitrepos[giturl]:
122 if commitid != e[2]:
123 raise BuildSetupException("More than one commit per git url, unsupported configuration")
124
125
126 layerlist = []
127
128 # 2. checkout the repositories
129 for giturl in gitrepos.keys():
130 localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
131 print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
132
133 # make sure our directory is a git repository
134 if os.path.exists(localdirname):
135 if not giturl in self._shellcmd("git remote -v", localdirname):
136 raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
137 else:
138 self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
139 # checkout the needed commit
140 commit = gitrepos[giturl][0][2]
141
142 # branch magic name "HEAD" will inhibit checkout
143 if commit != "HEAD":
144 print "DEBUG: checking out commit ", commit, "to", localdirname
145 self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
146
147 # take the localdirname as poky dir if we can find the oe-init-build-env
148 if self.pokydirname is None and os.path.exists(os.path.join(localdirname, "oe-init-build-env")):
149 print "DEBUG: selected poky dir name", localdirname
150 self.pokydirname = localdirname
151
152 # verify our repositories
153 for name, dirpath, commit in gitrepos[giturl]:
154 localdirpath = os.path.join(localdirname, dirpath)
155 if not os.path.exists(localdirpath):
156 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
157
158 if name != "bitbake":
159 layerlist.append(localdirpath)
160
161 print "DEBUG: current layer list ", layerlist
162
163 # 3. configure the build environment, so we have a conf/bblayers.conf
164 assert self.pokydirname is not None
165 self._setupBE()
166
167 # 4. update the bblayers.conf
168 bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
169 if not os.path.exists(bblayerconf):
170 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
171
172 conflines = open(bblayerconf, "r").readlines()
173
174 bblayerconffile = open(bblayerconf, "w")
175 for i in xrange(len(conflines)):
176 if conflines[i].startswith("# line added by toaster"):
177 i += 2
178 else:
179 bblayerconffile.write(conflines[i])
180
181 bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
182 bblayerconffile.close()
183
184 self.islayerset = True
185 return True
186
187 def release(self):
188 assert self.be.sourcedir and os.path.exists(self.be.builddir)
189 import shutil
190 shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
191 assert not os.path.exists(self.be.builddir)
diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
new file mode 100644
index 0000000000..64674953dc
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
@@ -0,0 +1,193 @@
1#
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# BitBake Toaster Implementation
6#
7# Copyright (C) 2014 Intel Corporation
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22
23import sys
24import re
25from django.db import transaction
26from django.db.models import Q
27from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
28import subprocess
29
30from toastermain import settings
31
32from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname
33
34def DN(path):
35 return "/".join(path.split("/")[0:-1])
36
37class SSHBEController(BuildEnvironmentController):
38 """ Implementation of the BuildEnvironmentController for the localhost;
39 this controller manages the default build directory,
40 the server setup and system start and stop for the localhost-type build environment
41
42 """
43
44 def __init__(self, be):
45 super(SSHBEController, self).__init__(be)
46 self.dburl = settings.getDATABASE_URL()
47 self.pokydirname = None
48 self.islayerset = False
49
50 def _shellcmd(self, command, cwd = None):
51 if cwd is None:
52 cwd = self.be.sourcedir
53
54 p = subprocess.Popen("ssh %s 'cd %s && %s'" % (self.be.address, cwd, command), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
55 (out,err) = p.communicate()
56 if p.returncode:
57 if len(err) == 0:
58 err = "command: %s \n%s" % (command, out)
59 else:
60 err = "command: %s \n%s" % (command, err)
61 raise ShellCmdException(err)
62 else:
63 return out.strip()
64
65 def _pathexists(self, path):
66 try:
67 self._shellcmd("test -e \"%s\"" % path)
68 return True
69 except ShellCmdException as e:
70 return False
71
72 def _pathcreate(self, path):
73 self._shellcmd("mkdir -p \"%s\"" % path)
74
75 def _setupBE(self):
76 assert self.pokydirname and self._pathexists(self.pokydirname)
77 self._pathcreate(self.be.builddir)
78 self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
79
80 def startBBServer(self):
81 assert self.pokydirname and self._pathexists(self.pokydirname)
82 assert self.islayerset
83 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
84 # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
85 # but since they start async without any return, we just wait a bit
86 print "Started server"
87 assert self.be.sourcedir and self._pathexists(self.be.builddir)
88 self.be.bbaddress = self.be.address.split("@")[-1]
89 self.be.bbport = "8200"
90 self.be.bbstate = BuildEnvironment.SERVER_STARTED
91 self.be.save()
92
93 def stopBBServer(self):
94 assert self.pokydirname and self._pathexists(self.pokydirname)
95 assert self.islayerset
96 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && %s source toaster stop\"" %
97 (self.pokydirname, self.be.builddir, (lambda: "" if self.be.bbtoken is None else "BBTOKEN=%s" % self.be.bbtoken)()))
98 self.be.bbstate = BuildEnvironment.SERVER_STOPPED
99 self.be.save()
100 print "Stopped server"
101
102 def setLayers(self, bitbakes, layers):
103 """ a word of attention: by convention, the first layer for any build will be poky! """
104
105 assert self.be.sourcedir is not None
106 assert len(bitbakes) == 1
107 # set layers in the layersource
108
109 # 1. get a list of repos, and map dirpaths for each layer
110 gitrepos = {}
111 gitrepos[bitbakes[0].giturl] = []
112 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
113
114 for layer in layers:
115 # we don't process local URLs
116 if layer.giturl.startswith("file://"):
117 continue
118 if not layer.giturl in gitrepos:
119 gitrepos[layer.giturl] = []
120 gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
121 for giturl in gitrepos.keys():
122 commitid = gitrepos[giturl][0][2]
123 for e in gitrepos[giturl]:
124 if commitid != e[2]:
125 raise BuildSetupException("More than one commit per git url, unsupported configuration")
126
127 layerlist = []
128
129 # 2. checkout the repositories
130 for giturl in gitrepos.keys():
131 import os
132 localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
133 print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
134
135 # make sure our directory is a git repository
136 if self._pathexists(localdirname):
137 if not giturl in self._shellcmd("git remote -v", localdirname):
138 raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
139 else:
140 self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
141 # checkout the needed commit
142 commit = gitrepos[giturl][0][2]
143
144 # branch magic name "HEAD" will inhibit checkout
145 if commit != "HEAD":
146 print "DEBUG: checking out commit ", commit, "to", localdirname
147 self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
148
149 # take the localdirname as poky dir if we can find the oe-init-build-env
150 if self.pokydirname is None and self._pathexists(os.path.join(localdirname, "oe-init-build-env")):
151 print "DEBUG: selected poky dir name", localdirname
152 self.pokydirname = localdirname
153
154 # verify our repositories
155 for name, dirpath, commit in gitrepos[giturl]:
156 localdirpath = os.path.join(localdirname, dirpath)
157 if not self._pathexists(localdirpath):
158 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
159
160 if name != "bitbake":
161 layerlist.append(localdirpath)
162
163 print "DEBUG: current layer list ", layerlist
164
165 # 3. configure the build environment, so we have a conf/bblayers.conf
166 assert self.pokydirname is not None
167 self._setupBE()
168
169 # 4. update the bblayers.conf
170 bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
171 if not self._pathexists(bblayerconf):
172 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
173
174 conflines = open(bblayerconf, "r").readlines()
175
176 bblayerconffile = open(bblayerconf, "w")
177 for i in xrange(len(conflines)):
178 if conflines[i].startswith("# line added by toaster"):
179 i += 2
180 else:
181 bblayerconffile.write(conflines[i])
182
183 bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
184 bblayerconffile.close()
185
186 self.islayerset = True
187 return True
188
189 def release(self):
190 assert self.be.sourcedir and self._pathexists(self.be.builddir)
191 import shutil
192 shutil.rmtree(os.path.join(self.be.sourcedir, "build"))
193 assert not self._pathexists(self.be.builddir)
diff --git a/bitbake/lib/toaster/bldcontrol/tests.py b/bitbake/lib/toaster/bldcontrol/tests.py
index ebe477d8a8..4577c3f03b 100644
--- a/bitbake/lib/toaster/bldcontrol/tests.py
+++ b/bitbake/lib/toaster/bldcontrol/tests.py
@@ -7,46 +7,114 @@ Replace this with more appropriate tests for your application.
7 7
8from django.test import TestCase 8from django.test import TestCase
9 9
10from bldcontrol.bbcontroller import LocalhostBEController, BitbakeController 10from bldcontrol.bbcontroller import BitbakeController
11from bldcontrol.localhostbecontroller import LocalhostBEController
12from bldcontrol.sshbecontroller import SSHBEController
11from bldcontrol.models import BuildEnvironment, BuildRequest 13from bldcontrol.models import BuildEnvironment, BuildRequest
12from bldcontrol.management.commands.runbuilds import Command 14from bldcontrol.management.commands.runbuilds import Command
13 15
14import socket 16import socket
15import subprocess 17import subprocess
16 18
17class LocalhostBEControllerTests(TestCase): 19# standard poky data hardcoded for testing
18 def test_StartAndStopServer(self): 20BITBAKE_LAYERS = [type('bitbake_info', (object,), { "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "", "commit": "HEAD"})]
19 obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL) 21POKY_LAYERS = [
20 lbc = LocalhostBEController(obe) 22 type('poky_info', (object,), { "name": "meta", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta", "commit": "HEAD"}),
23 type('poky_info', (object,), { "name": "meta-yocto", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto", "commit": "HEAD"}),
24 type('poky_info', (object,), { "name": "meta-yocto-bsp", "giturl": "git://git.yoctoproject.org/poky.git", "dirpath": "meta-yocto-bsp", "commit": "HEAD"}),
25 ]
26
21 27
22 # test start server and stop
23 self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Port already occupied")
24 lbc.startBBServer()
25 self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Server not answering")
26 28
27 lbc.stopBBServer() 29# we have an abstract test class designed to ensure that the controllers use a single interface
28 self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('localhost', 8200)), "Server not stopped") 30# specific controller tests only need to override the _getBuildEnvironment() method
29 31
30 # clean up 32class BEControllerTests(object):
31 import subprocess
32 out, err = subprocess.Popen("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
33 33
34 def _serverForceStop(self, bc):
35 err = bc._shellcmd("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill")
34 self.assertTrue(err == '', "bitbake server pid %s not stopped" % err) 36 self.assertTrue(err == '', "bitbake server pid %s not stopped" % err)
35 37
36 obe = BuildEnvironment.objects.create(lock = BuildEnvironment.LOCK_FREE, betype = BuildEnvironment.TYPE_LOCAL) 38 def test_serverStartAndStop(self):
37 lbc = LocalhostBEController(obe) 39 obe = self._getBuildEnvironment()
40 bc = self._getBEController(obe)
41 bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
42
43 hostname = self.test_address.split("@")[-1]
38 44
39 bbc = lbc.getBBController() 45 # test start server and stop
46 self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Port already occupied")
47 bc.startBBServer()
48 self.assertFalse(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not answering")
49
50 bc.stopBBServer()
51 self.assertTrue(socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex((hostname, 8200)), "Server not stopped")
52
53 self._serverForceStop(bc)
54
55 def test_getBBController(self):
56 obe = self._getBuildEnvironment()
57 bc = self._getBEController(obe)
58 bc.setLayers(BITBAKE_LAYERS, POKY_LAYERS) # setting layers, skip any layer info
59
60 bbc = bc.getBBController()
40 self.assertTrue(isinstance(bbc, BitbakeController)) 61 self.assertTrue(isinstance(bbc, BitbakeController))
41 # test set variable 62 # test set variable, use no build marker -1 for BR value
42 try: 63 try:
43 bbc.setVariable 64 bbc.setVariable("TOASTER_BRBE", "%d:%d" % (-1, obe.pk))
44 except Exception as e : 65 except Exception as e :
45 self.fail("setVariable raised %s", e) 66 self.fail("setVariable raised %s", e)
46 67
47 lbc.stopBBServer() 68 bc.stopBBServer()
48 out, err = subprocess.Popen("netstat -tapn 2>/dev/null | grep 8200 | awk '{print $7}' | sort -fu | cut -d \"/\" -f 1 | grep -v -- - | tee /dev/fd/2 | xargs -r kill", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() 69
49 self.assertTrue(err == '', "bitbake server pid %s not stopped" % err) 70 self._serverForceStop(bc)
71
72class LocalhostBEControllerTests(TestCase, BEControllerTests):
73 def __init__(self, *args):
74 super(LocalhostBEControllerTests, self).__init__(*args)
75 # hardcoded for Alex's machine; since the localhost BE is machine-dependent,
76 # I found no good way to abstractize this
77 self.test_sourcedir = "/home/ddalex/ssd/yocto"
78 self.test_builddir = "/home/ddalex/ssd/yocto/build"
79 self.test_address = "localhost"
80
81 def _getBuildEnvironment(self):
82 return BuildEnvironment.objects.create(
83 lock = BuildEnvironment.LOCK_FREE,
84 betype = BuildEnvironment.TYPE_LOCAL,
85 address = self.test_address,
86 sourcedir = self.test_sourcedir,
87 builddir = self.test_builddir )
88
89 def _getBEController(self, obe):
90 return LocalhostBEController(obe)
91
92class SSHBEControllerTests(TestCase, BEControllerTests):
93 def __init__(self, *args):
94 super(SSHBEControllerTests, self).__init__(*args)
95 self.test_address = "ddalex-desktop.local"
96 # hardcoded for ddalex-desktop.local machine; since the localhost BE is machine-dependent,
97 # I found no good way to abstractize this
98 self.test_sourcedir = "/home/ddalex/ssd/yocto"
99 self.test_builddir = "/home/ddalex/ssd/yocto/build"
100
101 def _getBuildEnvironment(self):
102 return BuildEnvironment.objects.create(
103 lock = BuildEnvironment.LOCK_FREE,
104 betype = BuildEnvironment.TYPE_SSH,
105 address = self.test_address,
106 sourcedir = self.test_sourcedir,
107 builddir = self.test_builddir )
108
109 def _getBEController(self, obe):
110 return SSHBEController(obe)
111
112 def test_pathExists(self):
113 obe = BuildEnvironment.objects.create(betype = BuildEnvironment.TYPE_SSH, address= self.test_address)
114 sbc = SSHBEController(obe)
115 self.assertTrue(sbc._pathexists("/"))
116 self.assertFalse(sbc._pathexists("/.deadbeef"))
117 self.assertTrue(sbc._pathexists(sbc._shellcmd("pwd")))
50 118
51 119
52class RunBuildsCommandTests(TestCase): 120class RunBuildsCommandTests(TestCase):
@@ -67,8 +135,8 @@ class RunBuildsCommandTests(TestCase):
67 self.assertRaises(IndexError, command._selectBuildEnvironment) 135 self.assertRaises(IndexError, command._selectBuildEnvironment)
68 136
69 def test_br_select(self): 137 def test_br_select(self):
70 from orm.models import Project 138 from orm.models import Project, Release, BitbakeVersion
71 p, created = Project.objects.get_or_create(pk=1) 139 p = Project.objects.create_project("test", Release.objects.get_or_create(name = "HEAD", bitbake_version = BitbakeVersion.objects.get_or_create(name="HEAD", branch="HEAD")[0])[0])
72 obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p) 140 obr = BuildRequest.objects.create(state = BuildRequest.REQ_QUEUED, project = p)
73 command = Command() 141 command = Command()
74 br = command._selectBuildRequest() 142 br = command._selectBuildRequest()