summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2014-07-15 19:31:10 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-07-23 20:06:58 +0100
commit1b9175af3fabf12af80c8446af94078afd0832ed (patch)
tree180743528e824556bb21672826971cb7f1c36538 /bitbake/lib
parent6e71c276b582135228419d95174b7e7784d496b2 (diff)
downloadpoky-1b9175af3fabf12af80c8446af94078afd0832ed.tar.gz
bitbake: toaster: properly set layers when running a build
This patch enables the localhost build controller to properly set the layers before the build runs. It creates the checkout directories under BuildEnvironment sourcedir directory, and runs the build in the buildir directory. Build launch errors are tracked in the newly added BRError table. These are different from build errors, in the sense that the build can't start due to these errors. (Bitbake rev: 1868d5635b517e0fe1b874674ea7a78910b26e2e) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/toaster/bldcontrol/bbcontroller.py108
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py23
-rw-r--r--bitbake/lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py112
-rw-r--r--bitbake/lib/toaster/bldcontrol/models.py8
4 files changed, 226 insertions, 25 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
index c3beba96f0..1e58c67fcd 100644
--- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
@@ -126,6 +126,8 @@ class BuildEnvironmentController(object):
126 def setLayers(self,ls): 126 def setLayers(self,ls):
127 """ Sets the layer variables in the config file, after validating local layer paths. 127 """ Sets the layer variables in the config file, after validating local layer paths.
128 The layer paths must be in a list of BRLayer object 128 The layer paths must be in a list of BRLayer object
129
130 a word of attention: by convention, the first layer for any build will be poky!
129 """ 131 """
130 raise Exception("Must override setLayers") 132 raise Exception("Must override setLayers")
131 133
@@ -165,25 +167,31 @@ class BuildEnvironmentController(object):
165class ShellCmdException(Exception): 167class ShellCmdException(Exception):
166 pass 168 pass
167 169
170
171class BuildSetupException(Exception):
172 pass
173
168class LocalhostBEController(BuildEnvironmentController): 174class LocalhostBEController(BuildEnvironmentController):
169 """ Implementation of the BuildEnvironmentController for the localhost; 175 """ Implementation of the BuildEnvironmentController for the localhost;
170 this controller manages the default build directory, 176 this controller manages the default build directory,
171 the server setup and system start and stop for the localhost-type build environment 177 the server setup and system start and stop for the localhost-type build environment
172 178
173 """ 179 """
174 from os.path import dirname as DN
175 180
176 def __init__(self, be): 181 def __init__(self, be):
177 super(LocalhostBEController, self).__init__(be) 182 super(LocalhostBEController, self).__init__(be)
178 from os.path import dirname as DN
179 self.dburl = settings.getDATABASE_URL() 183 self.dburl = settings.getDATABASE_URL()
184 self.pokydirname = None
185
186 def _shellcmd(self, command, cwd = None):
187 if cwd is None:
188 cwd = self.be.sourcedir
180 189
181 def _shellcmd(self, command): 190 p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
182 p = subprocess.Popen(command, cwd=self.be.sourcedir, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183 (out,err) = p.communicate() 191 (out,err) = p.communicate()
184 if p.returncode: 192 if p.returncode:
185 if len(err) == 0: 193 if len(err) == 0:
186 err = "command: %s" % command 194 err = "command: %s \n%s" % (command, out)
187 else: 195 else:
188 err = "command: %s \n%s" % (command, err) 196 err = "command: %s \n%s" % (command, err)
189 raise ShellCmdException(err) 197 raise ShellCmdException(err)
@@ -191,22 +199,20 @@ class LocalhostBEController(BuildEnvironmentController):
191 return out 199 return out
192 200
193 def _createdirpath(self, path): 201 def _createdirpath(self, path):
202 from os.path import dirname as DN
194 if not os.path.exists(DN(path)): 203 if not os.path.exists(DN(path)):
195 self._createdirpath(DN(path)) 204 self._createdirpath(DN(path))
196 if not os.path.exists(path): 205 if not os.path.exists(path):
197 os.mkdir(path, 0755) 206 os.mkdir(path, 0755)
198 207
199 def _startBE(self): 208 def _startBE(self):
200 assert self.be.sourcedir and os.path.exists(self.be.sourcedir) 209 assert self.pokydirname and os.path.exists(self.pokydirname)
201 self._createdirpath(self.be.builddir) 210 self._createdirpath(self.be.builddir)
202 self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.be.sourcedir, self.be.builddir)) 211 self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
203 212
204 def startBBServer(self): 213 def startBBServer(self):
205 assert self.be.sourcedir and os.path.exists(self.be.sourcedir) 214 assert self.pokydirname and os.path.exists(self.pokydirname)
206 215 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.pokydirname, self.be.builddir, self.dburl))
207 self._startBE()
208
209 print self._shellcmd("bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb && sleep 1\"" % (self.be.sourcedir, self.be.builddir, self.dburl))
210 # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected 216 # FIXME unfortunate sleep 1 - we need to make sure that bbserver is started and the toaster ui is connected
211 # but since they start async without any return, we just wait a bit 217 # but since they start async without any return, we just wait a bit
212 print "Started server" 218 print "Started server"
@@ -225,10 +231,82 @@ class LocalhostBEController(BuildEnvironmentController):
225 print "Stopped server" 231 print "Stopped server"
226 232
227 def setLayers(self, layers): 233 def setLayers(self, layers):
234 """ a word of attention: by convention, the first layer for any build will be poky! """
235
228 assert self.be.sourcedir is not None 236 assert self.be.sourcedir is not None
229 layerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") 237 # set layers in the layersource
230 if not os.path.exists(layerconf): 238
231 raise Exception("BE is not consistent: bblayers.conf file missing at ", layerconf) 239 # 1. get a list of repos, and map dirpaths for each layer
240 gitrepos = {}
241 for layer in layers:
242 if not layer.giturl in gitrepos:
243 gitrepos[layer.giturl] = []
244 gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
245 for giturl in gitrepos.keys():
246 commitid = gitrepos[giturl][0][2]
247 for e in gitrepos[giturl]:
248 if commitid != e[2]:
249 raise BuildSetupException("More than one commit per git url, unsupported configuration")
250
251 def _getgitdirectoryname(url):
252 import re
253 components = re.split(r'[\.\/]', url)
254 return components[-2] if components[-1] == "git" else components[-1]
255
256 layerlist = []
257
258 # 2. checkout the repositories
259 for giturl in gitrepos.keys():
260 localdirname = os.path.join(self.be.sourcedir, _getgitdirectoryname(giturl))
261 print "DEBUG: giturl checking out in current directory", localdirname
262
263 # make sure our directory is a git repository
264 if os.path.exists(localdirname):
265 if not giturl in self._shellcmd("git remote -v", localdirname):
266 raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
267 else:
268 self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
269 # checkout the needed commit
270 commit = gitrepos[giturl][0][2]
271 self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
272 print "DEBUG: checked out commit ", commit, "to", localdirname
273
274 # if this is the first checkout, take the localdirname as poky dir
275 if self.pokydirname is None:
276 print "DEBUG: selected poky dir name", localdirname
277 self.pokydirname = localdirname
278
279 # verify our repositories
280 for name, dirpath, commit in gitrepos[giturl]:
281 localdirpath = os.path.join(localdirname, dirpath)
282 if not os.path.exists(localdirpath):
283 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
284
285 layerlist.append(localdirpath)
286
287 print "DEBUG: current layer list ", layerlist
288
289 # 3. configure the build environment, so we have a conf/bblayers.conf
290 assert self.pokydirname is not None
291 self._startBE()
292
293 # 4. update the bblayers.conf
294 bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
295 if not os.path.exists(bblayerconf):
296 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
297
298 conflines = open(bblayerconf, "r").readlines()
299
300 bblayerconffile = open(bblayerconf, "w")
301 for i in xrange(len(conflines)):
302 if conflines[i].startswith("# line added by toaster"):
303 i += 2
304 else:
305 bblayerconffile.write(conflines[i])
306
307 bblayerconffile.write("\n# line added by toaster build control\nBBLAYERS = \"" + " ".join(layerlist) + "\"")
308 bblayerconffile.close()
309
232 return True 310 return True
233 311
234 def release(self): 312 def release(self):
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
index dd8f84b07a..fa8c1a9906 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -1,8 +1,8 @@
1from django.core.management.base import NoArgsCommand, CommandError 1from django.core.management.base import NoArgsCommand, CommandError
2from django.db import transaction 2from django.db import transaction
3from orm.models import Build 3from orm.models import Build
4from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException 4from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
5from bldcontrol.models import BuildRequest, BuildEnvironment 5from bldcontrol.models import BuildRequest, BuildEnvironment, BRError
6import os 6import os
7 7
8class Command(NoArgsCommand): 8class Command(NoArgsCommand):
@@ -25,6 +25,7 @@ class Command(NoArgsCommand):
25 return br 25 return br
26 26
27 def schedule(self): 27 def schedule(self):
28 import traceback
28 try: 29 try:
29 br = None 30 br = None
30 try: 31 try:
@@ -63,15 +64,19 @@ class Command(NoArgsCommand):
63 64
64 # cleanup to be performed by toaster when the deed is done 65 # cleanup to be performed by toaster when the deed is done
65 66
66 except ShellCmdException as e:
67 import traceback
68 print " EE Error executing shell command\n", e
69 traceback.format_exc(e)
70 67
71 except Exception as e: 68 except Exception as e:
72 import traceback 69 print " EE Error executing shell command\n", e
73 traceback.print_exc() 70 traceback.print_exc(e)
74 raise e 71 BRError.objects.create(req = br,
72 errtype = str(type(e)),
73 errmsg = str(e),
74 traceback = traceback.format_exc(e))
75 br.state = BuildRequest.REQ_FAILED
76 br.save()
77 bec.be.lock = BuildEnvironment.LOCK_FREE
78 bec.be.save()
79
75 80
76 def cleanup(self): 81 def cleanup(self):
77 from django.utils import timezone 82 from django.utils import timezone
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py b/bitbake/lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py
new file mode 100644
index 0000000000..98aeb41ceb
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/migrations/0005_auto__add_brerror.py
@@ -0,0 +1,112 @@
1# -*- coding: utf-8 -*-
2from south.utils import datetime_utils as datetime
3from south.db import db
4from south.v2 import SchemaMigration
5from django.db import models
6
7
8class Migration(SchemaMigration):
9
10 def forwards(self, orm):
11 # Adding model 'BRError'
12 db.create_table(u'bldcontrol_brerror', (
13 (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
14 ('req', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['bldcontrol.BuildRequest'])),
15 ('errtype', self.gf('django.db.models.fields.CharField')(max_length=100)),
16 ('errmsg', self.gf('django.db.models.fields.TextField')()),
17 ('traceback', self.gf('django.db.models.fields.TextField')()),
18 ))
19 db.send_create_signal(u'bldcontrol', ['BRError'])
20
21
22 def backwards(self, orm):
23 # Deleting model 'BRError'
24 db.delete_table(u'bldcontrol_brerror')
25
26
27 models = {
28 u'bldcontrol.brerror': {
29 'Meta': {'object_name': 'BRError'},
30 'errmsg': ('django.db.models.fields.TextField', [], {}),
31 'errtype': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
32 u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
33 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
34 'traceback': ('django.db.models.fields.TextField', [], {})
35 },
36 u'bldcontrol.brlayer': {
37 'Meta': {'object_name': 'BRLayer'},
38 'commit': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
39 'dirpath': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
40 'giturl': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
41 u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
42 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
43 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"})
44 },
45 u'bldcontrol.brtarget': {
46 'Meta': {'object_name': 'BRTarget'},
47 u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
48 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
49 'target': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
50 'task': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True'})
51 },
52 u'bldcontrol.brvariable': {
53 'Meta': {'object_name': 'BRVariable'},
54 u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
55 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
56 'req': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['bldcontrol.BuildRequest']"}),
57 'value': ('django.db.models.fields.TextField', [], {'blank': 'True'})
58 },
59 u'bldcontrol.buildenvironment': {
60 'Meta': {'object_name': 'BuildEnvironment'},
61 'address': ('django.db.models.fields.CharField', [], {'max_length': '254'}),
62 'bbaddress': ('django.db.models.fields.CharField', [], {'max_length': '254', 'blank': 'True'}),
63 'bbport': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
64 'bbstate': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
65 'bbtoken': ('django.db.models.fields.CharField', [], {'max_length': '126', 'blank': 'True'}),
66 'betype': ('django.db.models.fields.IntegerField', [], {}),
67 'builddir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
68 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
69 u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
70 'lock': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
71 'sourcedir': ('django.db.models.fields.CharField', [], {'max_length': '512', 'blank': 'True'}),
72 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
73 },
74 u'bldcontrol.buildrequest': {
75 'Meta': {'object_name': 'BuildRequest'},
76 'build': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Build']", 'null': 'True'}),
77 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
78 u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
79 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']"}),
80 'state': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
81 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
82 },
83 u'orm.build': {
84 'Meta': {'object_name': 'Build'},
85 'bitbake_version': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
86 'build_name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
87 'completed_on': ('django.db.models.fields.DateTimeField', [], {}),
88 'cooker_log_path': ('django.db.models.fields.CharField', [], {'max_length': '500'}),
89 'distro': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
90 'distro_version': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
91 'errors_no': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
92 u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
93 'machine': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
94 'outcome': ('django.db.models.fields.IntegerField', [], {'default': '2'}),
95 'project': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['orm.Project']", 'null': 'True'}),
96 'started_on': ('django.db.models.fields.DateTimeField', [], {}),
97 'timespent': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
98 'warnings_no': ('django.db.models.fields.IntegerField', [], {'default': '0'})
99 },
100 u'orm.project': {
101 'Meta': {'object_name': 'Project'},
102 'branch': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
103 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
104 u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
105 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
106 'short_description': ('django.db.models.fields.CharField', [], {'max_length': '50', 'blank': 'True'}),
107 'updated': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
108 'user_id': ('django.db.models.fields.IntegerField', [], {'null': 'True'})
109 }
110 }
111
112 complete_apps = ['bldcontrol'] \ No newline at end of file
diff --git a/bitbake/lib/toaster/bldcontrol/models.py b/bitbake/lib/toaster/bldcontrol/models.py
index 1f253ffb22..8c271ffa94 100644
--- a/bitbake/lib/toaster/bldcontrol/models.py
+++ b/bitbake/lib/toaster/bldcontrol/models.py
@@ -48,12 +48,14 @@ class BuildRequest(models.Model):
48 REQ_QUEUED = 1 48 REQ_QUEUED = 1
49 REQ_INPROGRESS = 2 49 REQ_INPROGRESS = 2
50 REQ_COMPLETED = 3 50 REQ_COMPLETED = 3
51 REQ_FAILED = 4
51 52
52 REQUEST_STATE = ( 53 REQUEST_STATE = (
53 (REQ_CREATED, "created"), 54 (REQ_CREATED, "created"),
54 (REQ_QUEUED, "queued"), 55 (REQ_QUEUED, "queued"),
55 (REQ_INPROGRESS, "in progress"), 56 (REQ_INPROGRESS, "in progress"),
56 (REQ_COMPLETED, "completed"), 57 (REQ_COMPLETED, "completed"),
58 (REQ_FAILED, "failed"),
57 ) 59 )
58 60
59 project = models.ForeignKey(Project) 61 project = models.ForeignKey(Project)
@@ -84,4 +86,8 @@ class BRTarget(models.Model):
84 target = models.CharField(max_length=100) 86 target = models.CharField(max_length=100)
85 task = models.CharField(max_length=100, null=True) 87 task = models.CharField(max_length=100, null=True)
86 88
87 89class BRError(models.Model):
90 req = models.ForeignKey(BuildRequest)
91 errtype = models.CharField(max_length=100)
92 errmsg = models.TextField()
93 traceback = models.TextField()