summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/toaster
diff options
context:
space:
mode:
authorAlexandru DAMIAN <alexandru.damian@intel.com>2015-01-28 13:18:09 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-01-30 15:14:58 +0000
commitafe85485fe185b663a1285b5c27de3160bf06cf7 (patch)
tree487cdf0fbbd11353094c1f68bc1034b3390dd7e6 /bitbake/lib/toaster
parentc58c94824b29665f52ae67ceae3f096930c89fe5 (diff)
downloadpoky-afe85485fe185b663a1285b5c27de3160bf06cf7.tar.gz
bitbake: toaster: new layer checkout logic
This patch implements a new layer checkout logic that brings more flexibility in getting layers under different commits work with Toaster. The new hibrid logic will checkout separately each layer and commit id; the task execution will be delegated to the checkedout bitbake, but the data logger will be executed from the current toaster version as to bring in enough data to sustain the updates in UI models. [YOCTO #7171] [YOCTO #6892] (Bitbake rev: c6eb0f7f16c59530c2525b2e5629fe86de4e8f0f) Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster')
-rw-r--r--bitbake/lib/toaster/bldcontrol/bbcontroller.py13
-rw-r--r--bitbake/lib/toaster/bldcontrol/localhostbecontroller.py107
-rw-r--r--bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py7
-rw-r--r--bitbake/lib/toaster/bldcontrol/sshbecontroller.py58
4 files changed, 92 insertions, 93 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
index 102606e929..dbfb2f3a04 100644
--- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py
@@ -81,12 +81,17 @@ def getBuildEnvironmentController(**kwargs):
81 raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) 81 raise Exception("FIXME: Implement BEC for type %s" % str(be.betype))
82 82
83 83
84def _getgitcheckoutdirectoryname(url): 84def _get_git_clonedirectory(url, branch):
85 """ Utility that returns the last component of a git path as directory 85 """ Utility that returns the last component of a git path as directory
86 """ 86 """
87 import re 87 import re
88 components = re.split(r'[:\.\/]', url) 88 components = re.split(r'[:\.\/]', url)
89 return components[-2] if components[-1] == "git" else components[-1] 89 base = components[-2] if components[-1] == "git" else components[-1]
90
91 if branch != "HEAD":
92 return "_%s_%s.toaster_cloned" % (base, branch)
93
94 return base
90 95
91 96
92class BuildEnvironmentController(object): 97class BuildEnvironmentController(object):
@@ -166,12 +171,12 @@ class BuildEnvironmentController(object):
166 raise Exception("Must override setLayers") 171 raise Exception("Must override setLayers")
167 172
168 173
169 def getBBController(self, brbe): 174 def getBBController(self):
170 """ returns a BitbakeController to an already started server; this is the point where the server 175 """ returns a BitbakeController to an already started server; this is the point where the server
171 starts if needed; or reconnects to the server if we can 176 starts if needed; or reconnects to the server if we can
172 """ 177 """
173 if not self.connection: 178 if not self.connection:
174 self.startBBServer(brbe) 179 self.startBBServer()
175 self.be.lock = BuildEnvironment.LOCK_RUNNING 180 self.be.lock = BuildEnvironment.LOCK_RUNNING
176 self.be.save() 181 self.be.save()
177 182
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index 22d31e33f2..980751fb96 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -30,11 +30,12 @@ import subprocess
30 30
31from toastermain import settings 31from toastermain import settings
32 32
33from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname 33from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory
34 34
35import logging 35import logging
36logger = logging.getLogger("toaster") 36logger = logging.getLogger("toaster")
37 37
38from pprint import pprint, pformat
38 39
39class LocalhostBEController(BuildEnvironmentController): 40class LocalhostBEController(BuildEnvironmentController):
40 """ Implementation of the BuildEnvironmentController for the localhost; 41 """ Implementation of the BuildEnvironmentController for the localhost;
@@ -55,15 +56,16 @@ class LocalhostBEController(BuildEnvironmentController):
55 56
56 p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 57 p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
57 (out,err) = p.communicate() 58 (out,err) = p.communicate()
59 p.wait()
58 if p.returncode: 60 if p.returncode:
59 if len(err) == 0: 61 if len(err) == 0:
60 err = "command: %s \n%s" % (command, out) 62 err = "command: %s \n%s" % (command, out)
61 else: 63 else:
62 err = "command: %s \n%s" % (command, err) 64 err = "command: %s \n%s" % (command, err)
63 logger.debug("localhostbecontroller: shellcmd error %s" % err) 65 #logger.debug("localhostbecontroller: shellcmd error %s" % err)
64 raise ShellCmdException(err) 66 raise ShellCmdException(err)
65 else: 67 else:
66 logger.debug("localhostbecontroller: shellcmd success") 68 #logger.debug("localhostbecontroller: shellcmd success")
67 return out 69 return out
68 70
69 def _createdirpath(self, path): 71 def _createdirpath(self, path):
@@ -80,10 +82,19 @@ class LocalhostBEController(BuildEnvironmentController):
80 self._createdirpath(self.be.builddir) 82 self._createdirpath(self.be.builddir)
81 self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) 83 self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir))
82 84
83 def startBBServer(self, brbe): 85 def startBBServer(self):
84 assert self.pokydirname and os.path.exists(self.pokydirname) 86 assert self.pokydirname and os.path.exists(self.pokydirname)
85 assert self.islayerset 87 assert self.islayerset
86 88
89 # find our own toasterui listener/bitbake
90 from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path
91
92 own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake"))
93
94 assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake)
95
96 logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake)
97
87 try: 98 try:
88 os.remove(os.path.join(self.be.builddir, "toaster_ui.log")) 99 os.remove(os.path.join(self.be.builddir, "toaster_ui.log"))
89 except OSError as e: 100 except OSError as e:
@@ -91,7 +102,10 @@ class LocalhostBEController(BuildEnvironmentController):
91 if e.errno != errno.ENOENT: 102 if e.errno != errno.ENOENT:
92 raise 103 raise
93 104
94 cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe) 105
106 cmd = "bash -c \"source %s/oe-init-build-env %s && bitbake --read conf/toaster-pre.conf --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 && DATABASE_URL=%s BBSERVER=0.0.0.0:-1 daemon -d -i -D %s -o toaster_ui.log -- %s --observe-only -u toasterui &\"" % (self.pokydirname, self.be.builddir,
107 self.dburl, self.be.builddir, own_bitbake)
108 logger.debug("fullcommand |%s| " % cmd)
95 port = "-1" 109 port = "-1"
96 for i in self._shellcmd(cmd).split("\n"): 110 for i in self._shellcmd(cmd).split("\n"):
97 if i.startswith("Bitbake server address"): 111 if i.startswith("Bitbake server address"):
@@ -113,6 +127,16 @@ class LocalhostBEController(BuildEnvironmentController):
113 time.sleep(0.5) 127 time.sleep(0.5)
114 128
115 logger.debug("localhostbecontroller: Started bitbake server") 129 logger.debug("localhostbecontroller: Started bitbake server")
130
131 while port == "-1":
132 # the port specification is "autodetect"; read the bitbake.lock file
133 with open("%s/bitbake.lock" % self.be.builddir, "r") as f:
134 for line in f.readlines():
135 if ":" in line:
136 port = line.split(":")[1].strip()
137 logger.debug("localhostbecontroller: Autodetected bitbake port %s", port)
138 break
139
116 assert self.be.sourcedir and os.path.exists(self.be.builddir) 140 assert self.be.sourcedir and os.path.exists(self.be.builddir)
117 self.be.bbaddress = "localhost" 141 self.be.bbaddress = "localhost"
118 self.be.bbport = port 142 self.be.bbport = port
@@ -135,42 +159,62 @@ class LocalhostBEController(BuildEnvironmentController):
135 assert len(bitbakes) == 1 159 assert len(bitbakes) == 1
136 # set layers in the layersource 160 # set layers in the layersource
137 161
138 # 1. get a list of repos, and map dirpaths for each layer 162 # 1. get a list of repos with branches, and map dirpaths for each layer
139 gitrepos = {} 163 gitrepos = {}
140 gitrepos[bitbakes[0].giturl] = [] 164
141 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) 165 gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = []
166 gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) )
142 167
143 for layer in layers: 168 for layer in layers:
144 # we don't process local URLs 169 # we don't process local URLs
145 if layer.giturl.startswith("file://"): 170 if layer.giturl.startswith("file://"):
146 continue 171 continue
147 if not layer.giturl in gitrepos: 172 if not (layer.giturl, layer.commit) in gitrepos:
148 gitrepos[layer.giturl] = [] 173 gitrepos[(layer.giturl, layer.commit)] = []
149 gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) 174 gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
150 for giturl in gitrepos.keys(): 175
151 commitid = gitrepos[giturl][0][2] 176
152 for e in gitrepos[giturl]: 177 logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
153 if commitid != e[2]: 178
154 import pprint 179
155 raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) 180 # 2. find checked-out git repos in the sourcedir directory that may help faster cloning
156 181
157 182 cached_layers = {}
158 logger.debug("localhostbecontroller, our git repos are %s" % gitrepos) 183 for ldir in os.listdir(self.be.sourcedir):
184 fldir = os.path.join(self.be.sourcedir, ldir)
185 if os.path.isdir(fldir):
186 try:
187 for line in self._shellcmd("git remote -v", fldir).split("\n"):
188 try:
189 remote = line.split("\t")[1].split(" ")[0]
190 if remote not in cached_layers:
191 cached_layers[remote] = fldir
192 except IndexError:
193 pass
194 except ShellCmdException:
195 # ignore any errors in collecting git remotes
196 pass
197
159 layerlist = [] 198 layerlist = []
160 199
161 # 2. checkout the repositories 200 # 3. checkout the repositories
162 for giturl in gitrepos.keys(): 201 for giturl, commit in gitrepos.keys():
163 localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) 202 localdirname = os.path.join(self.be.sourcedir, _get_git_clonedirectory(giturl, commit))
164 logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname)) 203 logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname))
165 204
166 # make sure our directory is a git repository 205 # make sure our directory is a git repository
167 if os.path.exists(localdirname): 206 if os.path.exists(localdirname):
168 if not giturl in self._shellcmd("git remote -v", localdirname): 207 if not giturl in self._shellcmd("git remote -v", localdirname):
169 raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) 208 raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
170 else: 209 else:
171 self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) 210 if giturl in cached_layers:
172 # checkout the needed commit 211 logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname))
173 commit = gitrepos[giturl][0][2] 212 self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname))
213 self._shellcmd("git remote remove origin", localdirname)
214 self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname)
215 else:
216 logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname))
217 self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname))
174 218
175 # branch magic name "HEAD" will inhibit checkout 219 # branch magic name "HEAD" will inhibit checkout
176 if commit != "HEAD": 220 if commit != "HEAD":
@@ -188,21 +232,22 @@ class LocalhostBEController(BuildEnvironmentController):
188 self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) 232 self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake')))
189 233
190 # verify our repositories 234 # verify our repositories
191 for name, dirpath, commit in gitrepos[giturl]: 235 for name, dirpath in gitrepos[(giturl, commit)]:
192 localdirpath = os.path.join(localdirname, dirpath) 236 localdirpath = os.path.join(localdirname, dirpath)
237 logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
193 if not os.path.exists(localdirpath): 238 if not os.path.exists(localdirpath):
194 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) 239 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
195 240
196 if name != "bitbake": 241 if name != "bitbake":
197 layerlist.append(localdirpath.rstrip("/")) 242 layerlist.append(localdirpath.rstrip("/"))
198 243
199 logger.debug("localhostbecontroller: current layer list %s " % layerlist) 244 logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
200 245
201 # 3. configure the build environment, so we have a conf/bblayers.conf 246 # 4. configure the build environment, so we have a conf/bblayers.conf
202 assert self.pokydirname is not None 247 assert self.pokydirname is not None
203 self._setupBE() 248 self._setupBE()
204 249
205 # 4. update the bblayers.conf 250 # 5. update the bblayers.conf
206 bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") 251 bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf")
207 if not os.path.exists(bblayerconf): 252 if not os.path.exists(bblayerconf):
208 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) 253 raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf)
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
index bdce6ee902..23ee855558 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -2,7 +2,7 @@ from django.core.management.base import NoArgsCommand, CommandError
2from django.db import transaction 2from django.db import transaction
3from orm.models import Build 3from orm.models import Build
4from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException 4from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException
5from bldcontrol.models import BuildRequest, BuildEnvironment, BRError 5from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable
6import os 6import os
7import logging 7import logging
8 8
@@ -47,6 +47,9 @@ class Command(NoArgsCommand):
47 return 47 return
48 48
49 logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be)) 49 logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be))
50
51 # write the build identification variable
52 BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk))
50 # let the build request know where it is being executed 53 # let the build request know where it is being executed
51 br.environment = bec.be 54 br.environment = bec.be
52 br.save() 55 br.save()
@@ -56,7 +59,7 @@ class Command(NoArgsCommand):
56 bec.writePreConfFile(br.brvariable_set.all()) 59 bec.writePreConfFile(br.brvariable_set.all())
57 60
58 # get the bb server running with the build req id and build env id 61 # get the bb server running with the build req id and build env id
59 bbctrl = bec.getBBController("%d:%d" % (br.pk, bec.be.pk)) 62 bbctrl = bec.getBBController()
60 63
61 # trigger the build command 64 # trigger the build command
62 task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, br.brtarget_set.all())) 65 task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, br.brtarget_set.all()))
diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
index 45e15392e5..be797c9486 100644
--- a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py
@@ -29,7 +29,7 @@ import subprocess
29 29
30from toastermain import settings 30from toastermain import settings
31 31
32from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname 32from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory
33 33
34def DN(path): 34def DN(path):
35 return "/".join(path.split("/")[0:-1]) 35 return "/".join(path.split("/")[0:-1])
@@ -124,62 +124,8 @@ class SSHBEController(BuildEnvironmentController):
124 assert len(bitbakes) == 1 124 assert len(bitbakes) == 1
125 # set layers in the layersource 125 # set layers in the layersource
126 126
127 # 1. get a list of repos, and map dirpaths for each layer
128 gitrepos = {}
129 gitrepos[bitbakes[0].giturl] = []
130 gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) )
131
132 for layer in layers:
133 # we don't process local URLs
134 if layer.giturl.startswith("file://"):
135 continue
136 if not layer.giturl in gitrepos:
137 gitrepos[layer.giturl] = []
138 gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit))
139 for giturl in gitrepos.keys():
140 commitid = gitrepos[giturl][0][2]
141 for e in gitrepos[giturl]:
142 if commitid != e[2]:
143 raise BuildSetupException("More than one commit per git url, unsupported configuration")
144
145 layerlist = []
146
147 # 2. checkout the repositories
148 for giturl in gitrepos.keys():
149 import os
150 localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl))
151 print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname
152
153 # make sure our directory is a git repository
154 if self._pathexists(localdirname):
155 if not giturl in self._shellcmd("git remote -v", localdirname):
156 raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl))
157 else:
158 self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname))
159 # checkout the needed commit
160 commit = gitrepos[giturl][0][2]
161
162 # branch magic name "HEAD" will inhibit checkout
163 if commit != "HEAD":
164 print "DEBUG: checking out commit ", commit, "to", localdirname
165 self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname)
166
167 # take the localdirname as poky dir if we can find the oe-init-build-env
168 if self.pokydirname is None and self._pathexists(os.path.join(localdirname, "oe-init-build-env")):
169 print "DEBUG: selected poky dir name", localdirname
170 self.pokydirname = localdirname
171
172 # verify our repositories
173 for name, dirpath, commit in gitrepos[giturl]:
174 localdirpath = os.path.join(localdirname, dirpath)
175 if not self._pathexists(localdirpath):
176 raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
177
178 if name != "bitbake":
179 layerlist.append(localdirpath)
180
181 print "DEBUG: current layer list ", layerlist
182 127
128 raise Exception("Not implemented: SSH setLayers")
183 # 3. configure the build environment, so we have a conf/bblayers.conf 129 # 3. configure the build environment, so we have a conf/bblayers.conf
184 assert self.pokydirname is not None 130 assert self.pokydirname is not None
185 self._setupBE() 131 self._setupBE()