diff options
author | Alexandru DAMIAN <alexandru.damian@intel.com> | 2015-01-28 13:18:09 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2015-01-30 15:14:58 +0000 |
commit | afe85485fe185b663a1285b5c27de3160bf06cf7 (patch) | |
tree | 487cdf0fbbd11353094c1f68bc1034b3390dd7e6 /bitbake/lib/toaster/bldcontrol/localhostbecontroller.py | |
parent | c58c94824b29665f52ae67ceae3f096930c89fe5 (diff) | |
download | poky-afe85485fe185b663a1285b5c27de3160bf06cf7.tar.gz |
bitbake: toaster: new layer checkout logic
This patch implements a new layer checkout logic that brings more
flexibility in getting layers under different commits work with
Toaster.
The new hibrid logic will checkout separately each layer and commit id;
the task execution will be delegated to the checkedout bitbake, but the
data logger will be executed from the current toaster version as to
bring in enough data to sustain the updates in UI models.
[YOCTO #7171]
[YOCTO #6892]
(Bitbake rev: c6eb0f7f16c59530c2525b2e5629fe86de4e8f0f)
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster/bldcontrol/localhostbecontroller.py')
-rw-r--r-- | bitbake/lib/toaster/bldcontrol/localhostbecontroller.py | 107 |
1 files changed, 76 insertions, 31 deletions
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 22d31e33f2..980751fb96 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py | |||
@@ -30,11 +30,12 @@ import subprocess | |||
30 | 30 | ||
31 | from toastermain import settings | 31 | from toastermain import settings |
32 | 32 | ||
33 | from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname | 33 | from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory |
34 | 34 | ||
35 | import logging | 35 | import logging |
36 | logger = logging.getLogger("toaster") | 36 | logger = logging.getLogger("toaster") |
37 | 37 | ||
38 | from pprint import pprint, pformat | ||
38 | 39 | ||
39 | class LocalhostBEController(BuildEnvironmentController): | 40 | class LocalhostBEController(BuildEnvironmentController): |
40 | """ Implementation of the BuildEnvironmentController for the localhost; | 41 | """ Implementation of the BuildEnvironmentController for the localhost; |
@@ -55,15 +56,16 @@ class LocalhostBEController(BuildEnvironmentController): | |||
55 | 56 | ||
56 | p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | 57 | p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
57 | (out,err) = p.communicate() | 58 | (out,err) = p.communicate() |
59 | p.wait() | ||
58 | if p.returncode: | 60 | if p.returncode: |
59 | if len(err) == 0: | 61 | if len(err) == 0: |
60 | err = "command: %s \n%s" % (command, out) | 62 | err = "command: %s \n%s" % (command, out) |
61 | else: | 63 | else: |
62 | err = "command: %s \n%s" % (command, err) | 64 | err = "command: %s \n%s" % (command, err) |
63 | logger.debug("localhostbecontroller: shellcmd error %s" % err) | 65 | #logger.debug("localhostbecontroller: shellcmd error %s" % err) |
64 | raise ShellCmdException(err) | 66 | raise ShellCmdException(err) |
65 | else: | 67 | else: |
66 | logger.debug("localhostbecontroller: shellcmd success") | 68 | #logger.debug("localhostbecontroller: shellcmd success") |
67 | return out | 69 | return out |
68 | 70 | ||
69 | def _createdirpath(self, path): | 71 | def _createdirpath(self, path): |
@@ -80,10 +82,19 @@ class LocalhostBEController(BuildEnvironmentController): | |||
80 | self._createdirpath(self.be.builddir) | 82 | self._createdirpath(self.be.builddir) |
81 | self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) | 83 | self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) |
82 | 84 | ||
83 | def startBBServer(self, brbe): | 85 | def startBBServer(self): |
84 | assert self.pokydirname and os.path.exists(self.pokydirname) | 86 | assert self.pokydirname and os.path.exists(self.pokydirname) |
85 | assert self.islayerset | 87 | assert self.islayerset |
86 | 88 | ||
89 | # find our own toasterui listener/bitbake | ||
90 | from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path | ||
91 | |||
92 | own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake")) | ||
93 | |||
94 | assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake) | ||
95 | |||
96 | logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake) | ||
97 | |||
87 | try: | 98 | try: |
88 | os.remove(os.path.join(self.be.builddir, "toaster_ui.log")) | 99 | os.remove(os.path.join(self.be.builddir, "toaster_ui.log")) |
89 | except OSError as e: | 100 | except OSError as e: |
@@ -91,7 +102,10 @@ class LocalhostBEController(BuildEnvironmentController): | |||
91 | if e.errno != errno.ENOENT: | 102 | if e.errno != errno.ENOENT: |
92 | raise | 103 | raise |
93 | 104 | ||
94 | cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe) | 105 | |
106 | cmd = "bash -c \"source %s/oe-init-build-env %s && bitbake --read conf/toaster-pre.conf --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 && DATABASE_URL=%s BBSERVER=0.0.0.0:-1 daemon -d -i -D %s -o toaster_ui.log -- %s --observe-only -u toasterui &\"" % (self.pokydirname, self.be.builddir, | ||
107 | self.dburl, self.be.builddir, own_bitbake) | ||
108 | logger.debug("fullcommand |%s| " % cmd) | ||
95 | port = "-1" | 109 | port = "-1" |
96 | for i in self._shellcmd(cmd).split("\n"): | 110 | for i in self._shellcmd(cmd).split("\n"): |
97 | if i.startswith("Bitbake server address"): | 111 | if i.startswith("Bitbake server address"): |
@@ -113,6 +127,16 @@ class LocalhostBEController(BuildEnvironmentController): | |||
113 | time.sleep(0.5) | 127 | time.sleep(0.5) |
114 | 128 | ||
115 | logger.debug("localhostbecontroller: Started bitbake server") | 129 | logger.debug("localhostbecontroller: Started bitbake server") |
130 | |||
131 | while port == "-1": | ||
132 | # the port specification is "autodetect"; read the bitbake.lock file | ||
133 | with open("%s/bitbake.lock" % self.be.builddir, "r") as f: | ||
134 | for line in f.readlines(): | ||
135 | if ":" in line: | ||
136 | port = line.split(":")[1].strip() | ||
137 | logger.debug("localhostbecontroller: Autodetected bitbake port %s", port) | ||
138 | break | ||
139 | |||
116 | assert self.be.sourcedir and os.path.exists(self.be.builddir) | 140 | assert self.be.sourcedir and os.path.exists(self.be.builddir) |
117 | self.be.bbaddress = "localhost" | 141 | self.be.bbaddress = "localhost" |
118 | self.be.bbport = port | 142 | self.be.bbport = port |
@@ -135,42 +159,62 @@ class LocalhostBEController(BuildEnvironmentController): | |||
135 | assert len(bitbakes) == 1 | 159 | assert len(bitbakes) == 1 |
136 | # set layers in the layersource | 160 | # set layers in the layersource |
137 | 161 | ||
138 | # 1. get a list of repos, and map dirpaths for each layer | 162 | # 1. get a list of repos with branches, and map dirpaths for each layer |
139 | gitrepos = {} | 163 | gitrepos = {} |
140 | gitrepos[bitbakes[0].giturl] = [] | 164 | |
141 | gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) | 165 | gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] |
166 | gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) ) | ||
142 | 167 | ||
143 | for layer in layers: | 168 | for layer in layers: |
144 | # we don't process local URLs | 169 | # we don't process local URLs |
145 | if layer.giturl.startswith("file://"): | 170 | if layer.giturl.startswith("file://"): |
146 | continue | 171 | continue |
147 | if not layer.giturl in gitrepos: | 172 | if not (layer.giturl, layer.commit) in gitrepos: |
148 | gitrepos[layer.giturl] = [] | 173 | gitrepos[(layer.giturl, layer.commit)] = [] |
149 | gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) | 174 | gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) |
150 | for giturl in gitrepos.keys(): | 175 | |
151 | commitid = gitrepos[giturl][0][2] | 176 | |
152 | for e in gitrepos[giturl]: | 177 | logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) |
153 | if commitid != e[2]: | 178 | |
154 | import pprint | 179 | |
155 | raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) | 180 | # 2. find checked-out git repos in the sourcedir directory that may help faster cloning |
156 | 181 | ||
157 | 182 | cached_layers = {} | |
158 | logger.debug("localhostbecontroller, our git repos are %s" % gitrepos) | 183 | for ldir in os.listdir(self.be.sourcedir): |
184 | fldir = os.path.join(self.be.sourcedir, ldir) | ||
185 | if os.path.isdir(fldir): | ||
186 | try: | ||
187 | for line in self._shellcmd("git remote -v", fldir).split("\n"): | ||
188 | try: | ||
189 | remote = line.split("\t")[1].split(" ")[0] | ||
190 | if remote not in cached_layers: | ||
191 | cached_layers[remote] = fldir | ||
192 | except IndexError: | ||
193 | pass | ||
194 | except ShellCmdException: | ||
195 | # ignore any errors in collecting git remotes | ||
196 | pass | ||
197 | |||
159 | layerlist = [] | 198 | layerlist = [] |
160 | 199 | ||
161 | # 2. checkout the repositories | 200 | # 3. checkout the repositories |
162 | for giturl in gitrepos.keys(): | 201 | for giturl, commit in gitrepos.keys(): |
163 | localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) | 202 | localdirname = os.path.join(self.be.sourcedir, _get_git_clonedirectory(giturl, commit)) |
164 | logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname)) | 203 | logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) |
165 | 204 | ||
166 | # make sure our directory is a git repository | 205 | # make sure our directory is a git repository |
167 | if os.path.exists(localdirname): | 206 | if os.path.exists(localdirname): |
168 | if not giturl in self._shellcmd("git remote -v", localdirname): | 207 | if not giturl in self._shellcmd("git remote -v", localdirname): |
169 | raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) | 208 | raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) |
170 | else: | 209 | else: |
171 | self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) | 210 | if giturl in cached_layers: |
172 | # checkout the needed commit | 211 | logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) |
173 | commit = gitrepos[giturl][0][2] | 212 | self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) |
213 | self._shellcmd("git remote remove origin", localdirname) | ||
214 | self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) | ||
215 | else: | ||
216 | logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname)) | ||
217 | self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) | ||
174 | 218 | ||
175 | # branch magic name "HEAD" will inhibit checkout | 219 | # branch magic name "HEAD" will inhibit checkout |
176 | if commit != "HEAD": | 220 | if commit != "HEAD": |
@@ -188,21 +232,22 @@ class LocalhostBEController(BuildEnvironmentController): | |||
188 | self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) | 232 | self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) |
189 | 233 | ||
190 | # verify our repositories | 234 | # verify our repositories |
191 | for name, dirpath, commit in gitrepos[giturl]: | 235 | for name, dirpath in gitrepos[(giturl, commit)]: |
192 | localdirpath = os.path.join(localdirname, dirpath) | 236 | localdirpath = os.path.join(localdirname, dirpath) |
237 | logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) | ||
193 | if not os.path.exists(localdirpath): | 238 | if not os.path.exists(localdirpath): |
194 | raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) | 239 | raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) |
195 | 240 | ||
196 | if name != "bitbake": | 241 | if name != "bitbake": |
197 | layerlist.append(localdirpath.rstrip("/")) | 242 | layerlist.append(localdirpath.rstrip("/")) |
198 | 243 | ||
199 | logger.debug("localhostbecontroller: current layer list %s " % layerlist) | 244 | logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) |
200 | 245 | ||
201 | # 3. configure the build environment, so we have a conf/bblayers.conf | 246 | # 4. configure the build environment, so we have a conf/bblayers.conf |
202 | assert self.pokydirname is not None | 247 | assert self.pokydirname is not None |
203 | self._setupBE() | 248 | self._setupBE() |
204 | 249 | ||
205 | # 4. update the bblayers.conf | 250 | # 5. update the bblayers.conf |
206 | bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") | 251 | bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") |
207 | if not os.path.exists(bblayerconf): | 252 | if not os.path.exists(bblayerconf): |
208 | raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) | 253 | raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) |