From afe85485fe185b663a1285b5c27de3160bf06cf7 Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Wed, 28 Jan 2015 13:18:09 +0000 Subject: bitbake: toaster: new layer checkout logic This patch implements a new layer checkout logic that brings more flexibility in getting layers under different commits work with Toaster. The new hibrid logic will checkout separately each layer and commit id; the task execution will be delegated to the checkedout bitbake, but the data logger will be executed from the current toaster version as to bring in enough data to sustain the updates in UI models. [YOCTO #7171] [YOCTO #6892] (Bitbake rev: c6eb0f7f16c59530c2525b2e5629fe86de4e8f0f) Signed-off-by: Alexandru DAMIAN Signed-off-by: Richard Purdie --- bitbake/lib/toaster/bldcontrol/bbcontroller.py | 13 ++- .../toaster/bldcontrol/localhostbecontroller.py | 107 +++++++++++++++------ .../bldcontrol/management/commands/runbuilds.py | 7 +- bitbake/lib/toaster/bldcontrol/sshbecontroller.py | 58 +---------- 4 files changed, 92 insertions(+), 93 deletions(-) (limited to 'bitbake') diff --git a/bitbake/lib/toaster/bldcontrol/bbcontroller.py b/bitbake/lib/toaster/bldcontrol/bbcontroller.py index 102606e929..dbfb2f3a04 100644 --- a/bitbake/lib/toaster/bldcontrol/bbcontroller.py +++ b/bitbake/lib/toaster/bldcontrol/bbcontroller.py @@ -81,12 +81,17 @@ def getBuildEnvironmentController(**kwargs): raise Exception("FIXME: Implement BEC for type %s" % str(be.betype)) -def _getgitcheckoutdirectoryname(url): +def _get_git_clonedirectory(url, branch): """ Utility that returns the last component of a git path as directory """ import re components = re.split(r'[:\.\/]', url) - return components[-2] if components[-1] == "git" else components[-1] + base = components[-2] if components[-1] == "git" else components[-1] + + if branch != "HEAD": + return "_%s_%s.toaster_cloned" % (base, branch) + + return base class BuildEnvironmentController(object): @@ -166,12 +171,12 @@ class BuildEnvironmentController(object): raise Exception("Must override setLayers") - def getBBController(self, brbe): + def getBBController(self): """ returns a BitbakeController to an already started server; this is the point where the server starts if needed; or reconnects to the server if we can """ if not self.connection: - self.startBBServer(brbe) + self.startBBServer() self.be.lock = BuildEnvironment.LOCK_RUNNING self.be.save() diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 22d31e33f2..980751fb96 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -30,11 +30,12 @@ import subprocess from toastermain import settings -from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory import logging logger = logging.getLogger("toaster") +from pprint import pprint, pformat class LocalhostBEController(BuildEnvironmentController): """ Implementation of the BuildEnvironmentController for the localhost; @@ -55,15 +56,16 @@ class LocalhostBEController(BuildEnvironmentController): p = subprocess.Popen(command, cwd = cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out,err) = p.communicate() + p.wait() if p.returncode: if len(err) == 0: err = "command: %s \n%s" % (command, out) else: err = "command: %s \n%s" % (command, err) - logger.debug("localhostbecontroller: shellcmd error %s" % err) + #logger.debug("localhostbecontroller: shellcmd error %s" % err) raise ShellCmdException(err) else: - logger.debug("localhostbecontroller: shellcmd success") + #logger.debug("localhostbecontroller: shellcmd success") return out def _createdirpath(self, path): @@ -80,10 +82,19 @@ class LocalhostBEController(BuildEnvironmentController): self._createdirpath(self.be.builddir) self._shellcmd("bash -c \"source %s/oe-init-build-env %s\"" % (self.pokydirname, self.be.builddir)) - def startBBServer(self, brbe): + def startBBServer(self): assert self.pokydirname and os.path.exists(self.pokydirname) assert self.islayerset + # find our own toasterui listener/bitbake + from toaster.bldcontrol.management.commands.loadconf import _reduce_canon_path + + own_bitbake = _reduce_canon_path(os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../bin/bitbake")) + + assert os.path.exists(own_bitbake) and os.path.isfile(own_bitbake) + + logger.debug("localhostbecontroller: running the listener at %s" % own_bitbake) + try: os.remove(os.path.join(self.be.builddir, "toaster_ui.log")) except OSError as e: @@ -91,7 +102,10 @@ class LocalhostBEController(BuildEnvironmentController): if e.errno != errno.ENOENT: raise - cmd = "bash -c \"source %s/oe-init-build-env %s && DATABASE_URL=%s source toaster start noweb brbe=%s\"" % (self.pokydirname, self.be.builddir, self.dburl, brbe) + + cmd = "bash -c \"source %s/oe-init-build-env %s && bitbake --read conf/toaster-pre.conf --postread conf/toaster.conf --server-only -t xmlrpc -B 0.0.0.0:0 && DATABASE_URL=%s BBSERVER=0.0.0.0:-1 daemon -d -i -D %s -o toaster_ui.log -- %s --observe-only -u toasterui &\"" % (self.pokydirname, self.be.builddir, + self.dburl, self.be.builddir, own_bitbake) + logger.debug("fullcommand |%s| " % cmd) port = "-1" for i in self._shellcmd(cmd).split("\n"): if i.startswith("Bitbake server address"): @@ -113,6 +127,16 @@ class LocalhostBEController(BuildEnvironmentController): time.sleep(0.5) logger.debug("localhostbecontroller: Started bitbake server") + + while port == "-1": + # the port specification is "autodetect"; read the bitbake.lock file + with open("%s/bitbake.lock" % self.be.builddir, "r") as f: + for line in f.readlines(): + if ":" in line: + port = line.split(":")[1].strip() + logger.debug("localhostbecontroller: Autodetected bitbake port %s", port) + break + assert self.be.sourcedir and os.path.exists(self.be.builddir) self.be.bbaddress = "localhost" self.be.bbport = port @@ -135,42 +159,62 @@ class LocalhostBEController(BuildEnvironmentController): assert len(bitbakes) == 1 # set layers in the layersource - # 1. get a list of repos, and map dirpaths for each layer + # 1. get a list of repos with branches, and map dirpaths for each layer gitrepos = {} - gitrepos[bitbakes[0].giturl] = [] - gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) + + gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)] = [] + gitrepos[(bitbakes[0].giturl, bitbakes[0].commit)].append( ("bitbake", bitbakes[0].dirpath) ) for layer in layers: # we don't process local URLs if layer.giturl.startswith("file://"): continue - if not layer.giturl in gitrepos: - gitrepos[layer.giturl] = [] - gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) - for giturl in gitrepos.keys(): - commitid = gitrepos[giturl][0][2] - for e in gitrepos[giturl]: - if commitid != e[2]: - import pprint - raise BuildSetupException("More than one commit per git url, unsupported configuration: \n%s" % pprint.pformat(gitrepos)) - - - logger.debug("localhostbecontroller, our git repos are %s" % gitrepos) + if not (layer.giturl, layer.commit) in gitrepos: + gitrepos[(layer.giturl, layer.commit)] = [] + gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) + + + logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) + + + # 2. find checked-out git repos in the sourcedir directory that may help faster cloning + + cached_layers = {} + for ldir in os.listdir(self.be.sourcedir): + fldir = os.path.join(self.be.sourcedir, ldir) + if os.path.isdir(fldir): + try: + for line in self._shellcmd("git remote -v", fldir).split("\n"): + try: + remote = line.split("\t")[1].split(" ")[0] + if remote not in cached_layers: + cached_layers[remote] = fldir + except IndexError: + pass + except ShellCmdException: + # ignore any errors in collecting git remotes + pass + layerlist = [] - # 2. checkout the repositories - for giturl in gitrepos.keys(): - localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) - logger.debug("localhostbecontroller: giturl %s checking out in current directory %s" % (giturl, localdirname)) + # 3. checkout the repositories + for giturl, commit in gitrepos.keys(): + localdirname = os.path.join(self.be.sourcedir, _get_git_clonedirectory(giturl, commit)) + logger.debug("localhostbecontroller: giturl %s:%s checking out in current directory %s" % (giturl, commit, localdirname)) # make sure our directory is a git repository if os.path.exists(localdirname): if not giturl in self._shellcmd("git remote -v", localdirname): raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) else: - self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) - # checkout the needed commit - commit = gitrepos[giturl][0][2] + if giturl in cached_layers: + logger.debug("localhostbecontroller git-copying %s to %s" % (cached_layers[giturl], localdirname)) + self._shellcmd("git clone \"%s\" \"%s\"" % (cached_layers[giturl], localdirname)) + self._shellcmd("git remote remove origin", localdirname) + self._shellcmd("git remote add origin \"%s\"" % giturl, localdirname) + else: + logger.debug("localhostbecontroller: cloning %s:%s in %s" % (giturl, commit, localdirname)) + self._shellcmd("git clone \"%s\" --single-branch --branch \"%s\" \"%s\"" % (giturl, commit, localdirname)) # branch magic name "HEAD" will inhibit checkout if commit != "HEAD": @@ -188,21 +232,22 @@ class LocalhostBEController(BuildEnvironmentController): self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbakes[0].commit, bitbakes[0].giturl, os.path.join(self.pokydirname, 'bitbake'))) # verify our repositories - for name, dirpath, commit in gitrepos[giturl]: + for name, dirpath in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) + logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": layerlist.append(localdirpath.rstrip("/")) - logger.debug("localhostbecontroller: current layer list %s " % layerlist) + logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) - # 3. configure the build environment, so we have a conf/bblayers.conf + # 4. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() - # 4. update the bblayers.conf + # 5. update the bblayers.conf bblayerconf = os.path.join(self.be.builddir, "conf/bblayers.conf") if not os.path.exists(bblayerconf): raise BuildSetupException("BE is not consistent: bblayers.conf file missing at %s" % bblayerconf) diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index bdce6ee902..23ee855558 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -2,7 +2,7 @@ from django.core.management.base import NoArgsCommand, CommandError from django.db import transaction from orm.models import Build from bldcontrol.bbcontroller import getBuildEnvironmentController, ShellCmdException, BuildSetupException -from bldcontrol.models import BuildRequest, BuildEnvironment, BRError +from bldcontrol.models import BuildRequest, BuildEnvironment, BRError, BRVariable import os import logging @@ -47,6 +47,9 @@ class Command(NoArgsCommand): return logger.debug("runbuilds: starting build %s, environment %s" % (br, bec.be)) + + # write the build identification variable + BRVariable.objects.create(req = br, name="TOASTER_BRBE", value="%d:%d" % (br.pk, bec.be.pk)) # let the build request know where it is being executed br.environment = bec.be br.save() @@ -56,7 +59,7 @@ class Command(NoArgsCommand): bec.writePreConfFile(br.brvariable_set.all()) # get the bb server running with the build req id and build env id - bbctrl = bec.getBBController("%d:%d" % (br.pk, bec.be.pk)) + bbctrl = bec.getBBController() # trigger the build command task = reduce(lambda x, y: x if len(y)== 0 else y, map(lambda y: y.task, br.brtarget_set.all())) diff --git a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py index 45e15392e5..be797c9486 100644 --- a/bitbake/lib/toaster/bldcontrol/sshbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/sshbecontroller.py @@ -29,7 +29,7 @@ import subprocess from toastermain import settings -from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _getgitcheckoutdirectoryname +from bbcontroller import BuildEnvironmentController, ShellCmdException, BuildSetupException, _get_git_clonedirectory def DN(path): return "/".join(path.split("/")[0:-1]) @@ -124,62 +124,8 @@ class SSHBEController(BuildEnvironmentController): assert len(bitbakes) == 1 # set layers in the layersource - # 1. get a list of repos, and map dirpaths for each layer - gitrepos = {} - gitrepos[bitbakes[0].giturl] = [] - gitrepos[bitbakes[0].giturl].append( ("bitbake", bitbakes[0].dirpath, bitbakes[0].commit) ) - - for layer in layers: - # we don't process local URLs - if layer.giturl.startswith("file://"): - continue - if not layer.giturl in gitrepos: - gitrepos[layer.giturl] = [] - gitrepos[layer.giturl].append( (layer.name, layer.dirpath, layer.commit)) - for giturl in gitrepos.keys(): - commitid = gitrepos[giturl][0][2] - for e in gitrepos[giturl]: - if commitid != e[2]: - raise BuildSetupException("More than one commit per git url, unsupported configuration") - - layerlist = [] - - # 2. checkout the repositories - for giturl in gitrepos.keys(): - import os - localdirname = os.path.join(self.be.sourcedir, _getgitcheckoutdirectoryname(giturl)) - print "DEBUG: giturl ", giturl ,"checking out in current directory", localdirname - - # make sure our directory is a git repository - if self._pathexists(localdirname): - if not giturl in self._shellcmd("git remote -v", localdirname): - raise BuildSetupException("Existing git repository at %s, but with different remotes (not '%s'). Aborting." % (localdirname, giturl)) - else: - self._shellcmd("git clone \"%s\" \"%s\"" % (giturl, localdirname)) - # checkout the needed commit - commit = gitrepos[giturl][0][2] - - # branch magic name "HEAD" will inhibit checkout - if commit != "HEAD": - print "DEBUG: checking out commit ", commit, "to", localdirname - self._shellcmd("git fetch --all && git checkout \"%s\"" % commit , localdirname) - - # take the localdirname as poky dir if we can find the oe-init-build-env - if self.pokydirname is None and self._pathexists(os.path.join(localdirname, "oe-init-build-env")): - print "DEBUG: selected poky dir name", localdirname - self.pokydirname = localdirname - - # verify our repositories - for name, dirpath, commit in gitrepos[giturl]: - localdirpath = os.path.join(localdirname, dirpath) - if not self._pathexists(localdirpath): - raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) - - if name != "bitbake": - layerlist.append(localdirpath) - - print "DEBUG: current layer list ", layerlist + raise Exception("Not implemented: SSH setLayers") # 3. configure the build environment, so we have a conf/bblayers.conf assert self.pokydirname is not None self._setupBE() -- cgit v1.2.3-54-g00ecf