bitbake: Toaster: Implement the project-specific feature and releated enhancements and defects.

Here is the primary driving enhancement:

*  Bug 12785 - Support Project Specific configuration for external
   tools (e.g. ISS, Eclipse)

  -  Isolated project-specific configuration page (full Toaster context
     hidden)
  -  Support for new project, reconfigure existing project, and import
     existing command line project
  -  Ability to define variables (e.g. image recipe) and pass them back
     to external GUI
  -  Ability to execute the cloning phase, so that external GUI receive
     a buildable project
  -  Ability to call back to the external GUI when updates are completed
     and ready
  -  Compatibility of above projects with the normal full Toaster interface
  -  Ability to pass to a 'complete' or 'cancel' web page so that the
     external GUI can immediately stop that Toaster instance, and not
     leave dangling servers nor edit sessions open

Here are the supporting enhancements, where at least the
back end is implemented:

*  Bug 12821 - Make Toaster conf changes compatible with command line usage
*  Bug 12822 - Support importing user changes to conf files into Toaster
*  Bug 12823 - Support importing user build directories into Toaster
*  Bug 12824 - Scan imported layers for content so that they are
               immediately available
*  Bug 12825 - show layer clone item in progress bar

Here are defects fixed:

*  Bug 12817 - builddelete.py requires explicit 'add_arguments'
*  Bug 12818 - Remove orphaned imported layers when project is deleted
*  Bug 12826 - fix imported layer management
*  Bug 12819 - build using selected bitbake env, not Toaster's env
*  Bug 12820 - Toaster randomizes the layer order in toaster_bblayers.conf

[YOCTO #12785]

(Bitbake rev: 985d6cec290bdd80998a63483561a73c75d82d65)

Signed-off-by: David Reyna <David.Reyna@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 139 insertions, 33 deletions

diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index 16c7c80441..6bdd743b8b 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -27,8 +27,9 @@ import shutil
 import time
 from django.db import transaction
 from django.db.models import Q
-from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake
-from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting
+from bldcontrol.models import BuildEnvironment, BuildRequest, BRLayer, BRVariable, BRTarget, BRBitbake, Build
+from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ProjectLayer, ToasterSetting
+from orm.models import signal_runbuilds
 import subprocess
 
 from toastermain import settings
@@ -38,6 +39,8 @@ from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdExceptio
 import logging
 logger = logging.getLogger("toaster")
 
+install_dir = os.environ.get('TOASTER_DIR')
+
 from pprint import pprint, pformat
 
 class LocalhostBEController(BuildEnvironmentController):
@@ -87,10 +90,10 @@ class LocalhostBEController(BuildEnvironmentController):
         #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path)
         return local_checkout_path
 
-
-    def setCloneStatus(self,bitbake,status,total,current):
+    def setCloneStatus(self,bitbake,status,total,current,repo_name):
         bitbake.req.build.repos_cloned=current
         bitbake.req.build.repos_to_clone=total
+        bitbake.req.build.progress_item=repo_name
         bitbake.req.build.save()
 
     def setLayers(self, bitbake, layers, targets):
@@ -100,6 +103,7 @@ class LocalhostBEController(BuildEnvironmentController):
 
         layerlist = []
         nongitlayerlist = []
+        layer_index = 0
         git_env = os.environ.copy()
         # (note: add custom environment settings here)
 
@@ -113,7 +117,7 @@ class LocalhostBEController(BuildEnvironmentController):
         if bitbake.giturl and bitbake.commit:
             gitrepos[(bitbake.giturl, bitbake.commit)] = []
             gitrepos[(bitbake.giturl, bitbake.commit)].append(
-                ("bitbake", bitbake.dirpath))
+                ("bitbake", bitbake.dirpath, 0))
 
         for layer in layers:
             # We don't need to git clone the layer for the CustomImageRecipe
@@ -124,12 +128,13 @@ class LocalhostBEController(BuildEnvironmentController):
             # If we have local layers then we don't need clone them
             # For local layers giturl will be empty
             if not layer.giturl:
-                nongitlayerlist.append(layer.layer_version.layer.local_source_dir)
+                nongitlayerlist.append( "%03d:%s" % (layer_index,layer.local_source_dir) )
                 continue
 
             if not (layer.giturl, layer.commit) in gitrepos:
                 gitrepos[(layer.giturl, layer.commit)] = []
-            gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) )
+            gitrepos[(layer.giturl, layer.commit)].append( (layer.name,layer.dirpath,layer_index) )
+            layer_index += 1
 
 
         logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos))
@@ -159,9 +164,9 @@ class LocalhostBEController(BuildEnvironmentController):
         # 3. checkout the repositories
         clone_count=0
         clone_total=len(gitrepos.keys())
-        self.setCloneStatus(bitbake,'Started',clone_total,clone_count)
+        self.setCloneStatus(bitbake,'Started',clone_total,clone_count,'')
         for giturl, commit in gitrepos.keys():
-            self.setCloneStatus(bitbake,'progress',clone_total,clone_count)
+            self.setCloneStatus(bitbake,'progress',clone_total,clone_count,gitrepos[(giturl, commit)][0][0])
             clone_count += 1
 
             localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
@@ -205,16 +210,16 @@ class LocalhostBEController(BuildEnvironmentController):
                     self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env)
 
             # verify our repositories
-            for name, dirpath in gitrepos[(giturl, commit)]:
+            for name, dirpath, index in gitrepos[(giturl, commit)]:
                 localdirpath = os.path.join(localdirname, dirpath)
-                logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath)
+                logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath)
                 if not os.path.exists(localdirpath):
                     raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
 
                 if name != "bitbake":
-                    layerlist.append(localdirpath.rstrip("/"))
+                    layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/")))
 
-        self.setCloneStatus(bitbake,'complete',clone_total,clone_count)
+        self.setCloneStatus(bitbake,'complete',clone_total,clone_count,'')
         logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist))
 
         if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")):
@@ -232,7 +237,7 @@ class LocalhostBEController(BuildEnvironmentController):
                     customrecipe, layers)
 
                 if os.path.isdir(custom_layer_path):
-                    layerlist.append(custom_layer_path)
+                    layerlist.append("%03d:%s" % (layer_index,custom_layer_path))
 
             except CustomImageRecipe.DoesNotExist:
                 continue  # not a custom recipe, skip
@@ -240,7 +245,11 @@ class LocalhostBEController(BuildEnvironmentController):
         layerlist.extend(nongitlayerlist)
         logger.debug("\n\nset layers gives this list %s" % pformat(layerlist))
         self.islayerset = True
-        return layerlist
+
+        # restore the order of layer list for bblayers.conf
+        layerlist.sort()
+        sorted_layerlist = [l[4:] for l in layerlist]
+        return sorted_layerlist
 
     def setup_custom_image_recipe(self, customrecipe, layers):
         """ Set up toaster-custom-images layer and recipe files """
@@ -310,31 +319,115 @@ class LocalhostBEController(BuildEnvironmentController):
 
     def triggerBuild(self, bitbake, layers, variables, targets, brbe):
         layers = self.setLayers(bitbake, layers, targets)
+        is_merged_attr = bitbake.req.project.merged_attr
+
+        git_env = os.environ.copy()
+        # (note: add custom environment settings here)
+        try:
+            # insure that the project init/build uses the selected bitbake, and not Toaster's
+            del git_env['TEMPLATECONF']
+            del git_env['BBBASEDIR']
+            del git_env['BUILDDIR']
+        except KeyError:
+            pass
 
         # init build environment from the clone
-        builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
+        if bitbake.req.project.builddir:
+            builddir = bitbake.req.project.builddir
+        else:
+            builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
         oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
         # init build environment
         try:
             custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
             custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
-            self._shellcmd("bash -c 'source %s'" % (custom_script))
+            self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env)
         except ToasterSetting.DoesNotExist:
             self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
-                       self.be.sourcedir)
+                       self.be.sourcedir,env=git_env)
 
         # update bblayers.conf
-        bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
-        with open(bblconfpath, 'w') as bblayers:
-            bblayers.write('# line added by toaster build control\n'
-                           'BBLAYERS = "%s"' % ' '.join(layers))
-
-        # write configuration file
-        confpath = os.path.join(builddir, 'conf/toaster.conf')
-        with open(confpath, 'w') as conf:
-            for var in variables:
-                conf.write('%s="%s"\n' % (var.name, var.value))
-            conf.write('INHERIT+="toaster buildhistory"')
+        if not is_merged_attr:
+            bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
+            with open(bblconfpath, 'w') as bblayers:
+                bblayers.write('# line added by toaster build control\n'
+                               'BBLAYERS = "%s"' % ' '.join(layers))
+
+            # write configuration file
+            confpath = os.path.join(builddir, 'conf/toaster.conf')
+            with open(confpath, 'w') as conf:
+                for var in variables:
+                    conf.write('%s="%s"\n' % (var.name, var.value))
+                conf.write('INHERIT+="toaster buildhistory"')
+        else:
+            # Append the Toaster-specific values directly to the bblayers.conf
+            bblconfpath = os.path.join(bitbake.req.project.builddir, "conf/bblayers.conf")
+            bblconfpath_save = os.path.join(bitbake.req.project.builddir, "conf/bblayers.conf.save")
+            shutil.copyfile(bblconfpath, bblconfpath_save)
+            with open(bblconfpath) as bblayers:
+                content = bblayers.readlines()
+            do_write = True
+            was_toaster = False
+            with open(bblconfpath,'w') as bblayers:
+                for line in content:
+                    #line = line.strip('\n')
+                    if 'TOASTER_CONFIG_PROLOG' in line:
+                        do_write = False
+                        was_toaster = True
+                    elif 'TOASTER_CONFIG_EPILOG' in line:
+                        do_write = True
+                    elif do_write:
+                        bblayers.write(line)
+                if not was_toaster:
+                    bblayers.write('\n')
+                bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n')
+                bblayers.write('BBLAYERS = "\\\n')
+                for layer in layers:
+                    bblayers.write('  %s \\\n' % layer)
+                bblayers.write('  "\n')
+                bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n')
+            # Append the Toaster-specific values directly to the local.conf
+            bbconfpath = os.path.join(bitbake.req.project.builddir, "conf/local.conf")
+            bbconfpath_save = os.path.join(bitbake.req.project.builddir, "conf/local.conf.save")
+            shutil.copyfile(bbconfpath, bbconfpath_save)
+            with open(bbconfpath) as f:
+                content = f.readlines()
+            do_write = True
+            was_toaster = False
+            with open(bbconfpath,'w') as conf:
+                for line in content:
+                    #line = line.strip('\n')
+                    if 'TOASTER_CONFIG_PROLOG' in line:
+                        do_write = False
+                        was_toaster = True
+                    elif 'TOASTER_CONFIG_EPILOG' in line:
+                        do_write = True
+                    elif do_write:
+                        conf.write(line)
+                if not was_toaster:
+                    conf.write('\n')
+                conf.write('#=== TOASTER_CONFIG_PROLOG ===\n')
+                for var in variables:
+                    if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"):
+                        conf.write('%s="%s"\n' % (var.name, var.value))
+                conf.write('#=== TOASTER_CONFIG_EPILOG ===\n')
+
+        # If 'target' is just the project preparation target, then we are done
+        for target in targets:
+            if "_PROJECT_PREPARE_" == target.target:
+                logger.debug('localhostbecontroller: Project has been prepared. Done.')
+                # Update the Build Request and release the build environment
+                bitbake.req.state = BuildRequest.REQ_COMPLETED
+                bitbake.req.save()
+                self.be.lock = BuildEnvironment.LOCK_FREE
+                self.be.save()
+                # Close the project build and progress bar
+                bitbake.req.build.outcome = Build.SUCCEEDED
+                bitbake.req.build.save()
+                # Update the project status
+                bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS)
+                signal_runbuilds()
+                return
 
         # clean the Toaster to build environment
         env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0
@@ -342,9 +435,14 @@ class LocalhostBEController(BuildEnvironmentController):
         # run bitbake server from the clone
         bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
         toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
-        self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
-                       '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
-                       builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
+        if not is_merged_attr:
+            self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
+                           '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
+                           builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
+        else:
+            self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s '
+                           '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
+                           builddir, bitbake), self.be.sourcedir)
 
         # read port number from bitbake.lock
         self.be.bbport = -1
@@ -390,12 +488,20 @@ class LocalhostBEController(BuildEnvironmentController):
         log = os.path.join(builddir, 'toaster_ui.log')
         local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                      'bitbake')
-        self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
+        if not is_merged_attr:
+            self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                         '%s %s -u toasterui  --read %s --read %s --token="" >>%s 2>&1;'
                         'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
                         % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
                         self.be.bbport, bitbake,)],
                         builddir, nowait=True)
+        else:
+            self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
+                        '%s %s -u toasterui  --token="" >>%s 2>&1;'
+                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
+                        % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log,
+                        self.be.bbport, bitbake,)],
+                        builddir, nowait=True)
 
         logger.debug('localhostbecontroller: Build launched, exiting. '
                      'Follow build logs at %s' % log)