From 08fbdc02e32d715ae94e3b9603fb3ec8351c8fd3 Mon Sep 17 00:00:00 2001 From: David Reyna Date: Wed, 15 Aug 2018 18:04:09 -0700 Subject: bitbake: Toaster: Implement the project-specific feature and releated enhancements and defects. Here is the primary driving enhancement: * Bug 12785 - Support Project Specific configuration for external tools (e.g. ISS, Eclipse) - Isolated project-specific configuration page (full Toaster context hidden) - Support for new project, reconfigure existing project, and import existing command line project - Ability to define variables (e.g. image recipe) and pass them back to external GUI - Ability to execute the cloning phase, so that external GUI receive a buildable project - Ability to call back to the external GUI when updates are completed and ready - Compatibility of above projects with the normal full Toaster interface - Ability to pass to a 'complete' or 'cancel' web page so that the external GUI can immediately stop that Toaster instance, and not leave dangling servers nor edit sessions open Here are the supporting enhancements, where at least the back end is implemented: * Bug 12821 - Make Toaster conf changes compatible with command line usage * Bug 12822 - Support importing user changes to conf files into Toaster * Bug 12823 - Support importing user build directories into Toaster * Bug 12824 - Scan imported layers for content so that they are immediately available * Bug 12825 - show layer clone item in progress bar Here are defects fixed: * Bug 12817 - builddelete.py requires explicit 'add_arguments' * Bug 12818 - Remove orphaned imported layers when project is deleted * Bug 12826 - fix imported layer management * Bug 12819 - build using selected bitbake env, not Toaster's env * Bug 12820 - Toaster randomizes the layer order in toaster_bblayers.conf [YOCTO #12785] (Bitbake rev: 985d6cec290bdd80998a63483561a73c75d82d65) Signed-off-by: David Reyna Signed-off-by: Richard Purdie --- .../toaster/bldcontrol/localhostbecontroller.py | 172 +++++++++++++++++---- 1 file changed, 139 insertions(+), 33 deletions(-) (limited to 'bitbake/lib/toaster/bldcontrol/localhostbecontroller.py') diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 16c7c80441..6bdd743b8b 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -27,8 +27,9 @@ import shutil import time from django.db import transaction from django.db.models import Q -from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake -from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting +from bldcontrol.models import BuildEnvironment, BuildRequest, BRLayer, BRVariable, BRTarget, BRBitbake, Build +from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ProjectLayer, ToasterSetting +from orm.models import signal_runbuilds import subprocess from toastermain import settings @@ -38,6 +39,8 @@ from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdExceptio import logging logger = logging.getLogger("toaster") +install_dir = os.environ.get('TOASTER_DIR') + from pprint import pprint, pformat class LocalhostBEController(BuildEnvironmentController): @@ -87,10 +90,10 @@ class LocalhostBEController(BuildEnvironmentController): #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) return local_checkout_path - - def setCloneStatus(self,bitbake,status,total,current): + def setCloneStatus(self,bitbake,status,total,current,repo_name): bitbake.req.build.repos_cloned=current bitbake.req.build.repos_to_clone=total + bitbake.req.build.progress_item=repo_name bitbake.req.build.save() def setLayers(self, bitbake, layers, targets): @@ -100,6 +103,7 @@ class LocalhostBEController(BuildEnvironmentController): layerlist = [] nongitlayerlist = [] + layer_index = 0 git_env = os.environ.copy() # (note: add custom environment settings here) @@ -113,7 +117,7 @@ class LocalhostBEController(BuildEnvironmentController): if bitbake.giturl and bitbake.commit: gitrepos[(bitbake.giturl, bitbake.commit)] = [] gitrepos[(bitbake.giturl, bitbake.commit)].append( - ("bitbake", bitbake.dirpath)) + ("bitbake", bitbake.dirpath, 0)) for layer in layers: # We don't need to git clone the layer for the CustomImageRecipe @@ -124,12 +128,13 @@ class LocalhostBEController(BuildEnvironmentController): # If we have local layers then we don't need clone them # For local layers giturl will be empty if not layer.giturl: - nongitlayerlist.append(layer.layer_version.layer.local_source_dir) + nongitlayerlist.append( "%03d:%s" % (layer_index,layer.local_source_dir) ) continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] - gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) + gitrepos[(layer.giturl, layer.commit)].append( (layer.name,layer.dirpath,layer_index) ) + layer_index += 1 logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) @@ -159,9 +164,9 @@ class LocalhostBEController(BuildEnvironmentController): # 3. checkout the repositories clone_count=0 clone_total=len(gitrepos.keys()) - self.setCloneStatus(bitbake,'Started',clone_total,clone_count) + self.setCloneStatus(bitbake,'Started',clone_total,clone_count,'') for giturl, commit in gitrepos.keys(): - self.setCloneStatus(bitbake,'progress',clone_total,clone_count) + self.setCloneStatus(bitbake,'progress',clone_total,clone_count,gitrepos[(giturl, commit)][0][0]) clone_count += 1 localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) @@ -205,16 +210,16 @@ class LocalhostBEController(BuildEnvironmentController): self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env) # verify our repositories - for name, dirpath in gitrepos[(giturl, commit)]: + for name, dirpath, index in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) - logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) + logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": - layerlist.append(localdirpath.rstrip("/")) + layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/"))) - self.setCloneStatus(bitbake,'complete',clone_total,clone_count) + self.setCloneStatus(bitbake,'complete',clone_total,clone_count,'') logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")): @@ -232,7 +237,7 @@ class LocalhostBEController(BuildEnvironmentController): customrecipe, layers) if os.path.isdir(custom_layer_path): - layerlist.append(custom_layer_path) + layerlist.append("%03d:%s" % (layer_index,custom_layer_path)) except CustomImageRecipe.DoesNotExist: continue # not a custom recipe, skip @@ -240,7 +245,11 @@ class LocalhostBEController(BuildEnvironmentController): layerlist.extend(nongitlayerlist) logger.debug("\n\nset layers gives this list %s" % pformat(layerlist)) self.islayerset = True - return layerlist + + # restore the order of layer list for bblayers.conf + layerlist.sort() + sorted_layerlist = [l[4:] for l in layerlist] + return sorted_layerlist def setup_custom_image_recipe(self, customrecipe, layers): """ Set up toaster-custom-images layer and recipe files """ @@ -310,31 +319,115 @@ class LocalhostBEController(BuildEnvironmentController): def triggerBuild(self, bitbake, layers, variables, targets, brbe): layers = self.setLayers(bitbake, layers, targets) + is_merged_attr = bitbake.req.project.merged_attr + + git_env = os.environ.copy() + # (note: add custom environment settings here) + try: + # insure that the project init/build uses the selected bitbake, and not Toaster's + del git_env['TEMPLATECONF'] + del git_env['BBBASEDIR'] + del git_env['BUILDDIR'] + except KeyError: + pass # init build environment from the clone - builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) + if bitbake.req.project.builddir: + builddir = bitbake.req.project.builddir + else: + builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') # init build environment try: custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value custom_script = custom_script.replace("%BUILDDIR%" ,builddir) - self._shellcmd("bash -c 'source %s'" % (custom_script)) + self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env) except ToasterSetting.DoesNotExist: self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), - self.be.sourcedir) + self.be.sourcedir,env=git_env) # update bblayers.conf - bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") - with open(bblconfpath, 'w') as bblayers: - bblayers.write('# line added by toaster build control\n' - 'BBLAYERS = "%s"' % ' '.join(layers)) - - # write configuration file - confpath = os.path.join(builddir, 'conf/toaster.conf') - with open(confpath, 'w') as conf: - for var in variables: - conf.write('%s="%s"\n' % (var.name, var.value)) - conf.write('INHERIT+="toaster buildhistory"') + if not is_merged_attr: + bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") + with open(bblconfpath, 'w') as bblayers: + bblayers.write('# line added by toaster build control\n' + 'BBLAYERS = "%s"' % ' '.join(layers)) + + # write configuration file + confpath = os.path.join(builddir, 'conf/toaster.conf') + with open(confpath, 'w') as conf: + for var in variables: + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('INHERIT+="toaster buildhistory"') + else: + # Append the Toaster-specific values directly to the bblayers.conf + bblconfpath = os.path.join(bitbake.req.project.builddir, "conf/bblayers.conf") + bblconfpath_save = os.path.join(bitbake.req.project.builddir, "conf/bblayers.conf.save") + shutil.copyfile(bblconfpath, bblconfpath_save) + with open(bblconfpath) as bblayers: + content = bblayers.readlines() + do_write = True + was_toaster = False + with open(bblconfpath,'w') as bblayers: + for line in content: + #line = line.strip('\n') + if 'TOASTER_CONFIG_PROLOG' in line: + do_write = False + was_toaster = True + elif 'TOASTER_CONFIG_EPILOG' in line: + do_write = True + elif do_write: + bblayers.write(line) + if not was_toaster: + bblayers.write('\n') + bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n') + bblayers.write('BBLAYERS = "\\\n') + for layer in layers: + bblayers.write(' %s \\\n' % layer) + bblayers.write(' "\n') + bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n') + # Append the Toaster-specific values directly to the local.conf + bbconfpath = os.path.join(bitbake.req.project.builddir, "conf/local.conf") + bbconfpath_save = os.path.join(bitbake.req.project.builddir, "conf/local.conf.save") + shutil.copyfile(bbconfpath, bbconfpath_save) + with open(bbconfpath) as f: + content = f.readlines() + do_write = True + was_toaster = False + with open(bbconfpath,'w') as conf: + for line in content: + #line = line.strip('\n') + if 'TOASTER_CONFIG_PROLOG' in line: + do_write = False + was_toaster = True + elif 'TOASTER_CONFIG_EPILOG' in line: + do_write = True + elif do_write: + conf.write(line) + if not was_toaster: + conf.write('\n') + conf.write('#=== TOASTER_CONFIG_PROLOG ===\n') + for var in variables: + if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"): + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('#=== TOASTER_CONFIG_EPILOG ===\n') + + # If 'target' is just the project preparation target, then we are done + for target in targets: + if "_PROJECT_PREPARE_" == target.target: + logger.debug('localhostbecontroller: Project has been prepared. Done.') + # Update the Build Request and release the build environment + bitbake.req.state = BuildRequest.REQ_COMPLETED + bitbake.req.save() + self.be.lock = BuildEnvironment.LOCK_FREE + self.be.save() + # Close the project build and progress bar + bitbake.req.build.outcome = Build.SUCCEEDED + bitbake.req.build.save() + # Update the project status + bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS) + signal_runbuilds() + return # clean the Toaster to build environment env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0 @@ -342,9 +435,14 @@ class LocalhostBEController(BuildEnvironmentController): # run bitbake server from the clone bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake') toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf") - self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' - '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, - builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) + if not is_merged_attr: + self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' + '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, + builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) + else: + self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s ' + '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, + builddir, bitbake), self.be.sourcedir) # read port number from bitbake.lock self.be.bbport = -1 @@ -390,12 +488,20 @@ class LocalhostBEController(BuildEnvironmentController): log = os.path.join(builddir, 'toaster_ui.log') local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), 'bitbake') - self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' + if not is_merged_attr: + self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;' 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log, self.be.bbport, bitbake,)], builddir, nowait=True) + else: + self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' + '%s %s -u toasterui --token="" >>%s 2>&1;' + 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ + % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log, + self.be.bbport, bitbake,)], + builddir, nowait=True) logger.debug('localhostbecontroller: Build launched, exiting. ' 'Follow build logs at %s' % log) -- cgit v1.2.3-54-g00ecf