From 08fbdc02e32d715ae94e3b9603fb3ec8351c8fd3 Mon Sep 17 00:00:00 2001 From: David Reyna Date: Wed, 15 Aug 2018 18:04:09 -0700 Subject: bitbake: Toaster: Implement the project-specific feature and releated enhancements and defects. Here is the primary driving enhancement: * Bug 12785 - Support Project Specific configuration for external tools (e.g. ISS, Eclipse) - Isolated project-specific configuration page (full Toaster context hidden) - Support for new project, reconfigure existing project, and import existing command line project - Ability to define variables (e.g. image recipe) and pass them back to external GUI - Ability to execute the cloning phase, so that external GUI receive a buildable project - Ability to call back to the external GUI when updates are completed and ready - Compatibility of above projects with the normal full Toaster interface - Ability to pass to a 'complete' or 'cancel' web page so that the external GUI can immediately stop that Toaster instance, and not leave dangling servers nor edit sessions open Here are the supporting enhancements, where at least the back end is implemented: * Bug 12821 - Make Toaster conf changes compatible with command line usage * Bug 12822 - Support importing user changes to conf files into Toaster * Bug 12823 - Support importing user build directories into Toaster * Bug 12824 - Scan imported layers for content so that they are immediately available * Bug 12825 - show layer clone item in progress bar Here are defects fixed: * Bug 12817 - builddelete.py requires explicit 'add_arguments' * Bug 12818 - Remove orphaned imported layers when project is deleted * Bug 12826 - fix imported layer management * Bug 12819 - build using selected bitbake env, not Toaster's env * Bug 12820 - Toaster randomizes the layer order in toaster_bblayers.conf [YOCTO #12785] (Bitbake rev: 985d6cec290bdd80998a63483561a73c75d82d65) Signed-off-by: David Reyna Signed-off-by: Richard Purdie --- .../toaster/bldcontrol/localhostbecontroller.py | 172 ++++-- .../bldcontrol/management/commands/runbuilds.py | 2 +- .../orm/migrations/0018_project_specific.py | 28 + bitbake/lib/toaster/orm/models.py | 61 ++- bitbake/lib/toaster/toastergui/api.py | 168 +++++- .../lib/toaster/toastergui/static/js/layerBtn.js | 12 + .../lib/toaster/toastergui/static/js/libtoaster.js | 105 ++++ .../lib/toaster/toastergui/static/js/mrbsection.js | 4 +- .../toaster/toastergui/static/js/projecttopbar.js | 22 + bitbake/lib/toaster/toastergui/tables.py | 4 + .../toastergui/templates/base_specific.html | 128 +++++ .../templates/baseprojectspecificpage.html | 48 ++ .../templates/generic-toastertable-page.html | 2 +- .../toaster/toastergui/templates/importlayer.html | 4 +- .../toastergui/templates/landing_specific.html | 50 ++ .../toaster/toastergui/templates/layerdetails.html | 3 +- .../toaster/toastergui/templates/mrb_section.html | 2 +- .../toastergui/templates/newcustomimage.html | 4 +- .../toastergui/templates/newproject_specific.html | 95 ++++ .../lib/toaster/toastergui/templates/project.html | 7 +- .../toastergui/templates/project_specific.html | 162 ++++++ .../templates/project_specific_topbar.html | 80 +++ .../toaster/toastergui/templates/projectconf.html | 7 +- .../toastergui/templates/recipe_add_btn.html | 23 + bitbake/lib/toaster/toastergui/urls.py | 13 + bitbake/lib/toaster/toastergui/views.py | 151 ++++++ bitbake/lib/toaster/toastergui/widgets.py | 6 + .../toastermain/management/commands/builddelete.py | 6 +- .../toastermain/management/commands/buildimport.py | 586 +++++++++++++++++++++ 29 files changed, 1900 insertions(+), 55 deletions(-) create mode 100644 bitbake/lib/toaster/orm/migrations/0018_project_specific.py create mode 100644 bitbake/lib/toaster/toastergui/templates/base_specific.html create mode 100644 bitbake/lib/toaster/toastergui/templates/baseprojectspecificpage.html create mode 100644 bitbake/lib/toaster/toastergui/templates/landing_specific.html create mode 100644 bitbake/lib/toaster/toastergui/templates/newproject_specific.html create mode 100644 bitbake/lib/toaster/toastergui/templates/project_specific.html create mode 100644 bitbake/lib/toaster/toastergui/templates/project_specific_topbar.html create mode 100644 bitbake/lib/toaster/toastergui/templates/recipe_add_btn.html mode change 100755 => 100644 bitbake/lib/toaster/toastergui/views.py create mode 100644 bitbake/lib/toaster/toastermain/management/commands/buildimport.py (limited to 'bitbake/lib/toaster') diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py index 16c7c80441..6bdd743b8b 100644 --- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py +++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py @@ -27,8 +27,9 @@ import shutil import time from django.db import transaction from django.db.models import Q -from bldcontrol.models import BuildEnvironment, BRLayer, BRVariable, BRTarget, BRBitbake -from orm.models import CustomImageRecipe, Layer, Layer_Version, ProjectLayer, ToasterSetting +from bldcontrol.models import BuildEnvironment, BuildRequest, BRLayer, BRVariable, BRTarget, BRBitbake, Build +from orm.models import CustomImageRecipe, Layer, Layer_Version, Project, ProjectLayer, ToasterSetting +from orm.models import signal_runbuilds import subprocess from toastermain import settings @@ -38,6 +39,8 @@ from bldcontrol.bbcontroller import BuildEnvironmentController, ShellCmdExceptio import logging logger = logging.getLogger("toaster") +install_dir = os.environ.get('TOASTER_DIR') + from pprint import pprint, pformat class LocalhostBEController(BuildEnvironmentController): @@ -87,10 +90,10 @@ class LocalhostBEController(BuildEnvironmentController): #logger.debug("localhostbecontroller: using HEAD checkout in %s" % local_checkout_path) return local_checkout_path - - def setCloneStatus(self,bitbake,status,total,current): + def setCloneStatus(self,bitbake,status,total,current,repo_name): bitbake.req.build.repos_cloned=current bitbake.req.build.repos_to_clone=total + bitbake.req.build.progress_item=repo_name bitbake.req.build.save() def setLayers(self, bitbake, layers, targets): @@ -100,6 +103,7 @@ class LocalhostBEController(BuildEnvironmentController): layerlist = [] nongitlayerlist = [] + layer_index = 0 git_env = os.environ.copy() # (note: add custom environment settings here) @@ -113,7 +117,7 @@ class LocalhostBEController(BuildEnvironmentController): if bitbake.giturl and bitbake.commit: gitrepos[(bitbake.giturl, bitbake.commit)] = [] gitrepos[(bitbake.giturl, bitbake.commit)].append( - ("bitbake", bitbake.dirpath)) + ("bitbake", bitbake.dirpath, 0)) for layer in layers: # We don't need to git clone the layer for the CustomImageRecipe @@ -124,12 +128,13 @@ class LocalhostBEController(BuildEnvironmentController): # If we have local layers then we don't need clone them # For local layers giturl will be empty if not layer.giturl: - nongitlayerlist.append(layer.layer_version.layer.local_source_dir) + nongitlayerlist.append( "%03d:%s" % (layer_index,layer.local_source_dir) ) continue if not (layer.giturl, layer.commit) in gitrepos: gitrepos[(layer.giturl, layer.commit)] = [] - gitrepos[(layer.giturl, layer.commit)].append( (layer.name, layer.dirpath) ) + gitrepos[(layer.giturl, layer.commit)].append( (layer.name,layer.dirpath,layer_index) ) + layer_index += 1 logger.debug("localhostbecontroller, our git repos are %s" % pformat(gitrepos)) @@ -159,9 +164,9 @@ class LocalhostBEController(BuildEnvironmentController): # 3. checkout the repositories clone_count=0 clone_total=len(gitrepos.keys()) - self.setCloneStatus(bitbake,'Started',clone_total,clone_count) + self.setCloneStatus(bitbake,'Started',clone_total,clone_count,'') for giturl, commit in gitrepos.keys(): - self.setCloneStatus(bitbake,'progress',clone_total,clone_count) + self.setCloneStatus(bitbake,'progress',clone_total,clone_count,gitrepos[(giturl, commit)][0][0]) clone_count += 1 localdirname = os.path.join(self.be.sourcedir, self.getGitCloneDirectory(giturl, commit)) @@ -205,16 +210,16 @@ class LocalhostBEController(BuildEnvironmentController): self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " % (bitbake.commit, bitbake.giturl, os.path.join(self.pokydirname, 'bitbake')),env=git_env) # verify our repositories - for name, dirpath in gitrepos[(giturl, commit)]: + for name, dirpath, index in gitrepos[(giturl, commit)]: localdirpath = os.path.join(localdirname, dirpath) - logger.debug("localhostbecontroller: localdirpath expected '%s'" % localdirpath) + logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath) if not os.path.exists(localdirpath): raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit)) if name != "bitbake": - layerlist.append(localdirpath.rstrip("/")) + layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/"))) - self.setCloneStatus(bitbake,'complete',clone_total,clone_count) + self.setCloneStatus(bitbake,'complete',clone_total,clone_count,'') logger.debug("localhostbecontroller: current layer list %s " % pformat(layerlist)) if self.pokydirname is None and os.path.exists(os.path.join(self.be.sourcedir, "oe-init-build-env")): @@ -232,7 +237,7 @@ class LocalhostBEController(BuildEnvironmentController): customrecipe, layers) if os.path.isdir(custom_layer_path): - layerlist.append(custom_layer_path) + layerlist.append("%03d:%s" % (layer_index,custom_layer_path)) except CustomImageRecipe.DoesNotExist: continue # not a custom recipe, skip @@ -240,7 +245,11 @@ class LocalhostBEController(BuildEnvironmentController): layerlist.extend(nongitlayerlist) logger.debug("\n\nset layers gives this list %s" % pformat(layerlist)) self.islayerset = True - return layerlist + + # restore the order of layer list for bblayers.conf + layerlist.sort() + sorted_layerlist = [l[4:] for l in layerlist] + return sorted_layerlist def setup_custom_image_recipe(self, customrecipe, layers): """ Set up toaster-custom-images layer and recipe files """ @@ -310,31 +319,115 @@ class LocalhostBEController(BuildEnvironmentController): def triggerBuild(self, bitbake, layers, variables, targets, brbe): layers = self.setLayers(bitbake, layers, targets) + is_merged_attr = bitbake.req.project.merged_attr + + git_env = os.environ.copy() + # (note: add custom environment settings here) + try: + # insure that the project init/build uses the selected bitbake, and not Toaster's + del git_env['TEMPLATECONF'] + del git_env['BBBASEDIR'] + del git_env['BUILDDIR'] + except KeyError: + pass # init build environment from the clone - builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) + if bitbake.req.project.builddir: + builddir = bitbake.req.project.builddir + else: + builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id) oe_init = os.path.join(self.pokydirname, 'oe-init-build-env') # init build environment try: custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value custom_script = custom_script.replace("%BUILDDIR%" ,builddir) - self._shellcmd("bash -c 'source %s'" % (custom_script)) + self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env) except ToasterSetting.DoesNotExist: self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir), - self.be.sourcedir) + self.be.sourcedir,env=git_env) # update bblayers.conf - bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") - with open(bblconfpath, 'w') as bblayers: - bblayers.write('# line added by toaster build control\n' - 'BBLAYERS = "%s"' % ' '.join(layers)) - - # write configuration file - confpath = os.path.join(builddir, 'conf/toaster.conf') - with open(confpath, 'w') as conf: - for var in variables: - conf.write('%s="%s"\n' % (var.name, var.value)) - conf.write('INHERIT+="toaster buildhistory"') + if not is_merged_attr: + bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf") + with open(bblconfpath, 'w') as bblayers: + bblayers.write('# line added by toaster build control\n' + 'BBLAYERS = "%s"' % ' '.join(layers)) + + # write configuration file + confpath = os.path.join(builddir, 'conf/toaster.conf') + with open(confpath, 'w') as conf: + for var in variables: + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('INHERIT+="toaster buildhistory"') + else: + # Append the Toaster-specific values directly to the bblayers.conf + bblconfpath = os.path.join(bitbake.req.project.builddir, "conf/bblayers.conf") + bblconfpath_save = os.path.join(bitbake.req.project.builddir, "conf/bblayers.conf.save") + shutil.copyfile(bblconfpath, bblconfpath_save) + with open(bblconfpath) as bblayers: + content = bblayers.readlines() + do_write = True + was_toaster = False + with open(bblconfpath,'w') as bblayers: + for line in content: + #line = line.strip('\n') + if 'TOASTER_CONFIG_PROLOG' in line: + do_write = False + was_toaster = True + elif 'TOASTER_CONFIG_EPILOG' in line: + do_write = True + elif do_write: + bblayers.write(line) + if not was_toaster: + bblayers.write('\n') + bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n') + bblayers.write('BBLAYERS = "\\\n') + for layer in layers: + bblayers.write(' %s \\\n' % layer) + bblayers.write(' "\n') + bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n') + # Append the Toaster-specific values directly to the local.conf + bbconfpath = os.path.join(bitbake.req.project.builddir, "conf/local.conf") + bbconfpath_save = os.path.join(bitbake.req.project.builddir, "conf/local.conf.save") + shutil.copyfile(bbconfpath, bbconfpath_save) + with open(bbconfpath) as f: + content = f.readlines() + do_write = True + was_toaster = False + with open(bbconfpath,'w') as conf: + for line in content: + #line = line.strip('\n') + if 'TOASTER_CONFIG_PROLOG' in line: + do_write = False + was_toaster = True + elif 'TOASTER_CONFIG_EPILOG' in line: + do_write = True + elif do_write: + conf.write(line) + if not was_toaster: + conf.write('\n') + conf.write('#=== TOASTER_CONFIG_PROLOG ===\n') + for var in variables: + if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"): + conf.write('%s="%s"\n' % (var.name, var.value)) + conf.write('#=== TOASTER_CONFIG_EPILOG ===\n') + + # If 'target' is just the project preparation target, then we are done + for target in targets: + if "_PROJECT_PREPARE_" == target.target: + logger.debug('localhostbecontroller: Project has been prepared. Done.') + # Update the Build Request and release the build environment + bitbake.req.state = BuildRequest.REQ_COMPLETED + bitbake.req.save() + self.be.lock = BuildEnvironment.LOCK_FREE + self.be.save() + # Close the project build and progress bar + bitbake.req.build.outcome = Build.SUCCEEDED + bitbake.req.build.save() + # Update the project status + bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS) + signal_runbuilds() + return # clean the Toaster to build environment env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0 @@ -342,9 +435,14 @@ class LocalhostBEController(BuildEnvironmentController): # run bitbake server from the clone bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake') toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf") - self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' - '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, - builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) + if not is_merged_attr: + self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s ' + '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, + builddir, bitbake, confpath, toasterlayers), self.be.sourcedir) + else: + self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s ' + '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init, + builddir, bitbake), self.be.sourcedir) # read port number from bitbake.lock self.be.bbport = -1 @@ -390,12 +488,20 @@ class LocalhostBEController(BuildEnvironmentController): log = os.path.join(builddir, 'toaster_ui.log') local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')), 'bitbake') - self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' + if not is_merged_attr: + self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' '%s %s -u toasterui --read %s --read %s --token="" >>%s 2>&1;' 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log, self.be.bbport, bitbake,)], builddir, nowait=True) + else: + self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" ' + '%s %s -u toasterui --token="" >>%s 2>&1;' + 'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \ + % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log, + self.be.bbport, bitbake,)], + builddir, nowait=True) logger.debug('localhostbecontroller: Build launched, exiting. ' 'Follow build logs at %s' % log) diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py index 791e53eabf..6a55dd46c8 100644 --- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py +++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py @@ -49,7 +49,7 @@ class Command(BaseCommand): # we could not find a BEC; postpone the BR br.state = BuildRequest.REQ_QUEUED br.save() - logger.debug("runbuilds: No build env") + logger.debug("runbuilds: No build env (%s)" % e) return logger.info("runbuilds: starting build %s, environment %s" % diff --git a/bitbake/lib/toaster/orm/migrations/0018_project_specific.py b/bitbake/lib/toaster/orm/migrations/0018_project_specific.py new file mode 100644 index 0000000000..084ecad7ba --- /dev/null +++ b/bitbake/lib/toaster/orm/migrations/0018_project_specific.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models + +class Migration(migrations.Migration): + + dependencies = [ + ('orm', '0017_distro_clone'), + ] + + operations = [ + migrations.AddField( + model_name='Project', + name='builddir', + field=models.TextField(), + ), + migrations.AddField( + model_name='Project', + name='merged_attr', + field=models.BooleanField(default=False) + ), + migrations.AddField( + model_name='Build', + name='progress_item', + field=models.CharField(max_length=40) + ), + ] diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py index 3a7dff8ca6..306c4fafa8 100644 --- a/bitbake/lib/toaster/orm/models.py +++ b/bitbake/lib/toaster/orm/models.py @@ -121,8 +121,15 @@ class ToasterSetting(models.Model): class ProjectManager(models.Manager): - def create_project(self, name, release): - if release is not None: + def create_project(self, name, release, existing_project=None): + if existing_project and (release is not None): + prj = existing_project + prj.bitbake_version = release.bitbake_version + prj.release = release + # Delete the previous ProjectLayer mappings + for pl in ProjectLayer.objects.filter(project=prj): + pl.delete() + elif release is not None: prj = self.model(name=name, bitbake_version=release.bitbake_version, release=release) @@ -130,15 +137,14 @@ class ProjectManager(models.Manager): prj = self.model(name=name, bitbake_version=None, release=None) - prj.save() for defaultconf in ToasterSetting.objects.filter( name__startswith="DEFCONF_"): name = defaultconf.name[8:] - ProjectVariable.objects.create(project=prj, - name=name, - value=defaultconf.value) + pv,create = ProjectVariable.objects.get_or_create(project=prj,name=name) + pv.value = defaultconf.value + pv.save() if release is None: return prj @@ -197,6 +203,11 @@ class Project(models.Model): user_id = models.IntegerField(null=True) objects = ProjectManager() + # build directory override (e.g. imported) + builddir = models.TextField() + # merge the Toaster configure attributes directly into the standard conf files + merged_attr = models.BooleanField(default=False) + # set to True for the project which is the default container # for builds initiated by the command line etc. is_default= models.BooleanField(default=False) @@ -305,6 +316,15 @@ class Project(models.Model): return layer_versions + def get_default_image_recipe(self): + try: + return self.projectvariable_set.get(name="DEFAULT_IMAGE").value + except (ProjectVariable.DoesNotExist,IndexError): + return None; + + def get_is_new(self): + return self.get_variable(Project.PROJECT_SPECIFIC_ISNEW) + def get_available_machines(self): """ Returns QuerySet of all Machines which are provided by the Layers currently added to the Project """ @@ -353,6 +373,32 @@ class Project(models.Model): return queryset + # Project Specific status management + PROJECT_SPECIFIC_STATUS = 'INTERNAL_PROJECT_SPECIFIC_STATUS' + PROJECT_SPECIFIC_CALLBACK = 'INTERNAL_PROJECT_SPECIFIC_CALLBACK' + PROJECT_SPECIFIC_ISNEW = 'INTERNAL_PROJECT_SPECIFIC_ISNEW' + PROJECT_SPECIFIC_DEFAULTIMAGE = 'PROJECT_SPECIFIC_DEFAULTIMAGE' + PROJECT_SPECIFIC_NONE = '' + PROJECT_SPECIFIC_NEW = '1' + PROJECT_SPECIFIC_EDIT = '2' + PROJECT_SPECIFIC_CLONING = '3' + PROJECT_SPECIFIC_CLONING_SUCCESS = '4' + PROJECT_SPECIFIC_CLONING_FAIL = '5' + + def get_variable(self,variable,default_value = ''): + try: + return self.projectvariable_set.get(name=variable).value + except (ProjectVariable.DoesNotExist,IndexError): + return default_value + + def set_variable(self,variable,value): + pv,create = ProjectVariable.objects.get_or_create(project = self, name = variable) + pv.value = value + pv.save() + + def get_default_image(self): + return self.get_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE) + def schedule_build(self): from bldcontrol.models import BuildRequest, BRTarget, BRLayer @@ -459,6 +505,9 @@ class Build(models.Model): # number of repos cloned so far for this build (default off) repos_cloned = models.IntegerField(default=1) + # Hint on current progress item + progress_item = models.CharField(max_length=40) + @staticmethod def get_recent(project=None): """ diff --git a/bitbake/lib/toaster/toastergui/api.py b/bitbake/lib/toaster/toastergui/api.py index ab6ba69e0e..1bec56d468 100644 --- a/bitbake/lib/toaster/toastergui/api.py +++ b/bitbake/lib/toaster/toastergui/api.py @@ -22,7 +22,9 @@ import os import re import logging import json +import subprocess from collections import Counter +from shutil import copyfile from orm.models import Project, ProjectTarget, Build, Layer_Version from orm.models import LayerVersionDependency, LayerSource, ProjectLayer @@ -38,6 +40,18 @@ from django.core.urlresolvers import reverse from django.db.models import Q, F from django.db import Error from toastergui.templatetags.projecttags import filtered_filesizeformat +from django.utils import timezone +import pytz + +# development/debugging support +verbose = 2 +def _log(msg): + if 1 == verbose: + print(msg) + elif 2 == verbose: + f1=open('/tmp/toaster.log', 'a') + f1.write("|" + msg + "|\n" ) + f1.close() logger = logging.getLogger("toaster") @@ -137,6 +151,130 @@ class XhrBuildRequest(View): return response +class XhrProjectUpdate(View): + + def get(self, request, *args, **kwargs): + return HttpResponse() + + def post(self, request, *args, **kwargs): + """ + Project Update + + Entry point: /xhr_projectupdate/ + Method: POST + + Args: + pid: pid of project to update + + Returns: + {"error": "ok"} + or + {"error": } + """ + + project = Project.objects.get(pk=kwargs['pid']) + logger.debug("ProjectUpdateCallback:project.pk=%d,project.builddir=%s" % (project.pk,project.builddir)) + + if 'do_update' in request.POST: + + # Extract any default image recipe + if 'default_image' in request.POST: + project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,str(request.POST['default_image'])) + else: + project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,'') + + logger.debug("ProjectUpdateCallback:Chain to the build request") + + # Chain to the build request + xhrBuildRequest = XhrBuildRequest() + return xhrBuildRequest.post(request, *args, **kwargs) + + logger.warning("ERROR:XhrProjectUpdate") + response = HttpResponse() + response.status_code = 500 + return response + +class XhrSetDefaultImageUrl(View): + + def get(self, request, *args, **kwargs): + return HttpResponse() + + def post(self, request, *args, **kwargs): + """ + Project Update + + Entry point: /xhr_setdefaultimage/ + Method: POST + + Args: + pid: pid of project to update default image + + Returns: + {"error": "ok"} + or + {"error": } + """ + + project = Project.objects.get(pk=kwargs['pid']) + logger.debug("XhrSetDefaultImageUrl:project.pk=%d" % (project.pk)) + + # set any default image recipe + if 'targets' in request.POST: + default_target = str(request.POST['targets']) + project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,default_target) + logger.debug("XhrSetDefaultImageUrl,project.pk=%d,project.builddir=%s" % (project.pk,project.builddir)) + return error_response('ok') + + logger.warning("ERROR:XhrSetDefaultImageUrl") + response = HttpResponse() + response.status_code = 500 + return response + + +# +# Layer Management +# +# Rules for 'local_source_dir' layers +# * Layers must have a unique name in the Layers table +# * A 'local_source_dir' layer is supposed to be shared +# by all projects that use it, so that it can have the +# same logical name +# * Each project that uses a layer will have its own +# LayerVersion and Project Layer for it +# * During the Paroject delete process, when the last +# LayerVersion for a 'local_source_dir' layer is deleted +# then the Layer record is deleted to remove orphans +# + +def scan_layer_content(layer,layer_version): + # if this is a local layer directory, we can immediately scan its content + if layer.local_source_dir: + try: + # recipes-*/*/*.bb + cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb')) + recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read() + recipes_list = recipes_list.decode("utf-8").strip() + if recipes_list and 'No such' not in recipes_list: + for recipe in recipes_list.split('\n'): + recipe_path = recipe[recipe.rfind('recipes-'):] + recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','') + recipe_ver = recipe_name.rfind('_') + if recipe_ver > 0: + recipe_name = recipe_name[0:recipe_ver] + if recipe_name: + ro, created = Recipe.objects.get_or_create( + layer_version=layer_version, + name=recipe_name + ) + if created: + ro.file_path = recipe_path + ro.summary = 'Recipe %s from layer %s' % (recipe_name,layer.name) + ro.description = ro.summary + ro.save() + + except Exception as e: + logger.warning("ERROR:scan_layer_content: %s" % e) + class XhrLayer(View): """ Delete, Get, Add and Update Layer information @@ -265,6 +403,7 @@ class XhrLayer(View): (csv)] """ + try: project = Project.objects.get(pk=kwargs['pid']) @@ -285,7 +424,13 @@ class XhrLayer(View): if layer_data['name'] in existing_layers: return JsonResponse({"error": "layer-name-exists"}) - layer = Layer.objects.create(name=layer_data['name']) + if ('local_source_dir' in layer_data): + # Local layer can be shared across projects. They have no 'release' + # and are not included in get_all_compatible_layer_versions() above + layer,created = Layer.objects.get_or_create(name=layer_data['name']) + _log("Local Layer created=%s" % created) + else: + layer = Layer.objects.create(name=layer_data['name']) layer_version = Layer_Version.objects.create( layer=layer, @@ -293,7 +438,7 @@ class XhrLayer(View): layer_source=LayerSource.TYPE_IMPORTED) # Local layer - if ('local_source_dir' in layer_data) and layer.local_source_dir: + if ('local_source_dir' in layer_data): ### and layer.local_source_dir: layer.local_source_dir = layer_data['local_source_dir'] # git layer elif 'vcs_url' in layer_data: @@ -325,6 +470,9 @@ class XhrLayer(View): 'layerdetailurl': layer_dep.get_detailspage_url(project.pk)}) + # Scan the layer's content and update components + scan_layer_content(layer,layer_version) + except Layer_Version.DoesNotExist: return error_response("layer-dep-not-found") except Project.DoesNotExist: @@ -1014,8 +1162,24 @@ class XhrProject(View): state=BuildRequest.REQ_INPROGRESS): XhrBuildRequest.cancel_build(br) + # gather potential orphaned local layers attached to this project + project_local_layer_list = [] + for pl in ProjectLayer.objects.filter(project=project): + if pl.layercommit.layer_source == LayerSource.TYPE_IMPORTED: + project_local_layer_list.append(pl.layercommit.layer) + + # deep delete the project and its dependencies project.delete() + # delete any local layers now orphaned + _log("LAYER_ORPHAN_CHECK:Check for orphaned layers") + for layer in project_local_layer_list: + layer_refs = Layer_Version.objects.filter(layer=layer) + _log("LAYER_ORPHAN_CHECK:Ref Count for '%s' = %d" % (layer.name,len(layer_refs))) + if 0 == len(layer_refs): + _log("LAYER_ORPHAN_CHECK:DELETE orpahned '%s'" % (layer.name)) + Layer.objects.filter(pk=layer.id).delete() + except Project.DoesNotExist: return error_response("Project %s does not exist" % kwargs['project_id']) diff --git a/bitbake/lib/toaster/toastergui/static/js/layerBtn.js b/bitbake/lib/toaster/toastergui/static/js/layerBtn.js index 9f9eda1e1e..a5a6563d1a 100644 --- a/bitbake/lib/toaster/toastergui/static/js/layerBtn.js +++ b/bitbake/lib/toaster/toastergui/static/js/layerBtn.js @@ -67,6 +67,18 @@ function layerBtnsInit() { }); }); + $("td .set-default-recipe-btn").unbind('click'); + $("td .set-default-recipe-btn").click(function(e){ + e.preventDefault(); + var recipe = $(this).data('recipe-name'); + + libtoaster.setDefaultImage(null, recipe, + function(){ + /* Success */ + window.location.replace(libtoaster.ctx.projectSpecificPageUrl); + }); + }); + $(".customise-btn").unbind('click'); $(".customise-btn").click(function(e){ diff --git a/bitbake/lib/toaster/toastergui/static/js/libtoaster.js b/bitbake/lib/toaster/toastergui/static/js/libtoaster.js index 6f9b5d0f00..2e8863af26 100644 --- a/bitbake/lib/toaster/toastergui/static/js/libtoaster.js +++ b/bitbake/lib/toaster/toastergui/static/js/libtoaster.js @@ -465,6 +465,108 @@ var libtoaster = (function () { $.cookie('toaster-notification', JSON.stringify(data), { path: '/'}); } + /* _updateProject: + * url: xhrProjectUpdateUrl or null for current project + * onsuccess: callback for successful execution + * onfail: callback for failed execution + */ + function _updateProject (url, targets, default_image, onsuccess, onfail) { + + if (!url) + url = libtoaster.ctx.xhrProjectUpdateUrl; + + /* Flatten the array of targets into a space spearated list */ + if (targets instanceof Array){ + targets = targets.reduce(function(prevV, nextV){ + return prev + ' ' + next; + }); + } + + $.ajax( { + type: "POST", + url: url, + data: { 'do_update' : 'True' , 'targets' : targets , 'default_image' : default_image , }, + headers: { 'X-CSRFToken' : $.cookie('csrftoken')}, + success: function (_data) { + if (_data.error !== "ok") { + console.warn(_data.error); + } else { + if (onsuccess !== undefined) onsuccess(_data); + } + }, + error: function (_data) { + console.warn("Call failed"); + console.warn(_data); + if (onfail) onfail(data); + } }); + } + + /* _cancelProject: + * url: xhrProjectUpdateUrl or null for current project + * onsuccess: callback for successful execution + * onfail: callback for failed execution + */ + function _cancelProject (url, onsuccess, onfail) { + + if (!url) + url = libtoaster.ctx.xhrProjectCancelUrl; + + $.ajax( { + type: "POST", + url: url, + data: { 'do_cancel' : 'True' }, + headers: { 'X-CSRFToken' : $.cookie('csrftoken')}, + success: function (_data) { + if (_data.error !== "ok") { + console.warn(_data.error); + } else { + if (onsuccess !== undefined) onsuccess(_data); + } + }, + error: function (_data) { + console.warn("Call failed"); + console.warn(_data); + if (onfail) onfail(data); + } }); + } + + /* _setDefaultImage: + * url: xhrSetDefaultImageUrl or null for current project + * targets: an array or space separated list of targets to set as default + * onsuccess: callback for successful execution + * onfail: callback for failed execution + */ + function _setDefaultImage (url, targets, onsuccess, onfail) { + + if (!url) + url = libtoaster.ctx.xhrSetDefaultImageUrl; + + /* Flatten the array of targets into a space spearated list */ + if (targets instanceof Array){ + targets = targets.reduce(function(prevV, nextV){ + return prev + ' ' + next; + }); + } + + $.ajax( { + type: "POST", + url: url, + data: { 'targets' : targets }, + headers: { 'X-CSRFToken' : $.cookie('csrftoken')}, + success: function (_data) { + if (_data.error !== "ok") { + console.warn(_data.error); + } else { + if (onsuccess !== undefined) onsuccess(_data); + } + }, + error: function (_data) { + console.warn("Call failed"); + console.warn(_data); + if (onfail) onfail(data); + } }); + } + return { enableAjaxLoadingTimer: _enableAjaxLoadingTimer, disableAjaxLoadingTimer: _disableAjaxLoadingTimer, @@ -485,6 +587,9 @@ var libtoaster = (function () { createCustomRecipe: _createCustomRecipe, makeProjectNameValidation: _makeProjectNameValidation, setNotification: _setNotification, + updateProject : _updateProject, + cancelProject : _cancelProject, + setDefaultImage : _setDefaultImage, }; })(); diff --git a/bitbake/lib/toaster/toastergui/static/js/mrbsection.js b/bitbake/lib/toaster/toastergui/static/js/mrbsection.js index c0c5fa9589..f07ccf8181 100644 --- a/bitbake/lib/toaster/toastergui/static/js/mrbsection.js +++ b/bitbake/lib/toaster/toastergui/static/js/mrbsection.js @@ -86,7 +86,7 @@ function mrbSectionInit(ctx){ if (buildFinished(build)) { // a build finished: reload the whole page so that the build // shows up in the builds table - window.location.reload(); + window.location.reload(true); } else if (stateChanged(build)) { // update the whole template @@ -110,6 +110,8 @@ function mrbSectionInit(ctx){ // update the clone progress text selector = '#repos-cloned-percentage-' + build.id; $(selector).html(build.repos_cloned_percentage); + selector = '#repos-cloned-progressitem-' + build.id; + $(selector).html('('+build.progress_item+')'); // update the recipe progress bar selector = '#repos-cloned-percentage-bar-' + build.id; diff --git a/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js b/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js index 69220aaf57..3f9e186708 100644 --- a/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js +++ b/bitbake/lib/toaster/toastergui/static/js/projecttopbar.js @@ -14,6 +14,9 @@ function projectTopBarInit(ctx) { var newBuildTargetBuildBtn = $("#build-button"); var selectedTarget; + var updateProjectBtn = $("#update-project-button"); + var cancelProjectBtn = $("#cancel-project-button"); + /* Project name change functionality */ projectNameFormToggle.click(function(e){ e.preventDefault(); @@ -89,6 +92,25 @@ function projectTopBarInit(ctx) { }, null); }); + updateProjectBtn.click(function (e) { + e.preventDefault(); + + selectedTarget = { name: "_PROJECT_PREPARE_" }; + + /* Save current default build image, fire off the build */ + libtoaster.updateProject(null, selectedTarget.name, newBuildTargetInput.val().trim(), + function(){ + window.location.replace(libtoaster.ctx.projectSpecificPageUrl); + }, null); + }); + + cancelProjectBtn.click(function (e) { + e.preventDefault(); + + /* redirect to 'done/canceled' landing page */ + window.location.replace(libtoaster.ctx.landingSpecificCancelURL); + }); + /* Call makeProjectNameValidation function */ libtoaster.makeProjectNameValidation($("#project-name-change-input"), $("#hint-error-project-name"), $("#validate-project-name"), diff --git a/bitbake/lib/toaster/toastergui/tables.py b/bitbake/lib/toaster/toastergui/tables.py index dca2fa2913..03bd2ae9c6 100644 --- a/bitbake/lib/toaster/toastergui/tables.py +++ b/bitbake/lib/toaster/toastergui/tables.py @@ -35,6 +35,8 @@ from toastergui.tablefilter import TableFilterActionToggle from toastergui.tablefilter import TableFilterActionDateRange from toastergui.tablefilter import TableFilterActionDay +import os + class ProjectFilters(object): @staticmethod def in_project(project_layers): @@ -339,6 +341,8 @@ class RecipesTable(ToasterTable): 'filter_name' : "in_current_project", 'static_data_name' : "add-del-layers", 'static_data_template' : '{% include "recipe_btn.html" %}'} + if '1' == os.environ.get('TOASTER_PROJECTSPECIFIC'): + build_col['static_data_template'] = '{% include "recipe_add_btn.html" %}' def get_context_data(self, **kwargs): project = Project.objects.get(pk=kwargs['pid']) diff --git a/bitbake/lib/toaster/toastergui/templates/base_specific.html b/bitbake/lib/toaster/toastergui/templates/base_specific.html new file mode 100644 index 0000000000..e377cadd73 --- /dev/null +++ b/bitbake/lib/toaster/toastergui/templates/base_specific.html @@ -0,0 +1,128 @@ + +{% load static %} +{% load projecttags %} +{% load project_url_tag %} + + + + {% block title %} Toaster {% endblock %} + + + + + + + + + + + + + + + + {% if DEBUG %} + + {% endif %} + + {% block extraheadcontent %} + {% endblock %} + + + + + {% csrf_token %} + + + + + + +
+ {% block pagecontent %} + {% endblock %} +
+ + diff --git a/bitbake/lib/toaster/toastergui/templates/baseprojectspecificpage.html b/bitbake/lib/toaster/toastergui/templates/baseprojectspecificpage.html new file mode 100644 index 0000000000..d0b588de98 --- /dev/null +++ b/bitbake/lib/toaster/toastergui/templates/baseprojectspecificpage.html @@ -0,0 +1,48 @@ +{% extends "base_specific.html" %} + +{% load projecttags %} +{% load humanize %} + +{% block title %} {{title}} - {{project.name}} - Toaster {% endblock %} + +{% block pagecontent %} + +
+ {% include "project_specific_topbar.html" %} + + + +
+ +
+
+ {% block projectinfomain %}{% endblock %} +
+ +
+{% endblock %} + diff --git a/bitbake/lib/toaster/toastergui/templates/generic-toastertable-page.html b/bitbake/lib/toaster/toastergui/templates/generic-toastertable-page.html index b3eabe1a26..99fbb38970 100644 --- a/bitbake/lib/toaster/toastergui/templates/generic-toastertable-page.html +++ b/bitbake/lib/toaster/toastergui/templates/generic-toastertable-page.html @@ -1,4 +1,4 @@ -{% extends "baseprojectpage.html" %} +{% extends project_specific|yesno:"baseprojectspecificpage.html,baseprojectpage.html" %} {% load projecttags %} {% load humanize %} {% load static %} diff --git a/bitbake/lib/toaster/toastergui/templates/importlayer.html b/bitbake/lib/toaster/toastergui/templates/importlayer.html index 97d52c76c1..e0c987eef1 100644 --- a/bitbake/lib/toaster/toastergui/templates/importlayer.html +++ b/bitbake/lib/toaster/toastergui/templates/importlayer.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends project_specific|yesno:"baseprojectspecificpage.html,base.html" %} {% load projecttags %} {% load humanize %} {% load static %} @@ -6,7 +6,7 @@ {% block pagecontent %}
- {% include "projecttopbar.html" %} + {% include project_specific|yesno:"project_specific_topbar.html,projecttopbar.html" %} {% if project and project.release %} diff --git a/bitbake/lib/toaster/toastergui/templates/landing_specific.html b/bitbake/lib/toaster/toastergui/templates/landing_specific.html new file mode 100644 index 0000000000..e289c7d4a5 --- /dev/null +++ b/bitbake/lib/toaster/toastergui/templates/landing_specific.html @@ -0,0 +1,50 @@ +{% extends "base_specific.html" %} + +{% load static %} +{% load projecttags %} +{% load humanize %} + +{% block title %} Welcome to Toaster {% endblock %} + +{% block pagecontent %} + +
+
+ +
+

+ Configuration {% if status == "cancel" %}Canceled{% else %}Completed{% endif %}! You can now close this window. +

+
+
+

+ Your project configuration {% if status == "cancel" %}changes have been canceled{% else %}has completed!{% endif %} +
+
+

    +
  • + The Toaster instance for project configuration has been shut down +
    • +
  • + You can start Toaster independently for advanced project management and analysis: + 
    
+         Set up bitbake environment:
+         $ cd {{install_dir}}
+         $ . oe-init-build-env [toaster_server]
+
+         Option 1: Start a local Toaster server, open local browser to "localhost:8000"
+         $ . toaster start webport=8000
+
+         Option 2: Start a shared Toaster server, open any browser to "[host_ip]:8000"
+         $ . toaster start webport=0.0.0.0:8000
+
+         To stop the Toaster server:
+         $ . toaster stop
+
    +
    • +
+

+
+
+ +{% endblock %} diff --git a/bitbake/lib/toaster/toastergui/templates/layerdetails.html b/bitbake/lib/toaster/toastergui/templates/layerdetails.html index e0069db80c..1e26e31c8b 100644 --- a/bitbake/lib/toaster/toastergui/templates/layerdetails.html +++ b/bitbake/lib/toaster/toastergui/templates/layerdetails.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends project_specific|yesno:"baseprojectspecificpage.html,base.html" %} {% load projecttags %} {% load humanize %} {% load static %} @@ -310,6 +310,7 @@ {% endwith %} {% endwith %}
+
diff --git a/bitbake/lib/toaster/toastergui/templates/mrb_section.html b/bitbake/lib/toaster/toastergui/templates/mrb_section.html index c5b9fe90d3..98d9fac822 100644 --- a/bitbake/lib/toaster/toastergui/templates/mrb_section.html +++ b/bitbake/lib/toaster/toastergui/templates/mrb_section.html @@ -119,7 +119,7 @@ title="Toaster is cloning the repos required for your build"> - Cloning <%:repos_cloned_percentage%>% complete + Cloning <%:repos_cloned_percentage%>% complete (<%:progress_item%>) <%include tmpl='#cancel-template'/%> diff --git a/bitbake/lib/toaster/toastergui/templates/newcustomimage.html b/bitbake/lib/toaster/toastergui/templates/newcustomimage.html index 980179a406..0766e5e4cf 100644 --- a/bitbake/lib/toaster/toastergui/templates/newcustomimage.html +++ b/bitbake/lib/toaster/toastergui/templates/newcustomimage.html @@ -1,4 +1,4 @@ -{% extends "base.html" %} +{% extends project_specific|yesno:"baseprojectspecificpage.html,base.html" %} {% load projecttags %} {% load humanize %} {% load static %} @@ -8,7 +8,7 @@
- {% include "projecttopbar.html" %} + {% include project_specific|yesno:"project_specific_topbar.html,projecttopbar.html" %}
{% url table_name project.id as xhr_table_url %} diff --git a/bitbake/lib/toaster/toastergui/templates/newproject_specific.html b/bitbake/lib/toaster/toastergui/templates/newproject_specific.html new file mode 100644 index 0000000000..cfa77f2e40 --- /dev/null +++ b/bitbake/lib/toaster/toastergui/templates/newproject_specific.html @@ -0,0 +1,95 @@ +{% extends "base.html" %} +{% load projecttags %} +{% load humanize %} + +{% block title %} Create a new project - Toaster {% endblock %} + +{% block pagecontent %} +
+
+
+

Create a new project

+
+ {% if alert %} + + {% endif %} + +
{% csrf_token %} +
+ + +
+ + + + + {% if releases.count > 0 %} +
+ {% if releases.count > 1 %} + + +
+
+ {% for release in releases %} + + {% endfor %} + {% else %} + + {% endif %} +
+
+ + {% endif %} +
+ + To create a project, you need to specify the release +
+ + +
+
+ + + +{% endblock %} diff --git a/bitbake/lib/toaster/toastergui/templates/project.html b/bitbake/lib/toaster/toastergui/templates/project.html index 11603d1e12..fa41e3c909 100644 --- a/bitbake/lib/toaster/toastergui/templates/project.html +++ b/bitbake/lib/toaster/toastergui/templates/project.html @@ -1,4 +1,4 @@ -{% extends "baseprojectpage.html" %} +{% extends project_specific|yesno:"baseprojectspecificpage.html,baseprojectpage.html" %} {% load projecttags %} {% load humanize %} @@ -18,7 +18,7 @@ try { projectPageInit(ctx); } catch (e) { - document.write("Sorry, An error has occurred loading this page"); + document.write("Sorry, An error has occurred loading this page (project):"+e); console.warn(e); } }); @@ -93,6 +93,7 @@
+ {% if not project_specific %}

Most built recipes

@@ -105,6 +106,7 @@
+ {% endif %}

Project release

@@ -157,5 +159,6 @@
+
{% endblock %} diff --git a/bitbake/lib/toaster/toastergui/templates/project_specific.html b/bitbake/lib/toaster/toastergui/templates/project_specific.html new file mode 100644 index 0000000000..f625d18baf --- /dev/null +++ b/bitbake/lib/toaster/toastergui/templates/project_specific.html @@ -0,0 +1,162 @@ +{% extends "baseprojectspecificpage.html" %} + +{% load projecttags %} +{% load humanize %} +{% load static %} + +{% block title %} Configuration - {{project.name}} - Toaster {% endblock %} +{% block projectinfomain %} + + + + + + + + +