1 files changed, 586 insertions, 0 deletions

diff --git a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
new file mode 100644
index 0000000000..791d528231
--- /dev/null
+++ b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
@@ -0,0 +1,586 @@
+#
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# BitBake Toaster Implementation
+#
+# Copyright (C) 2018        Wind River Systems
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# buildimport: import a project for project specific configuration
+#
+# Usage:
+#  (a) Set up Toaster environent
+#
+#  (b) Call buildimport
+#      $ /path/to/bitbake/lib/toaster/manage.py buildimport \
+#        --name=$PROJECTNAME \
+#        --path=$BUILD_DIRECTORY \
+#        --callback="$CALLBACK_SCRIPT" \
+#        --command="configure|reconfigure|import"
+#
+#  (c) Return is "|Default_image=%s|Project_id=%d"
+#
+#  (d) Open Toaster to this project using for example:
+#      $ xdg-open http://localhost:$toaster_port/toastergui/project_specific/$project_id
+#
+#  (e) To delete a project:
+#      $ /path/to/bitbake/lib/toaster/manage.py buildimport \
+#        --name=$PROJECTNAME --delete-project
+#
+
+
+# ../bitbake/lib/toaster/manage.py buildimport --name=test --path=`pwd` --callback="" --command=import
+
+from django.core.management.base import BaseCommand, CommandError
+from django.core.exceptions import ObjectDoesNotExist
+from orm.models import ProjectManager, Project, Release, ProjectVariable
+from orm.models import Layer, Layer_Version, LayerSource, ProjectLayer
+from toastergui.api import scan_layer_content
+from django.db import OperationalError
+
+import os
+import re
+import os.path
+import subprocess
+
+# Toaster variable section delimiters
+TOASTER_PROLOG = '#=== TOASTER_CONFIG_PROLOG ==='
+TOASTER_EPILOG = '#=== TOASTER_CONFIG_EPILOG ==='
+
+# quick development/debugging support
+verbose = 2
+def _log(msg):
+    if 1 == verbose:
+        print(msg)
+    elif 2 == verbose:
+        f1=open('/tmp/toaster.log', 'a')
+        f1.write("|" + msg + "|\n" )
+        f1.close()
+
+
+__config_regexp__  = re.compile( r"""
+    ^
+    (?P<exp>export\s+)?
+    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
+    (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
+
+    \s* (
+        (?P<colon>:=) |
+        (?P<lazyques>\?\?=) |
+        (?P<ques>\?=) |
+        (?P<append>\+=) |
+        (?P<prepend>=\+) |
+        (?P<predot>=\.) |
+        (?P<postdot>\.=) |
+        =
+    ) \s*
+
+    (?!'[^']*'[^']*'$)
+    (?!\"[^\"]*\"[^\"]*\"$)
+    (?P<apo>['\"])
+    (?P<value>.*)
+    (?P=apo)
+    $
+    """, re.X)
+
+class Command(BaseCommand):
+    args    = "<name> <path> <release>"
+    help    = "Import a command line build directory"
+    vars    = {}
+    toaster_vars = {}
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--name', dest='name', required=True,
+            help='name of the project',
+            )
+        parser.add_argument(
+            '--path', dest='path', required=True,
+            help='path to the project',
+            )
+        parser.add_argument(
+            '--release', dest='release', required=False,
+            help='release for the project',
+            )
+        parser.add_argument(
+            '--callback', dest='callback', required=False,
+            help='callback for project config update',
+            )
+        parser.add_argument(
+            '--delete-project', dest='delete_project', required=False,
+            help='delete this project from the database',
+            )
+        parser.add_argument(
+            '--command', dest='command', required=False,
+            help='command (configure,reconfigure,import)',
+            )
+
+    # Extract the bb variables from a conf file
+    def scan_conf(self,fn):
+        vars = self.vars
+        toaster_vars = self.toaster_vars
+
+        #_log("scan_conf:%s" % fn)
+        if not os.path.isfile(fn):
+            return
+        f = open(fn, 'r')
+
+        #statements = ast.StatementGroup()
+        lineno = 0
+        is_toaster_section = False
+        while True:
+            lineno = lineno + 1
+            s = f.readline()
+            if not s:
+                break
+            w = s.strip()
+            # skip empty lines
+            if not w:
+                continue
+            # evaluate Toaster sections
+            if w.startswith(TOASTER_PROLOG):
+                is_toaster_section = True
+                continue
+            if w.startswith(TOASTER_EPILOG):
+                is_toaster_section = False
+                continue
+            s = s.rstrip()
+            while s[-1] == '\\':
+                s2 = f.readline().strip()
+                lineno = lineno + 1
+                if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
+                    echo("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+                s = s[:-1] + s2
+            # skip comments
+            if s[0] == '#':
+                continue
+            # process the line for just assignments
+            m = __config_regexp__.match(s)
+            if m:
+                groupd = m.groupdict()
+                var = groupd['var']
+                value = groupd['value']
+
+                if groupd['lazyques']:
+                    if not var in vars:
+                        vars[var] = value
+                    continue
+                if groupd['ques']:
+                    if not var in vars:
+                        vars[var] = value
+                    continue
+                # preset empty blank for remaining operators
+                if not var in vars:
+                    vars[var] = ''
+                if groupd['append']:
+                    vars[var] += value
+                elif groupd['prepend']:
+                    vars[var] = "%s%s" % (value,vars[var])
+                elif groupd['predot']:
+                    vars[var] = "%s %s" % (value,vars[var])
+                elif groupd['postdot']:
+                    vars[var] = "%s %s" % (vars[var],value)
+                else:
+                    vars[var] = "%s" % (value)
+                # capture vars in a Toaster section
+                if is_toaster_section:
+                    toaster_vars[var] = vars[var]
+
+        # DONE WITH PARSING
+        f.close()
+        self.vars = vars
+        self.toaster_vars = toaster_vars
+
+    # Update the scanned project variables
+    def update_project_vars(self,project,name):
+        pv, create = ProjectVariable.objects.get_or_create(project = project, name = name)
+        if (not name in self.vars.keys()) or (not self.vars[name]):
+            self.vars[name] = pv.value
+        else:
+            if pv.value != self.vars[name]:
+                pv.value = self.vars[name]
+        pv.save()
+
+    # Find the git version of the installation
+    def find_layer_dir_version(self,path):
+        #  * rocko               ...
+
+        install_version = ''
+        cwd = os.getcwd()
+        os.chdir(path)
+        p = subprocess.Popen(['git', 'branch', '-av'], stdout=subprocess.PIPE,
+                                                stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        out = out.decode("utf-8")
+        for branch in out.split('\n'):
+            if ('*' == branch[0:1]) and ('no branch' not in branch):
+                install_version = re.sub(' .*','',branch[2:])
+                break
+            if 'remotes/m/master' in branch:
+                install_version = re.sub('.*base/','',branch)
+                break
+        os.chdir(cwd)
+        return install_version
+
+    # Compute table of the installation's registered layer versions (branch or commit)
+    def find_layer_dir_versions(self,INSTALL_URL_PREFIX):
+        lv_dict = {}
+        layer_versions = Layer_Version.objects.all()
+        for lv in layer_versions:
+            layer = Layer.objects.filter(pk=lv.layer.pk)[0]
+            if layer.vcs_url:
+                url_short = layer.vcs_url.replace(INSTALL_URL_PREFIX,'')
+            else:
+                url_short = ''
+            # register the core, branch, and the version variations
+            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,'')] = (lv.id,layer.name)
+            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.branch)] = (lv.id,layer.name)
+            lv_dict["%s,%s,%s" % (url_short,lv.dirpath,lv.commit)] = (lv.id,layer.name)
+            #_log("  (%s,%s,%s|%s) = (%s,%s)" % (url_short,lv.dirpath,lv.branch,lv.commit,lv.id,layer.name))
+        return lv_dict
+
+    # Apply table of all layer versions
+    def extract_bblayers(self):
+        # set up the constants
+        bblayer_str = self.vars['BBLAYERS']
+        TOASTER_DIR = os.environ.get('TOASTER_DIR')
+        INSTALL_CLONE_PREFIX = os.path.dirname(TOASTER_DIR) + "/"
+        TOASTER_CLONE_PREFIX = TOASTER_DIR + "/_toaster_clones/"
+        INSTALL_URL_PREFIX = ''
+        layers = Layer.objects.filter(name='openembedded-core')
+        for layer in layers:
+            if layer.vcs_url:
+                INSTALL_URL_PREFIX = layer.vcs_url
+                break
+        INSTALL_URL_PREFIX = INSTALL_URL_PREFIX.replace("/poky","/")
+        INSTALL_VERSION_DIR = TOASTER_DIR
+        INSTALL_URL_POSTFIX = INSTALL_URL_PREFIX.replace(':','_')
+        INSTALL_URL_POSTFIX = INSTALL_URL_POSTFIX.replace('/','_')
+        INSTALL_URL_POSTFIX = "%s_%s" % (TOASTER_CLONE_PREFIX,INSTALL_URL_POSTFIX)
+
+        # get the set of available layer:layer_versions
+        lv_dict = self.find_layer_dir_versions(INSTALL_URL_PREFIX)
+
+        # compute the layer matches
+        layers_list = []
+        for line in bblayer_str.split(' '):
+            if not line:
+                continue
+            if line.endswith('/local'):
+                continue
+
+            # isolate the repo
+            layer_path = line
+            line = line.replace(INSTALL_URL_POSTFIX,'').replace(INSTALL_CLONE_PREFIX,'').replace('/layers/','/').replace('/poky/','/')
+
+            # isolate the sub-path
+            path_index = line.rfind('/')
+            if path_index > 0:
+                sub_path = line[path_index+1:]
+                line = line[0:path_index]
+            else:
+                sub_path = ''
+
+            # isolate the version
+            if TOASTER_CLONE_PREFIX in layer_path:
+                is_toaster_clone = True
+                # extract version from name syntax
+                version_index = line.find('_')
+                if version_index > 0:
+                    version = line[version_index+1:]
+                    line = line[0:version_index]
+                else:
+                    version = ''
+                _log("TOASTER_CLONE(%s/%s), version=%s" % (line,sub_path,version))
+            else:
+                is_toaster_clone = False
+                # version is from the installation
+                version = self.find_layer_dir_version(layer_path)
+                _log("LOCAL_CLONE(%s/%s), version=%s" % (line,sub_path,version))
+
+            # capture the layer information into layers_list
+            layers_list.append( (line,sub_path,version,layer_path,is_toaster_clone) )
+        return layers_list,lv_dict
+
+    #
+    def find_import_release(self,layers_list,lv_dict,default_release):
+        #   poky,meta,rocko => 4;openembedded-core
+        release = default_release
+        for line,path,version,layer_path,is_toaster_clone in layers_list:
+            key = "%s,%s,%s" % (line,path,version)
+            if key in lv_dict:
+                lv_id = lv_dict[key]
+                if 'openembedded-core' == lv_id[1]:
+                    _log("Find_import_release(%s):version=%s,Toaster=%s" % (lv_id[1],version,is_toaster_clone))
+                    # only versions in Toaster managed layers are accepted
+                    if not is_toaster_clone:
+                        break
+                    try:
+                        release = Release.objects.get(name=version)
+                    except:
+                        pass
+                    break
+        _log("Find_import_release:RELEASE=%s" % release.name)
+        return release
+
+    # Apply the found conf layers
+    def apply_conf_bblayers(self,layers_list,lv_dict,project,release=None):
+        for line,path,version,layer_path,is_toaster_clone in layers_list:
+            # Assert release promote if present
+            if release:
+                version = release
+            # try to match the key to a layer_version
+            key = "%s,%s,%s" % (line,path,version)
+            key_short = "%s,%s,%s" % (line,path,'')
+            lv_id = ''
+            if key in lv_dict:
+                lv_id = lv_dict[key]
+                lv = Layer_Version.objects.get(pk=int(lv_id[0]))
+                pl,created = ProjectLayer.objects.get_or_create(project=project,
+                                                   layercommit=lv)
+                pl.optional=False
+                pl.save()
+                _log("  %s => %s;%s" % (key,lv_id[0],lv_id[1]))
+            elif key_short in lv_dict:
+                lv_id = lv_dict[key_short]
+                lv = Layer_Version.objects.get(pk=int(lv_id[0]))
+                pl,created = ProjectLayer.objects.get_or_create(project=project,
+                                                   layercommit=lv)
+                pl.optional=False
+                pl.save()
+                _log("  %s ?> %s" % (key,lv_dict[key_short]))
+            else:
+                _log("%s <= %s" % (key,layer_path))
+                found = False
+                # does local layer already exist in this project?
+                try:
+                    for pl in ProjectLayer.objects.filter(project=project):
+                        if pl.layercommit.layer.local_source_dir == layer_path:
+                            found = True
+                            _log("  Project Local Layer found!")
+                except Exception as e:
+                    _log("ERROR: Local Layer '%s'" % e)
+                    pass
+
+                if not found:
+                    # Does Layer name+path already exist?
+                    try:
+                        layer_name_base = os.path.basename(layer_path)
+                        _log("Layer_lookup: try '%s','%s'" % (layer_name_base,layer_path))
+                        layer = Layer.objects.get(name=layer_name_base,local_source_dir = layer_path)
+                        # Found! Attach layer_version and ProjectLayer
+                        layer_version = Layer_Version.objects.create(
+                            layer=layer,
+                            project=project,
+                            layer_source=LayerSource.TYPE_IMPORTED)
+                        layer_version.save()
+                        pl,created = ProjectLayer.objects.get_or_create(project=project,
+                                                           layercommit=layer_version)
+                        pl.optional=False
+                        pl.save()
+                        found = True
+                        # add layer contents to this layer version
+                        scan_layer_content(layer,layer_version)
+                        _log("  Parent Local Layer found in db!")
+                    except Exception as e:
+                        _log("Layer_exists_test_failed: Local Layer '%s'" % e)
+                        pass
+
+                if not found:
+                    # Insure that layer path exists, in case of user typo
+                    if not os.path.isdir(layer_path):
+                        _log("ERROR:Layer path '%s' not found" % layer_path)
+                        continue
+                    # Add layer to db and attach project to it
+                    layer_name_base = os.path.basename(layer_path)
+                    # generate a unique layer name
+                    layer_name_matches = {}
+                    for layer in Layer.objects.filter(name__contains=layer_name_base):
+                        layer_name_matches[layer.name] = '1'
+                    layer_name_idx = 0
+                    layer_name_test = layer_name_base
+                    while layer_name_test in layer_name_matches.keys():
+                        layer_name_idx += 1
+                        layer_name_test = "%s_%d" % (layer_name_base,layer_name_idx)
+                    # create the layer and layer_verion objects
+                    layer = Layer.objects.create(name=layer_name_test)
+                    layer.local_source_dir = layer_path
+                    layer_version = Layer_Version.objects.create(
+                        layer=layer,
+                        project=project,
+                        layer_source=LayerSource.TYPE_IMPORTED)
+                    layer.save()
+                    layer_version.save()
+                    pl,created = ProjectLayer.objects.get_or_create(project=project,
+                                                       layercommit=layer_version)
+                    pl.optional=False
+                    pl.save()
+                    # register the layer's content
+                    _log("  Local Layer Add content")
+                    scan_layer_content(layer,layer_version)
+                    _log("  Local Layer Added '%s'!" % layer_name_test)
+
+    # Scan the project's conf files (if any)
+    def scan_conf_variables(self,project_path):
+        # scan the project's settings, add any new layers or variables
+        if os.path.isfile("%s/conf/local.conf" % project_path):
+            self.scan_conf("%s/conf/local.conf" % project_path)
+            self.scan_conf("%s/conf/bblayers.conf" % project_path)
+            # Import then disable old style Toaster conf files (before 'merged_attr')
+            old_toaster_local = "%s/conf/toaster.conf" % project_path
+            if os.path.isfile(old_toaster_local):
+                self.scan_conf(old_toaster_local)
+                shutil.move(old_toaster_local, old_toaster_local+"_old")
+            old_toaster_layer = "%s/conf/toaster-bblayers.conf" % project_path
+            if os.path.isfile(old_toaster_layer):
+                self.scan_conf(old_toaster_layer)
+                shutil.move(old_toaster_layer, old_toaster_layer+"_old")
+
+    # Scan the found conf variables (if any)
+    def apply_conf_variables(self,project,layers_list,lv_dict,release=None):
+        if self.vars:
+            # Catch vars relevant to Toaster (in case no Toaster section)
+            self.update_project_vars(project,'DISTRO')
+            self.update_project_vars(project,'MACHINE')
+            self.update_project_vars(project,'IMAGE_INSTALL_append')
+            self.update_project_vars(project,'IMAGE_FSTYPES')
+            self.update_project_vars(project,'PACKAGE_CLASSES')
+            # These vars are typically only assigned by Toaster
+            #self.update_project_vars(project,'DL_DIR')
+            #self.update_project_vars(project,'SSTATE_DIR')
+
+            # Assert found Toaster vars
+            for var in self.toaster_vars.keys():
+                pv, create = ProjectVariable.objects.get_or_create(project = project, name = var)
+                pv.value = self.toaster_vars[var]
+                _log("* Add/update Toaster var '%s' = '%s'" % (pv.name,pv.value))
+                pv.save()
+
+            # Assert found BBLAYERS
+            if 0 < verbose:
+                for pl in ProjectLayer.objects.filter(project=project):
+                    release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
+                    print(" BEFORE:ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
+            self.apply_conf_bblayers(layers_list,lv_dict,project,release)
+            if 0 < verbose:
+                for pl in ProjectLayer.objects.filter(project=project):
+                    release_name = 'None' if not pl.layercommit.release else pl.layercommit.release.name
+                    print(" AFTER :ProjectLayer=%s,%s,%s,%s" % (pl.layercommit.layer.name,release_name,pl.layercommit.branch,pl.layercommit.commit))
+
+
+    def handle(self, *args, **options):
+        project_name = options['name']
+        project_path = options['path']
+        project_callback = options['callback']
+        if options['release']:
+            release_name = options['release']
+        else:
+            release_name = ''
+
+        #
+        # Delete project
+        #
+
+        if options['delete_project']:
+            try:
+                print("Project '%s' delete from Toaster database" % (project_name))
+                project = Project.objects.get(name=project_name)
+                # TODO: deep project delete
+                project.delete()
+                print("Project '%s' Deleted" % (project_name))
+                return
+            except Exception as e:
+                print("Project '%s' not found, not deleted (%s)" % (project_name,e))
+                return
+
+        #
+        # Create/Update/Import project
+        #
+
+        # See if project (by name) exists
+        project = None
+        try:
+            # Project already exists
+            project = Project.objects.get(name=project_name)
+        except Exception as e:
+            pass
+
+        # Find the installation's default release
+        default_release = Release.objects.get(id=1)
+
+        # SANITY: if 'reconfig' but project does not exist (deleted externally), switch to 'import'
+        if ("reconfigure" == options['command']) and (None == project):
+            options['command'] = 'import'
+
+        # 'Configure':
+        if "configure" == options['command']:
+            # Note: ignore any existing conf files
+            # create project, SANITY: reuse any project of same name
+            project = Project.objects.create_project(project_name,default_release,project)
+
+        # 'Re-configure':
+        if "reconfigure" == options['command']:
+            # Scan the directory's conf files
+            self.scan_conf_variables(project_path)
+            # Scan the layer list
+            layers_list,lv_dict = self.extract_bblayers()
+            # Apply any new layers or variables
+            self.apply_conf_variables(project,layers_list,lv_dict)
+
+        # 'Import':
+        if "import" == options['command']:
+            # Scan the directory's conf files
+            self.scan_conf_variables(project_path)
+            # Remove these Toaster controlled variables
+            for var in ('DL_DIR','SSTATE_DIR'):
+                self.vars.pop(var, None)
+                self.toaster_vars.pop(var, None)
+            # Scan the layer list
+            layers_list,lv_dict = self.extract_bblayers()
+            # Find the directory's release, and promote to default_release if local paths
+            release = self.find_import_release(layers_list,lv_dict,default_release)
+            # create project, SANITY: reuse any project of same name
+            project = Project.objects.create_project(project_name,release,project)
+            # Apply any new layers or variables
+            self.apply_conf_variables(project,layers_list,lv_dict,release)
+            # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
+            project.set_variable('INTERNAL_PROJECT_SPECIFIC_SKIPRELEASE','1')
+
+        # Set up the project's meta data
+        project.builddir = project_path
+        project.merged_attr = True
+        project.set_variable(Project.PROJECT_SPECIFIC_CALLBACK,project_callback)
+        project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_EDIT)
+        if ("configure" == options['command']) or ("import" == options['command']):
+            # preset the mode and default image recipe
+            project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NEW)
+            project.set_variable(Project.PROJECT_SPECIFIC_DEFAULTIMAGE,"core-image-minimal")
+            # Assert any extended/custom actions or variables for new non-Toaster projects
+            if not len(self.toaster_vars):
+                pass
+        else:
+            project.set_variable(Project.PROJECT_SPECIFIC_ISNEW,Project.PROJECT_SPECIFIC_NONE)
+
+        # Save the updated Project
+        project.save()
+
+        _log("Buildimport:project='%s' at '%d'" % (project_name,project.id))
+
+        if ('DEFAULT_IMAGE' in self.vars) and (self.vars['DEFAULT_IMAGE']):
+            print("|Default_image=%s|Project_id=%d" % (self.vars['DEFAULT_IMAGE'],project.id))
+        else:
+            print("|Project_id=%d" % (project.id))
+