bitbake: toastergui: layer name correlation

This patch modifies how layers are identified and matched.

Layers were primarely organized by the source of layer information,
and Releases were separated by both layer git branches and originating
source of layer information. This setup prevented mixing layers from
different sources for a certain release, which didn't match the way
people use Yocto Project / bitbake.

This patch brings name-based indentification, where layers with the
same name are assumed to be equivalent, in the sense of being able
to substitute one another. To facilitate this identification to
humans, layers are differentiated by GIT URI instead of layer sources,
which was a rather arbitrary abstraction.

Additional changes include modification to models in order accomodate
for the new data structure, and to config file loading to match
the new toasterconf.json layout.

(Bitbake rev: 4357200aed522ad56cfd84917f877645b83b6a70)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 54 insertions, 22 deletions

diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
index 56e4e1bf0c..cd604eba7e 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/checksettings.py
@@ -34,6 +34,22 @@ class Command(NoArgsCommand):
                 return ret
         return None
 
+    def _recursive_list_directories(self, startdirectory, level = 0):
+        if level < 0:
+            return []
+        dirs = []
+        try:
+            for i in os.listdir(startdirectory):
+                j = os.path.join(startdirectory, i)
+                if os.path.isdir(j):
+                    dirs.append(j)
+        except OSError:
+            pass
+        for j in dirs:
+                dirs = dirs + self._recursive_list_directories(j, level - 1)
+        return dirs
+
+
     def _get_suggested_sourcedir(self, be):
         if be.betype != BuildEnvironment.TYPE_LOCAL:
             return ""
@@ -67,7 +83,6 @@ class Command(NoArgsCommand):
                 print("Verifying the Build Environment type %s id %d." % (be.get_betype_display(), be.pk))
                 if len(be.sourcedir) == 0:
                     suggesteddir = self._get_suggested_sourcedir(be)
-                    homesourcedir = suggesteddir
                     be.sourcedir = raw_input(" -- Layer sources checkout directory may not be empty [guessed \"%s\"]:" % suggesteddir)
                     if len(be.sourcedir) == 0 and len(suggesteddir) > 0:
                         be.sourcedir = suggesteddir
@@ -94,17 +109,25 @@ class Command(NoArgsCommand):
                     be.save()
 
                 if is_changed and be.betype == BuildEnvironment.TYPE_LOCAL:
-                    baselayerdir = DN(DN(self._find_first_path_for_file(homesourcedir, "toasterconf.json", 3)))
-                    if baselayerdir:
-                        i = raw_input(" -- Do you want to import basic layer configuration from \"%s\" ? (y/N):" % baselayerdir)
-                        if len(i) and i.upper()[0] == 'Y':
-                            from loadconf import Command as LoadConfigCommand
-                            LoadConfigCommand()._import_layer_config(os.path.join(baselayerdir, "meta/conf/toasterconf.json"))
-                            # we run lsupdates after config update
-                            print "Updating information from the layer source, please wait."
-                            from django.core.management import call_command
-                            call_command("lsupdates")
-                        pass
+                    for dirname in self._recursive_list_directories(be.sourcedir,2):
+                        if os.path.exists(os.path.join(dirname, ".templateconf")):
+                            import subprocess
+                            conffilepath, error = subprocess.Popen('bash -c ". '+os.path.join(dirname, ".templateconf")+'; echo \"\$TEMPLATECONF\""', shell=True, stdout=subprocess.PIPE).communicate()
+                            conffilepath = os.path.join(conffilepath.strip(), "toasterconf.json")
+                            candidatefilepath = os.path.join(dirname, conffilepath)
+                            if os.path.exists(candidatefilepath):
+                                i = raw_input(" -- Do you want to import basic layer configuration from \"%s\" ? (y/N):" % candidatefilepath)
+                                if len(i) and i.upper()[0] == 'Y':
+                                    from loadconf import Command as LoadConfigCommand
+
+                                    LoadConfigCommand()._import_layer_config(candidatefilepath)
+                                    # we run lsupdates after config update
+                                    print "Layer configuration imported. Updating information from the layer source, please wait."
+                                    from django.core.management import call_command
+                                    call_command("lsupdates")
+
+                                    # we don't look for any other config files
+                                    return is_changed
 
                 return is_changed
 
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
index c9af487d9d..6e1f97a9f9 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/loadconf.py
@@ -1,6 +1,6 @@
 from django.core.management.base import BaseCommand, CommandError
 from orm.models import LayerSource, ToasterSetting, Branch, Layer, Layer_Version
-from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer
+from orm.models import BitbakeVersion, Release, ReleaseDefaultLayer, ReleaseLayerSourcePriority
 import os
 
 from checksettings import DN
@@ -71,17 +71,23 @@ class Command(BaseCommand):
             assert 'name' in lsi
             assert 'branches' in lsi
 
-            if lsi['sourcetype'] == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"):
+            def _get_id_for_sourcetype(s):
+                for i in LayerSource.SOURCE_TYPE:
+                    if s == i[1]:
+                        return i[0]
+                raise Exception("Could not find definition for sourcetype " + s)
+
+            if _get_id_for_sourcetype(lsi['sourcetype']) == LayerSource.TYPE_LAYERINDEX or lsi['apiurl'].startswith("/"):
                 apiurl = lsi['apiurl']
             else:
                 apiurl = self._reduce_canon_path(os.path.join(DN(filepath), lsi['apiurl']))
 
             try:
-                ls = LayerSource.objects.get(sourcetype = lsi['sourcetype'], apiurl = apiurl)
+                ls = LayerSource.objects.get(sourcetype = _get_id_for_sourcetype(lsi['sourcetype']), apiurl = apiurl)
             except LayerSource.DoesNotExist:
                 ls = LayerSource.objects.create(
                     name = lsi['name'],
-                    sourcetype = lsi['sourcetype'],
+                    sourcetype = _get_id_for_sourcetype(lsi['sourcetype']),
                     apiurl = apiurl
                 )
 
@@ -121,17 +127,20 @@ class Command(BaseCommand):
             bvo = BitbakeVersion.objects.get(name = ri['bitbake'])
             assert bvo is not None
 
-            ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch = Branch.objects.get( layer_source__name = ri['layersource'], name=ri['branch']))
+            ro, created = Release.objects.get_or_create(name = ri['name'], bitbake_version = bvo, branch_name = ri['branch'])
             ro.description = ri['description']
             ro.helptext = ri['helptext']
             ro.save()
 
+            # save layer source priority for release
+            for ls_name in ri['layersourcepriority'].keys():
+                rlspo, created = ReleaseLayerSourcePriority.objects.get_or_create(release = ro, layer_source = LayerSource.objects.get(name=ls_name))
+                rlspo.priority = ri['layersourcepriority'][ls_name]
+                rlspo.save()
+
             for dli in ri['defaultlayers']:
-                layer, created = Layer.objects.get_or_create(
-                        layer_source = LayerSource.objects.get(name = ri['layersource']),
-                        name = dli
-                    )
-                ReleaseDefaultLayer.objects.get_or_create( release = ro, layer = layer)
+                # find layers with the same name
+                ReleaseDefaultLayer.objects.get_or_create( release = ro, layer_name = dli)
 
         # set default release
         if ToasterSetting.objects.filter(name = "DEFAULT_RELEASE").count() > 0: