bitbake: toastergui: layer name correlation

This patch modifies how layers are identified and matched.

Layers were primarely organized by the source of layer information,
and Releases were separated by both layer git branches and originating
source of layer information. This setup prevented mixing layers from
different sources for a certain release, which didn't match the way
people use Yocto Project / bitbake.

This patch brings name-based indentification, where layers with the
same name are assumed to be equivalent, in the sense of being able
to substitute one another. To facilitate this identification to
humans, layers are differentiated by GIT URI instead of layer sources,
which was a rather arbitrary abstraction.

Additional changes include modification to models in order accomodate
for the new data structure, and to config file loading to match
the new toasterconf.json layout.

(Bitbake rev: 4357200aed522ad56cfd84917f877645b83b6a70)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

4 files changed, 55 insertions, 41 deletions

diff --git a/bitbake/lib/toaster/toastergui/static/js/projectapp.js b/bitbake/lib/toaster/toastergui/static/js/projectapp.js
index 9f9a06476a..e9b07c7848 100644
--- a/bitbake/lib/toaster/toastergui/static/js/projectapp.js
+++ b/bitbake/lib/toaster/toastergui/static/js/projectapp.js
@@ -173,6 +173,8 @@ projectApp.controller('prjCtrl', function($scope, $modal, $http, $interval, $loc
             if (_data.error != "ok") {
                 alert("Failed XHR request (" + _status + "): " + _data.error);
                 console.error("Failed XHR request: ", _data, _status, _headers, _config);
+                // stop refreshing hte page
+                $interval.cancel($scope.pollHandle);
                 deffered.reject(_data.error);
             }
             else {
diff --git a/bitbake/lib/toaster/toastergui/templates/layers.html b/bitbake/lib/toaster/toastergui/templates/layers.html
index 8cb079d0a3..2bca84b503 100644
--- a/bitbake/lib/toaster/toastergui/templates/layers.html
+++ b/bitbake/lib/toaster/toastergui/templates/layers.html
@@ -9,9 +9,9 @@
 {% block projectinfomain %}
                 <div class="page-header">
                     <h1>
-  {% if request.GET.filter and objects.paginator.count >  0 or request.GET.search and objects.paginator.count >  0 %}
-      {{objects.paginator.count}} layer{{objects.paginator.count|pluralize}} found
-  {% elif request.GET.filter and objects.paginator.count ==  0 or request.GET.search and objects.paginator.count == 0 %}
+  {% if request.GET.filter and total_count >  0 or request.GET.search and total_count >  0 %}
+      {{total_count}} layer{{total_count|pluralize}} found
+  {% elif request.GET.filter and total_count ==  0 or request.GET.search and total_count == 0 %}
       No layers found
   {%else%}
       All layers
diff --git a/bitbake/lib/toaster/toastergui/templates/project.html b/bitbake/lib/toaster/toastergui/templates/project.html
index 4e8a7e29aa..e1ef824779 100644
--- a/bitbake/lib/toaster/toastergui/templates/project.html
+++ b/bitbake/lib/toaster/toastergui/templates/project.html
@@ -236,9 +236,9 @@ vim: expandtab tabstop=2
       <p><a href="{% url 'layers' %}">View all layers</a> | <a href="{% url 'importlayer' %}">Import layer</a></p>
       <ul class="unstyled configuration-list">
           <li ng-repeat="l in layers track by l.id" class="animate-repeat">
-            <a href="{[l.layerdetailurl]}" target="_#" class="layer-info" data-toggle="tooltip" tooltip="{[l.branch.layersource]} | {[l.branch.name]}">{[l.name]} </a>
+            <a href="{[l.layerdetailurl]}" target="_#" class="layer-info" data-toggle="tooltip" tooltip="{[l.giturl]} | {[l.branch.name]}">{[l.name]}</a>
             <i class="icon-trash" ng-click="layerDel(l.id)" tooltip="Delete"></i>
-          </li>
+                      </li>
       </ul>
     </div>
 
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py
index 5e92c24a8d..1b4bb9ff69 100755
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -1881,7 +1881,10 @@ if toastermain.settings.MANAGED:
             "MANAGED" : toastermain.settings.MANAGED
         }
         if 'project_id' in request.session:
-            ret['project'] = Project.objects.get(pk = request.session['project_id'])
+            try:
+                ret['project'] = Project.objects.get(pk = request.session['project_id'])
+            except Project.DoesNotExist:
+                del request.session['project_id']
         return ret
 
     # new project
@@ -1989,6 +1992,7 @@ if toastermain.settings.MANAGED:
                         "id": x.layercommit.pk,
                         "orderid": x.pk,
                         "name" : x.layercommit.layer.name,
+                        "giturl": x.layercommit.layer.vcs_url,
                         "url": x.layercommit.layer.layer_index_url,
                         "layerdetailurl": reverse("layerdetails", args=(x.layercommit.layer.pk,)),
                         "branch" : { "name" : x.layercommit.up_branch.name, "layersource" : x.layercommit.up_branch.layer_source.name}},
@@ -2053,6 +2057,9 @@ if toastermain.settings.MANAGED:
         except Exception as e:
             return HttpResponse(jsonfilter({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json")
 
+
+
+
     def xhr_projectedit(request, pid):
         try:
             prj = Project.objects.get(id = pid)
@@ -2078,11 +2085,14 @@ if toastermain.settings.MANAGED:
                 # we need to change the layers
                 for i in prj.projectlayer_set.all():
                     # find and add a similarly-named layer on the new branch
-                    lv = Layer_Version.objects.filter(layer__name = i.layercommit.layer.name, up_branch__release = prj.release)
-                    if lv.count() == 1:
-                        ProjectLayer.objects.get_or_create(project = prj, layercommit = lv[0])
-                    # get rid of the old entry
-                    i.delete()
+                    try:
+                        lv = Layer_Version.objects.filter(layer__name = i.layercommit.layer.name, up_branch__name = prj.release.branch_name)[0].get_equivalents_wpriority(prj)[0]
+                        ProjectLayer.objects.get_or_create(project = prj, layercommit = lv)
+                    except IndexError:
+                        pass
+                    finally:
+                        # get rid of the old entry
+                        i.delete()
 
             if 'machineName' in request.POST:
                 machinevar = prj.projectvariable_set.get(name="MACHINE")
@@ -2092,7 +2102,7 @@ if toastermain.settings.MANAGED:
             # return all project settings
             return HttpResponse(jsonfilter( {
                 "error": "ok",
-                "layers" :  map(lambda x: {"id": x.layercommit.pk, "orderid" : x.pk, "name" : x.layercommit.layer.name, "url": x.layercommit.layer.layer_index_url, "layerdetailurl": reverse("layerdetails", args=(x.layercommit.layer.pk,)), "branch" : { "name" : x.layercommit.up_branch.name, "layersource" : x.layercommit.up_branch.layer_source.name}}, prj.projectlayer_set.all().order_by("id")),
+                "layers" :  map(lambda x: {"id": x.layercommit.pk, "orderid" : x.pk, "name" : x.layercommit.layer.name, "giturl" : x.layercommit.layer.vcs_url, "url": x.layercommit.layer.layer_index_url, "layerdetailurl": reverse("layerdetails", args=(x.layercommit.layer.pk,)), "branch" : { "name" : x.layercommit.up_branch.name, "layersource" : x.layercommit.up_branch.layer_source.name}}, prj.projectlayer_set.all().order_by("id")),
                 "builds" : _project_recent_build_list(prj),
                 "variables": map(lambda x: (x.name, x.value), prj.projectvariable_set.all()),
                 "machine": {"name": prj.projectvariable_set.get(name="MACHINE").value},
@@ -2107,45 +2117,46 @@ if toastermain.settings.MANAGED:
     @csrf_exempt
     def xhr_datatypeahead(request):
         try:
+            prj = None
+            if 'project_id' in request.session:
+                prj = Project.objects.get(pk = request.session['project_id'])
+
             # returns layers for current project release that are not in the project set
             if request.GET['type'] == "layers":
-                queryset_all = Layer_Version.objects.all()
-                if 'project_id' in request.session:
-                    prj = Project.objects.get(pk = request.session['project_id'])
-                    queryset_all = queryset_all.filter(up_branch__release = prj.release).exclude(pk__in = map(lambda x: x.layercommit_id, prj.projectlayer_set.all()))
-                queryset_all = queryset_all.filter(layer__name__icontains=request.GET.get('value',''))
+                queryset_all = Layer_Version.objects.filter(layer__name__icontains=request.GET.get('value',''))
+                queryset_all = queryset_all.filter(up_branch__name= prj.release.branch_name).exclude(pk__in = [x.id for x in reduce(lambda x, y: list(x) + list(y), map(lambda x: x.layercommit.get_equivalents_wpriority(prj), prj.projectlayer_set.all()))])
+
+                queryset_all = set([x.get_equivalents_wpriority(prj)[0] for x in queryset_all[:8]])
+
                 return HttpResponse(jsonfilter( { "error":"ok",
-                    "list" : map( lambda x: {"id": x.pk, "name": x.layer.name, "detail": "(" + x.layer.layer_source.name + (")" if x.up_branch == None else " | "+x.up_branch.name+")")},
-                            queryset_all[:8])
+                    "list" : map( lambda x: {"id": x.pk, "name": "%s" % (x.layer.name, ), "detail": "(" + x.layer.vcs_url + (")" if x.up_branch == None else " | "+x.up_branch.name+")")},
+                            queryset_all)
                     }), content_type = "application/json")
 
+
             # returns layer dependencies for a layer, excluding current project layers
             if request.GET['type'] == "layerdeps":
                 queryset_all = LayerVersionDependency.objects.filter(layer_version_id = request.GET['value'])
-
-                if 'project_id' in request.session:
-                    prj = Project.objects.get(pk = request.session['project_id'])
-                    queryset_all = queryset_all.exclude(depends_on__in = map(lambda x: x.layercommit, prj.projectlayer_set.all()))
-
+                queryset_all = queryset_all.exclude(depends_on__in = reduce(lambda x, y: list(x) + list(y), map(lambda x: x.layercommit.get_equivalents_wpriority(prj), prj.projectlayer_set.all())))
                 queryset_all.order_by("-up_id");
 
                 return HttpResponse(jsonfilter( { "error":"ok",
                     "list" : map(
                         lambda x: {"id": x.pk, "name": x.layer.name, "detail": "(" + x.layer.layer_source.name + (")" if x.up_branch == None else " | "+x.up_branch.name+")"),
-                                   "layerdetailurl" : reverse('layerdetails', args=(x.pk,))},
-                        map(lambda x: x.depends_on, queryset_all))
+                                   "giturl": x.layer.vcs_url, "layerdetailurl" : reverse('layerdetails', args=(x.pk,))},
+                        map(lambda x: x.depends_on.get_equivalents_wpriority(prj)[0], queryset_all))
                     }), content_type = "application/json")
 
+
             # returns layer versions that would be deleted on the new release__pk
             if request.GET['type'] == "versionlayers":
                 if not 'project_id' in request.session:
                     raise Exception("This call cannot makes no sense outside a project context")
 
                 retval = []
-                prj = Project.objects.get(pk = request.session['project_id'])
                 for i in prj.projectlayer_set.all():
-                    lv = Layer_Version.objects.filter(layer__name = i.layercommit.layer.name, up_branch__release__pk=request.GET['value'])
-                    if lv.count() != 1:     # there is no layer_version with the new release id, and the same name
+                    lv = Layer_Version.objects.filter(layer__name = i.layercommit.layer.name, up_branch__name = Release.objects.get(pk=request.GET['value']).branch_name)
+                    if lv.count() < 1:     # there is no layer_version with the new release id, and the same name
                         retval.append(i)
 
                 return HttpResponse(jsonfilter( {"error":"ok",
@@ -2153,11 +2164,11 @@ if toastermain.settings.MANAGED:
                         lambda x: {"id": x.layercommit.pk, "name": x.layercommit.layer.name, "detail": "(" + x.layercommit.layer.layer_source.name + (")" if x.layercommit.up_branch == None else " | "+x.layercommit.up_branch.name+")")},
                         retval) }), content_type = "application/json")
 
+
             # returns targets provided by current project layers
             if request.GET['type'] == "targets":
                 queryset_all = Recipe.objects.all()
-                if 'project_id' in request.session:
-                    queryset_all = queryset_all.filter(layer_version__layer__in = map(lambda x: x.layercommit.layer, ProjectLayer.objects.filter(project_id=request.session['project_id'])))
+                queryset_all = queryset_all.filter(layer_version__in = reduce(lambda x, y: list(x) + list(y), map(lambda x: x.layercommit.get_equivalents_wpriority(prj), ProjectLayer.objects.filter(project = prj))))
                 return HttpResponse(jsonfilter({ "error":"ok",
                     "list" : map ( lambda x: {"id": x.pk, "name": x.name, "detail":"[" + x.layer_version.layer.name+ (" | " + x.layer_version.up_branch.name + "]" if x.layer_version.up_branch is not None else "]")},
                         queryset_all.filter(name__icontains=request.GET.get('value',''))[:8]),
@@ -2208,13 +2219,14 @@ if toastermain.settings.MANAGED:
         queryset_all = Layer_Version.objects.all()
 
         prj = Project.objects.get(pk = request.session['project_id'])
-        queryset_all = queryset_all.filter(up_branch__release = prj.release)
+        queryset_all = queryset_all.filter(up_branch__name = prj.release.branch_name)
 
-        queryset_with_search = _get_queryset(Layer_Version, queryset_all, None, search_term, ordering_string, '-layer__name')
-        queryset = _get_queryset(Layer_Version, queryset_all, filter_string, search_term, ordering_string, '-layer__name')
+        queryset_all = _get_queryset(Layer_Version, queryset_all, filter_string, search_term, ordering_string, '-layer__name')
+
+        objects_all= list(set([x.get_equivalents_wpriority(prj)[0] for x in queryset_all[:pagesize]]))
 
         # retrieve the objects that will be displayed in the table; layers a paginator and gets a page range to display
-        layer_info = _build_page_range(Paginator(queryset, request.GET.get('count', 10)),request.GET.get('page', 1))
+        layer_info = _build_page_range(Paginator(objects_all, request.GET.get('count', 10)),request.GET.get('page', 1))
 
 
         context = {
@@ -2222,7 +2234,7 @@ if toastermain.settings.MANAGED:
             'objects' : layer_info,
             'objectname' : "layers",
             'default_orderby' : 'layer__name:+',
-            'total_count': queryset_with_search.count(),
+            'total_count': queryset_all.count(),
 
             'tablecols' : [
                 {   'name': 'Layer',
@@ -2241,7 +2253,7 @@ if toastermain.settings.MANAGED:
                     'filter': {
                         'class': 'layer',
                         'label': 'Show:',
-                        'options': map(lambda x: (x.name + " layers", 'layer_source__pk:' + str(x.id), queryset_with_search.filter(layer_source__pk = x.id).count() ), LayerSource.objects.all()),
+                        'options': map(lambda x: (x.name + " layers", 'layer_source__pk:' + str(x.id), queryset_all.filter(layer_source__pk = x.id).count() ), LayerSource.objects.all()),
                     }
                 },
                 {   'name': 'Git repository URL',
@@ -2269,8 +2281,8 @@ if toastermain.settings.MANAGED:
                         'class': 'add-del-layers',
                         'label': 'Show:',
                         'options': [
-              ('Layers added to this project', "projectlayer__project:" + str(prj.id), queryset_with_search.filter(projectlayer__project = prj.id).count()),
-              ('Layers not added to this project', "projectlayer__project:NOT" + str(prj.id), queryset_with_search.exclude(projectlayer__project = prj.id).count()),
+              ('Layers added to this project', "projectlayer__project:" + str(prj.id), queryset_all.filter(projectlayer__project = prj.id).count()),
+              ('Layers not added to this project', "projectlayer__project:NOT" + str(prj.id), queryset_all.exclude(projectlayer__project = prj.id).count()),
                                    ]
 
                     }
@@ -2300,7 +2312,7 @@ if toastermain.settings.MANAGED:
         (filter_string, search_term, ordering_string) = _search_tuple(request, Recipe)
 
         prj = Project.objects.get(pk = request.session['project_id'])
-        queryset_all = Recipe.objects.filter(Q(layer_version__up_branch__release = prj.release) | Q(layer_version__build__in = prj.build_set.all()))
+        queryset_all = Recipe.objects.filter(Q(layer_version__up_branch__name= prj.release.name) | Q(layer_version__build__in = prj.build_set.all()))
 
         queryset_with_search = _get_queryset(Recipe, queryset_all, None, search_term, ordering_string, '-name')
 
@@ -2404,7 +2416,7 @@ if toastermain.settings.MANAGED:
 
         queryset_all = Machine.objects.all()
 #        if 'project_id' in request.session:
-#            queryset_all = queryset_all.filter(Q(layer_version__up_branch__release = Project.objects.get(request.session['project_id']).release) | Q(layer_version__build__in = Project.objects.get(request.session['project_id']).build_set.all()))
+#            queryset_all = queryset_all.filter(Q(layer_version__up_branch__name = Project.objects.get(request.session['project_id']).release.branch_name) | Q(layer_version__build__in = Project.objects.get(request.session['project_id']).build_set.all()))
 
         queryset_with_search = _get_queryset(Machine, queryset_all, None, search_term, ordering_string, '-name')
         queryset = _get_queryset(Machine, queryset_all, filter_string, search_term, ordering_string, '-name')