bitbake: toaster: Move Custom image recipe rest api to api file

We now have a dedicated file for the rest API so move and rework for
class based views. Also clean up all flake8 identified warnings.

Remove unused imports from toastergui views.

The original work for this API was done by Elliot Smith, Ed Bartosh,
Michael Wood and Dave Lerner

(Bitbake rev: 37c2b4f105d7334cdd83d9675af787f4327e7fe7)

Signed-off-by: Michael Wood <michael.g.wood@intel.com>
Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 11 insertions, 436 deletions

diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py
index 940ea255fd..365a1e88ff 100755
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -19,43 +19,37 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-# pylint: disable=method-hidden
-# Gives E:848, 4: An attribute defined in json.encoder line 162 hides this method (method-hidden)
-# which is an invalid warning
 
-import operator,re
+import re
 
-from django.db.models import F, Q, Sum, Count, Max
-from django.db import IntegrityError, Error
+from django.db.models import F, Q, Sum
+from django.db import IntegrityError
 from django.shortcuts import render, redirect, get_object_or_404
-from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable
-from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency
-from orm.models import Target_Installed_Package, Target_File, Target_Image_File, CustomImagePackage
-from orm.models import TargetKernelFile, TargetSDKFile
+from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe
+from orm.models import LogMessage, Variable, Package_Dependency, Package
+from orm.models import Task_Dependency, Package_File
+from orm.models import Target_Installed_Package, Target_File
+from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File
 from orm.models import BitbakeVersion, CustomImageRecipe
-from bldcontrol import bbcontroller
-from django.views.decorators.cache import cache_control
+
 from django.core.urlresolvers import reverse, resolve
 from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
-from django.http import HttpResponseBadRequest, HttpResponseNotFound
+from django.http import HttpResponseNotFound
 from django.utils import timezone
-from django.utils.html import escape
 from datetime import timedelta, datetime
-from django.utils import formats
 from toastergui.templatetags.projecttags import json as jsonfilter
 from decimal import Decimal
 import json
 import os
 from os.path import dirname
-from functools import wraps
-import itertools
 import mimetypes
 
 import logging
 
 logger = logging.getLogger("toaster")
 
+
 class MimeTypeFinder(object):
     # setting this to False enables additional non-standard mimetypes
     # to be included in the guess
@@ -1498,18 +1492,6 @@ if True:
 
         return context
 
-    def xhr_response(fun):
-        """
-        Decorator for REST methods.
-        calls jsonfilter on the returned dictionary and returns result
-        as HttpResponse object of content_type application/json
-        """
-        @wraps(fun)
-        def wrapper(*args, **kwds):
-            return HttpResponse(jsonfilter(fun(*args, **kwds)),
-                                content_type="application/json")
-        return wrapper
-
     def jsunittests(request):
         """ Provides a page for the js unit tests """
         bbv = BitbakeVersion.objects.filter(branch="master").first()
@@ -1767,187 +1749,6 @@ if True:
 
         return HttpResponse(jsonfilter(json_response), content_type = "application/json")
 
-    @xhr_response
-    def xhr_customrecipe(request):
-        """
-        Custom image recipe REST API
-
-        Entry point: /xhr_customrecipe/
-        Method: POST
-
-        Args:
-            name: name of custom recipe to create
-            project: target project id of orm.models.Project
-            base: base recipe id of orm.models.Recipe
-
-        Returns:
-            {"error": "ok",
-             "url": <url of the created recipe>}
-            or
-            {"error": <error message>}
-        """
-        # check if request has all required parameters
-        for param in ('name', 'project', 'base'):
-            if param not in request.POST:
-                return {"error": "Missing parameter '%s'" % param}
-
-        # get project and baserecipe objects
-        params = {}
-        for name, model in [("project", Project),
-                            ("base", Recipe)]:
-            value = request.POST[name]
-            try:
-                params[name] = model.objects.get(id=value)
-            except model.DoesNotExist:
-                return {"error": "Invalid %s id %s" % (name, value)}
-
-        # create custom recipe
-        try:
-
-            # Only allowed chars in name are a-z, 0-9 and -
-            if re.search(r'[^a-z|0-9|-]', request.POST["name"]):
-                return {"error": "invalid-name"}
-
-            custom_images = CustomImageRecipe.objects.all()
-
-            # Are there any recipes with this name already in our project?
-            existing_image_recipes_in_project = custom_images.filter(
-                name=request.POST["name"], project=params["project"])
-
-            if existing_image_recipes_in_project.count() > 0:
-                return {"error": "image-already-exists"}
-
-            # Are there any recipes with this name which aren't custom
-            # image recipes?
-            custom_image_ids = custom_images.values_list('id', flat=True)
-            existing_non_image_recipes = Recipe.objects.filter(
-                Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids)
-            )
-
-            if existing_non_image_recipes.count() > 0:
-                return {"error": "recipe-already-exists"}
-
-            # create layer 'Custom layer' and verion if needed
-            layer = Layer.objects.get_or_create(
-                name=CustomImageRecipe.LAYER_NAME,
-                summary="Layer for custom recipes",
-                vcs_url="file:///toaster_created_layer")[0]
-
-            # Check if we have a layer version already
-            # We don't use get_or_create here because the dirpath will change
-            # and is a required field
-            lver = Layer_Version.objects.filter(Q(project=params['project']) &
-                                                Q(layer=layer) &
-                                                Q(build=None)).last()
-            if lver == None:
-                lver, created = Layer_Version.objects.get_or_create(
-                    project=params['project'],
-                    layer=layer,
-                    dirpath="toaster_created_layer")
-
-            # Add a dependency on our layer to the base recipe's layer
-            LayerVersionDependency.objects.get_or_create(
-                layer_version=lver,
-                depends_on=params["base"].layer_version)
-
-            # Add it to our current project if needed
-            ProjectLayer.objects.get_or_create(project=params['project'],
-                                               layercommit=lver,
-                                               optional=False)
-
-            # Create the actual recipe
-            recipe, created = CustomImageRecipe.objects.get_or_create(
-                name=request.POST["name"],
-                base_recipe=params["base"],
-                project=params["project"],
-                layer_version=lver,
-                is_image=True)
-
-            # If we created the object then setup these fields. They may get
-            # overwritten later on and cause the get_or_create to create a
-            # duplicate if they've changed.
-            if created:
-                recipe.file_path = request.POST["name"]
-                recipe.license = "MIT"
-                recipe.version = "0.1"
-                recipe.save()
-
-        except Error as err:
-            return {"error": "Can't create custom recipe: %s" % err}
-
-        # Find the package list from the last build of this recipe/target
-        target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) &
-                                       Q(build__project=params['project']) &
-                                       (Q(target=params['base'].name) |
-                                        Q(target=recipe.name))).last()
-        if target:
-            # Copy in every package
-            # We don't want these packages to be linked to anything because
-            # that underlying data may change e.g. delete a build
-            for tpackage in target.target_installed_package_set.all():
-                try:
-                    built_package = tpackage.package
-                    # The package had no recipe information so is a ghost
-                    # package skip it
-                    if built_package.recipe == None:
-                        continue;
-
-                    config_package = CustomImagePackage.objects.get(
-                        name=built_package.name)
-
-                    recipe.includes_set.add(config_package)
-                except Exception as e:
-                    logger.warning("Error adding package %s %s" %
-                                   (tpackage.package.name, e))
-                    pass
-
-        return {"error": "ok",
-                "packages" : recipe.get_all_packages().count(),
-                "url": reverse('customrecipe', args=(params['project'].pk,
-                                                     recipe.id))}
-
-    @xhr_response
-    def xhr_customrecipe_id(request, recipe_id):
-        """
-        Set of ReST API processors working with recipe id.
-
-        Entry point: /xhr_customrecipe/<recipe_id>
-
-        Methods:
-            GET - Get details of custom image recipe
-            DELETE - Delete custom image recipe
-
-        Returns:
-            GET:
-            {"error": "ok",
-             "info": dictionary of field name -> value pairs
-                     of the CustomImageRecipe model}
-            DELETE:
-            {"error": "ok"}
-            or
-            {"error": <error message>}
-        """
-        try:
-            custom_recipe = CustomImageRecipe.objects.get(id=recipe_id)
-        except CustomImageRecipe.DoesNotExist:
-            return {"error": "Custom recipe with id=%s "
-                             "not found" % recipe_id}
-
-        if request.method == 'GET':
-            info = {"id" : custom_recipe.id,
-                    "name" : custom_recipe.name,
-                    "base_recipe_id": custom_recipe.base_recipe.id,
-                    "project_id": custom_recipe.project.id,
-                   }
-
-            return {"error": "ok", "info": info}
-
-        elif request.method == 'DELETE':
-            custom_recipe.delete()
-            return {"error": "ok"}
-        else:
-            return {"error": "Method %s is not supported" % request.method}
-
     def customrecipe_download(request, pid, recipe_id):
         recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id)
 
@@ -1960,232 +1761,6 @@ if True:
 
         return response
 
-    def _traverse_dependents(next_package_id, rev_deps, all_current_packages, tree_level=0):
-        """
-        Recurse through reverse dependency tree for next_package_id.
-        Limit the reverse dependency search to packages not already scanned,
-        that is, not already in rev_deps.
-        Limit the scan to a depth (tree_level) not exceeding the count of
-        all packages in the custom image, and if that depth is exceeded
-        return False, pop out of the recursion, and write a warning
-        to the log, but this is unlikely, suggesting a dependency loop
-        not caught by bitbake.
-        On return, the input/output arg rev_deps is appended with queryset
-        dictionary elements, annotated for use in the customimage template.
-        The list has unsorted, but unique elements.
-        """
-        max_dependency_tree_depth = all_current_packages.count()
-        if tree_level >= max_dependency_tree_depth:
-            logger.warning(
-                "The number of reverse dependencies "
-                "for this package exceeds " + max_dependency_tree_depth +
-                " and the remaining reverse dependencies will not be removed")
-            return True
-
-        package = CustomImagePackage.objects.get(id=next_package_id)
-        dependents = \
-            package.package_dependencies_target.annotate(
-                name=F('package__name'),
-                pk=F('package__pk'),
-                size=F('package__size'),
-            ).values("name", "pk", "size").exclude(
-                ~Q(pk__in=all_current_packages)
-            )
-
-        for pkg in dependents:
-            if pkg in rev_deps:
-                # already seen, skip dependent search
-                continue
-
-            rev_deps.append(pkg)
-            if (_traverse_dependents(
-                pkg["pk"], rev_deps, all_current_packages, tree_level+1)):
-                return True
-
-        return False
-
-    def _get_all_dependents(package_id, all_current_packages):
-        """
-        Returns sorted list of recursive reverse dependencies for package_id,
-        as a list of dictionary items, by recursing through dependency
-        relationships.
-        """
-        rev_deps = []
-        _traverse_dependents(package_id, rev_deps, all_current_packages)
-        rev_deps = sorted(rev_deps, key=lambda x: x["name"])
-        return rev_deps
-
-    @xhr_response
-    def xhr_customrecipe_packages(request, recipe_id, package_id):
-        """
-        ReST API to add/remove packages to/from custom recipe.
-
-        Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id>
-
-        Methods:
-            PUT - Add package to the recipe
-            DELETE - Delete package from the recipe
-            GET - Get package information
-
-        Returns:
-            {"error": "ok"}
-            or
-            {"error": <error message>}
-        """
-        try:
-            recipe = CustomImageRecipe.objects.get(id=recipe_id)
-        except CustomImageRecipe.DoesNotExist:
-            return {"error": "Custom recipe with id=%s "
-                             "not found" % recipe_id}
-
-        if package_id:
-            try:
-                package = CustomImagePackage.objects.get(id=package_id)
-            except Package.DoesNotExist:
-                return {"error": "Package with id=%s "
-                        "not found" % package_id}
-
-        if request.method == 'GET':
-            # If no package_id then list the current packages
-            if not package_id:
-                total_size = 0
-                packages = recipe.get_all_packages().values("id",
-                                                            "name",
-                                                            "version",
-                                                            "size")
-                for package in packages:
-                    package['size_formatted'] = \
-                            filtered_filesizeformat(package['size'])
-                    total_size += package['size']
-
-                return {"error": "ok",
-                        "packages" : list(packages),
-                        "total" : len(packages),
-                        "total_size" : total_size,
-                        "total_size_formatted" :
-                        filtered_filesizeformat(total_size)
-                       }
-            else:
-                all_current_packages = recipe.get_all_packages()
-
-                # Dependencies for package which aren't satisfied by the
-                # current packages in the custom image recipe
-                deps =\
-                    package.package_dependencies_source.for_target_or_none(
-                        recipe.name)['packages'].annotate(
-                    name=F('depends_on__name'),
-                    pk=F('depends_on__pk'),
-                    size=F('depends_on__size'),
-                ).values("name", "pk", "size").filter(
-                    # There are two depends types we don't know why
-                    (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) |
-                    Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) &
-                    ~Q(pk__in=all_current_packages)
-                )
-
-                # Reverse dependencies which are needed by packages that are
-                # in the image. Recursive search providing all dependents,
-                # not just immediate dependents.
-                reverse_deps = _get_all_dependents(package_id, all_current_packages)
-                total_size_deps = 0
-                total_size_reverse_deps = 0
-
-                for dep in deps:
-                    dep['size_formatted'] = \
-                            filtered_filesizeformat(dep['size'])
-                    total_size_deps += dep['size']
-
-                for dep in reverse_deps:
-                    dep['size_formatted'] = \
-                            filtered_filesizeformat(dep['size'])
-                    total_size_reverse_deps += dep['size']
-
-
-                return {"error": "ok",
-                        "id": package.pk,
-                        "name": package.name,
-                        "version": package.version,
-                        "unsatisfied_dependencies": list(deps),
-                        "unsatisfied_dependencies_size": total_size_deps,
-                        "unsatisfied_dependencies_size_formatted":
-                        filtered_filesizeformat(total_size_deps),
-                        "reverse_dependencies": list(reverse_deps),
-                        "reverse_dependencies_size": total_size_reverse_deps,
-                        "reverse_dependencies_size_formatted":
-                        filtered_filesizeformat(total_size_reverse_deps)}
-
-        included_packages = recipe.includes_set.values_list('pk', flat=True)
-
-        if request.method == 'PUT':
-            # If we're adding back a package which used to be included in this
-            # image all we need to do is remove it from the excludes
-            if package.pk in included_packages:
-                try:
-                   recipe.excludes_set.remove(package)
-                   return {"error": "ok"}
-                except Package.DoesNotExist:
-                   return {"error":
-                           "Package %s not found in excludes but was in "
-                           "included list" % package.name}
-
-            else:
-                recipe.appends_set.add(package)
-                # Make sure that package is not in the excludes set
-                try:
-                    recipe.excludes_set.remove(package)
-                except:
-                    pass
-                # Add the dependencies we think will be added to the recipe
-                # as a result of appending this package.
-                # TODO this should recurse down the entire deps tree
-                for dep in package.package_dependencies_source.all_depends():
-                    try:
-                        cust_package = CustomImagePackage.objects.get(
-                                           name=dep.depends_on.name)
-
-                        recipe.includes_set.add(cust_package)
-                        try:
-                            # When adding the pre-requisite package, make
-                            # sure it's not in the excluded list from a
-                            # prior removal.
-                            recipe.excludes_set.remove(cust_package)
-                        except Package.DoesNotExist:
-                            # Don't care if the package had never been excluded
-                            pass
-                    except:
-                        logger.warning("Could not add package's suggested"
-                                       "dependencies to the list")
-
-            return {"error": "ok"}
-
-        elif request.method == 'DELETE':
-            try:
-                # If we're deleting a package which is included we need to
-                # Add it to the excludes list.
-                if package.pk in included_packages:
-                    recipe.excludes_set.add(package)
-                else:
-                    recipe.appends_set.remove(package)
-                all_current_packages = recipe.get_all_packages()
-                reverse_deps_dictlist = _get_all_dependents(package.pk, all_current_packages)
-                ids = [entry['pk'] for entry in reverse_deps_dictlist]
-                reverse_deps = CustomImagePackage.objects.filter(id__in=ids)
-                for r in reverse_deps:
-                    try:
-                        if r.id in included_packages:
-                            recipe.excludes_set.add(r)
-                        else:
-                            recipe.appends_set.remove(r)
-                    except:
-                        pass
-
-                return {"error": "ok"}
-            except CustomImageRecipe.DoesNotExist:
-                return {"error": "Tried to remove package that wasn't present"}
-
-        else:
-            return {"error": "Method %s is not supported" % request.method}
-
     def importlayer(request, pid):
         template = "importlayer.html"
         context = {