diff options
Diffstat (limited to 'bitbake/lib/toaster/toastergui/views.py')
-rwxr-xr-x | bitbake/lib/toaster/toastergui/views.py | 447 |
1 files changed, 11 insertions, 436 deletions
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 940ea255fd..365a1e88ff 100755 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py | |||
@@ -19,43 +19,37 @@ | |||
19 | # with this program; if not, write to the Free Software Foundation, Inc., | 19 | # with this program; if not, write to the Free Software Foundation, Inc., |
20 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | 20 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
21 | 21 | ||
22 | # pylint: disable=method-hidden | ||
23 | # Gives E:848, 4: An attribute defined in json.encoder line 162 hides this method (method-hidden) | ||
24 | # which is an invalid warning | ||
25 | 22 | ||
26 | import operator,re | 23 | import re |
27 | 24 | ||
28 | from django.db.models import F, Q, Sum, Count, Max | 25 | from django.db.models import F, Q, Sum |
29 | from django.db import IntegrityError, Error | 26 | from django.db import IntegrityError |
30 | from django.shortcuts import render, redirect, get_object_or_404 | 27 | from django.shortcuts import render, redirect, get_object_or_404 |
31 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable | 28 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe |
32 | from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency | 29 | from orm.models import LogMessage, Variable, Package_Dependency, Package |
33 | from orm.models import Target_Installed_Package, Target_File, Target_Image_File, CustomImagePackage | 30 | from orm.models import Task_Dependency, Package_File |
34 | from orm.models import TargetKernelFile, TargetSDKFile | 31 | from orm.models import Target_Installed_Package, Target_File |
32 | from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File | ||
35 | from orm.models import BitbakeVersion, CustomImageRecipe | 33 | from orm.models import BitbakeVersion, CustomImageRecipe |
36 | from bldcontrol import bbcontroller | 34 | |
37 | from django.views.decorators.cache import cache_control | ||
38 | from django.core.urlresolvers import reverse, resolve | 35 | from django.core.urlresolvers import reverse, resolve |
39 | from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist | 36 | from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist |
40 | from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger | 37 | from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger |
41 | from django.http import HttpResponseBadRequest, HttpResponseNotFound | 38 | from django.http import HttpResponseNotFound |
42 | from django.utils import timezone | 39 | from django.utils import timezone |
43 | from django.utils.html import escape | ||
44 | from datetime import timedelta, datetime | 40 | from datetime import timedelta, datetime |
45 | from django.utils import formats | ||
46 | from toastergui.templatetags.projecttags import json as jsonfilter | 41 | from toastergui.templatetags.projecttags import json as jsonfilter |
47 | from decimal import Decimal | 42 | from decimal import Decimal |
48 | import json | 43 | import json |
49 | import os | 44 | import os |
50 | from os.path import dirname | 45 | from os.path import dirname |
51 | from functools import wraps | ||
52 | import itertools | ||
53 | import mimetypes | 46 | import mimetypes |
54 | 47 | ||
55 | import logging | 48 | import logging |
56 | 49 | ||
57 | logger = logging.getLogger("toaster") | 50 | logger = logging.getLogger("toaster") |
58 | 51 | ||
52 | |||
59 | class MimeTypeFinder(object): | 53 | class MimeTypeFinder(object): |
60 | # setting this to False enables additional non-standard mimetypes | 54 | # setting this to False enables additional non-standard mimetypes |
61 | # to be included in the guess | 55 | # to be included in the guess |
@@ -1498,18 +1492,6 @@ if True: | |||
1498 | 1492 | ||
1499 | return context | 1493 | return context |
1500 | 1494 | ||
1501 | def xhr_response(fun): | ||
1502 | """ | ||
1503 | Decorator for REST methods. | ||
1504 | calls jsonfilter on the returned dictionary and returns result | ||
1505 | as HttpResponse object of content_type application/json | ||
1506 | """ | ||
1507 | @wraps(fun) | ||
1508 | def wrapper(*args, **kwds): | ||
1509 | return HttpResponse(jsonfilter(fun(*args, **kwds)), | ||
1510 | content_type="application/json") | ||
1511 | return wrapper | ||
1512 | |||
1513 | def jsunittests(request): | 1495 | def jsunittests(request): |
1514 | """ Provides a page for the js unit tests """ | 1496 | """ Provides a page for the js unit tests """ |
1515 | bbv = BitbakeVersion.objects.filter(branch="master").first() | 1497 | bbv = BitbakeVersion.objects.filter(branch="master").first() |
@@ -1767,187 +1749,6 @@ if True: | |||
1767 | 1749 | ||
1768 | return HttpResponse(jsonfilter(json_response), content_type = "application/json") | 1750 | return HttpResponse(jsonfilter(json_response), content_type = "application/json") |
1769 | 1751 | ||
1770 | @xhr_response | ||
1771 | def xhr_customrecipe(request): | ||
1772 | """ | ||
1773 | Custom image recipe REST API | ||
1774 | |||
1775 | Entry point: /xhr_customrecipe/ | ||
1776 | Method: POST | ||
1777 | |||
1778 | Args: | ||
1779 | name: name of custom recipe to create | ||
1780 | project: target project id of orm.models.Project | ||
1781 | base: base recipe id of orm.models.Recipe | ||
1782 | |||
1783 | Returns: | ||
1784 | {"error": "ok", | ||
1785 | "url": <url of the created recipe>} | ||
1786 | or | ||
1787 | {"error": <error message>} | ||
1788 | """ | ||
1789 | # check if request has all required parameters | ||
1790 | for param in ('name', 'project', 'base'): | ||
1791 | if param not in request.POST: | ||
1792 | return {"error": "Missing parameter '%s'" % param} | ||
1793 | |||
1794 | # get project and baserecipe objects | ||
1795 | params = {} | ||
1796 | for name, model in [("project", Project), | ||
1797 | ("base", Recipe)]: | ||
1798 | value = request.POST[name] | ||
1799 | try: | ||
1800 | params[name] = model.objects.get(id=value) | ||
1801 | except model.DoesNotExist: | ||
1802 | return {"error": "Invalid %s id %s" % (name, value)} | ||
1803 | |||
1804 | # create custom recipe | ||
1805 | try: | ||
1806 | |||
1807 | # Only allowed chars in name are a-z, 0-9 and - | ||
1808 | if re.search(r'[^a-z|0-9|-]', request.POST["name"]): | ||
1809 | return {"error": "invalid-name"} | ||
1810 | |||
1811 | custom_images = CustomImageRecipe.objects.all() | ||
1812 | |||
1813 | # Are there any recipes with this name already in our project? | ||
1814 | existing_image_recipes_in_project = custom_images.filter( | ||
1815 | name=request.POST["name"], project=params["project"]) | ||
1816 | |||
1817 | if existing_image_recipes_in_project.count() > 0: | ||
1818 | return {"error": "image-already-exists"} | ||
1819 | |||
1820 | # Are there any recipes with this name which aren't custom | ||
1821 | # image recipes? | ||
1822 | custom_image_ids = custom_images.values_list('id', flat=True) | ||
1823 | existing_non_image_recipes = Recipe.objects.filter( | ||
1824 | Q(name=request.POST["name"]) & ~Q(pk__in=custom_image_ids) | ||
1825 | ) | ||
1826 | |||
1827 | if existing_non_image_recipes.count() > 0: | ||
1828 | return {"error": "recipe-already-exists"} | ||
1829 | |||
1830 | # create layer 'Custom layer' and verion if needed | ||
1831 | layer = Layer.objects.get_or_create( | ||
1832 | name=CustomImageRecipe.LAYER_NAME, | ||
1833 | summary="Layer for custom recipes", | ||
1834 | vcs_url="file:///toaster_created_layer")[0] | ||
1835 | |||
1836 | # Check if we have a layer version already | ||
1837 | # We don't use get_or_create here because the dirpath will change | ||
1838 | # and is a required field | ||
1839 | lver = Layer_Version.objects.filter(Q(project=params['project']) & | ||
1840 | Q(layer=layer) & | ||
1841 | Q(build=None)).last() | ||
1842 | if lver == None: | ||
1843 | lver, created = Layer_Version.objects.get_or_create( | ||
1844 | project=params['project'], | ||
1845 | layer=layer, | ||
1846 | dirpath="toaster_created_layer") | ||
1847 | |||
1848 | # Add a dependency on our layer to the base recipe's layer | ||
1849 | LayerVersionDependency.objects.get_or_create( | ||
1850 | layer_version=lver, | ||
1851 | depends_on=params["base"].layer_version) | ||
1852 | |||
1853 | # Add it to our current project if needed | ||
1854 | ProjectLayer.objects.get_or_create(project=params['project'], | ||
1855 | layercommit=lver, | ||
1856 | optional=False) | ||
1857 | |||
1858 | # Create the actual recipe | ||
1859 | recipe, created = CustomImageRecipe.objects.get_or_create( | ||
1860 | name=request.POST["name"], | ||
1861 | base_recipe=params["base"], | ||
1862 | project=params["project"], | ||
1863 | layer_version=lver, | ||
1864 | is_image=True) | ||
1865 | |||
1866 | # If we created the object then setup these fields. They may get | ||
1867 | # overwritten later on and cause the get_or_create to create a | ||
1868 | # duplicate if they've changed. | ||
1869 | if created: | ||
1870 | recipe.file_path = request.POST["name"] | ||
1871 | recipe.license = "MIT" | ||
1872 | recipe.version = "0.1" | ||
1873 | recipe.save() | ||
1874 | |||
1875 | except Error as err: | ||
1876 | return {"error": "Can't create custom recipe: %s" % err} | ||
1877 | |||
1878 | # Find the package list from the last build of this recipe/target | ||
1879 | target = Target.objects.filter(Q(build__outcome=Build.SUCCEEDED) & | ||
1880 | Q(build__project=params['project']) & | ||
1881 | (Q(target=params['base'].name) | | ||
1882 | Q(target=recipe.name))).last() | ||
1883 | if target: | ||
1884 | # Copy in every package | ||
1885 | # We don't want these packages to be linked to anything because | ||
1886 | # that underlying data may change e.g. delete a build | ||
1887 | for tpackage in target.target_installed_package_set.all(): | ||
1888 | try: | ||
1889 | built_package = tpackage.package | ||
1890 | # The package had no recipe information so is a ghost | ||
1891 | # package skip it | ||
1892 | if built_package.recipe == None: | ||
1893 | continue; | ||
1894 | |||
1895 | config_package = CustomImagePackage.objects.get( | ||
1896 | name=built_package.name) | ||
1897 | |||
1898 | recipe.includes_set.add(config_package) | ||
1899 | except Exception as e: | ||
1900 | logger.warning("Error adding package %s %s" % | ||
1901 | (tpackage.package.name, e)) | ||
1902 | pass | ||
1903 | |||
1904 | return {"error": "ok", | ||
1905 | "packages" : recipe.get_all_packages().count(), | ||
1906 | "url": reverse('customrecipe', args=(params['project'].pk, | ||
1907 | recipe.id))} | ||
1908 | |||
1909 | @xhr_response | ||
1910 | def xhr_customrecipe_id(request, recipe_id): | ||
1911 | """ | ||
1912 | Set of ReST API processors working with recipe id. | ||
1913 | |||
1914 | Entry point: /xhr_customrecipe/<recipe_id> | ||
1915 | |||
1916 | Methods: | ||
1917 | GET - Get details of custom image recipe | ||
1918 | DELETE - Delete custom image recipe | ||
1919 | |||
1920 | Returns: | ||
1921 | GET: | ||
1922 | {"error": "ok", | ||
1923 | "info": dictionary of field name -> value pairs | ||
1924 | of the CustomImageRecipe model} | ||
1925 | DELETE: | ||
1926 | {"error": "ok"} | ||
1927 | or | ||
1928 | {"error": <error message>} | ||
1929 | """ | ||
1930 | try: | ||
1931 | custom_recipe = CustomImageRecipe.objects.get(id=recipe_id) | ||
1932 | except CustomImageRecipe.DoesNotExist: | ||
1933 | return {"error": "Custom recipe with id=%s " | ||
1934 | "not found" % recipe_id} | ||
1935 | |||
1936 | if request.method == 'GET': | ||
1937 | info = {"id" : custom_recipe.id, | ||
1938 | "name" : custom_recipe.name, | ||
1939 | "base_recipe_id": custom_recipe.base_recipe.id, | ||
1940 | "project_id": custom_recipe.project.id, | ||
1941 | } | ||
1942 | |||
1943 | return {"error": "ok", "info": info} | ||
1944 | |||
1945 | elif request.method == 'DELETE': | ||
1946 | custom_recipe.delete() | ||
1947 | return {"error": "ok"} | ||
1948 | else: | ||
1949 | return {"error": "Method %s is not supported" % request.method} | ||
1950 | |||
1951 | def customrecipe_download(request, pid, recipe_id): | 1752 | def customrecipe_download(request, pid, recipe_id): |
1952 | recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) | 1753 | recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id) |
1953 | 1754 | ||
@@ -1960,232 +1761,6 @@ if True: | |||
1960 | 1761 | ||
1961 | return response | 1762 | return response |
1962 | 1763 | ||
1963 | def _traverse_dependents(next_package_id, rev_deps, all_current_packages, tree_level=0): | ||
1964 | """ | ||
1965 | Recurse through reverse dependency tree for next_package_id. | ||
1966 | Limit the reverse dependency search to packages not already scanned, | ||
1967 | that is, not already in rev_deps. | ||
1968 | Limit the scan to a depth (tree_level) not exceeding the count of | ||
1969 | all packages in the custom image, and if that depth is exceeded | ||
1970 | return False, pop out of the recursion, and write a warning | ||
1971 | to the log, but this is unlikely, suggesting a dependency loop | ||
1972 | not caught by bitbake. | ||
1973 | On return, the input/output arg rev_deps is appended with queryset | ||
1974 | dictionary elements, annotated for use in the customimage template. | ||
1975 | The list has unsorted, but unique elements. | ||
1976 | """ | ||
1977 | max_dependency_tree_depth = all_current_packages.count() | ||
1978 | if tree_level >= max_dependency_tree_depth: | ||
1979 | logger.warning( | ||
1980 | "The number of reverse dependencies " | ||
1981 | "for this package exceeds " + max_dependency_tree_depth + | ||
1982 | " and the remaining reverse dependencies will not be removed") | ||
1983 | return True | ||
1984 | |||
1985 | package = CustomImagePackage.objects.get(id=next_package_id) | ||
1986 | dependents = \ | ||
1987 | package.package_dependencies_target.annotate( | ||
1988 | name=F('package__name'), | ||
1989 | pk=F('package__pk'), | ||
1990 | size=F('package__size'), | ||
1991 | ).values("name", "pk", "size").exclude( | ||
1992 | ~Q(pk__in=all_current_packages) | ||
1993 | ) | ||
1994 | |||
1995 | for pkg in dependents: | ||
1996 | if pkg in rev_deps: | ||
1997 | # already seen, skip dependent search | ||
1998 | continue | ||
1999 | |||
2000 | rev_deps.append(pkg) | ||
2001 | if (_traverse_dependents( | ||
2002 | pkg["pk"], rev_deps, all_current_packages, tree_level+1)): | ||
2003 | return True | ||
2004 | |||
2005 | return False | ||
2006 | |||
2007 | def _get_all_dependents(package_id, all_current_packages): | ||
2008 | """ | ||
2009 | Returns sorted list of recursive reverse dependencies for package_id, | ||
2010 | as a list of dictionary items, by recursing through dependency | ||
2011 | relationships. | ||
2012 | """ | ||
2013 | rev_deps = [] | ||
2014 | _traverse_dependents(package_id, rev_deps, all_current_packages) | ||
2015 | rev_deps = sorted(rev_deps, key=lambda x: x["name"]) | ||
2016 | return rev_deps | ||
2017 | |||
2018 | @xhr_response | ||
2019 | def xhr_customrecipe_packages(request, recipe_id, package_id): | ||
2020 | """ | ||
2021 | ReST API to add/remove packages to/from custom recipe. | ||
2022 | |||
2023 | Entry point: /xhr_customrecipe/<recipe_id>/packages/<package_id> | ||
2024 | |||
2025 | Methods: | ||
2026 | PUT - Add package to the recipe | ||
2027 | DELETE - Delete package from the recipe | ||
2028 | GET - Get package information | ||
2029 | |||
2030 | Returns: | ||
2031 | {"error": "ok"} | ||
2032 | or | ||
2033 | {"error": <error message>} | ||
2034 | """ | ||
2035 | try: | ||
2036 | recipe = CustomImageRecipe.objects.get(id=recipe_id) | ||
2037 | except CustomImageRecipe.DoesNotExist: | ||
2038 | return {"error": "Custom recipe with id=%s " | ||
2039 | "not found" % recipe_id} | ||
2040 | |||
2041 | if package_id: | ||
2042 | try: | ||
2043 | package = CustomImagePackage.objects.get(id=package_id) | ||
2044 | except Package.DoesNotExist: | ||
2045 | return {"error": "Package with id=%s " | ||
2046 | "not found" % package_id} | ||
2047 | |||
2048 | if request.method == 'GET': | ||
2049 | # If no package_id then list the current packages | ||
2050 | if not package_id: | ||
2051 | total_size = 0 | ||
2052 | packages = recipe.get_all_packages().values("id", | ||
2053 | "name", | ||
2054 | "version", | ||
2055 | "size") | ||
2056 | for package in packages: | ||
2057 | package['size_formatted'] = \ | ||
2058 | filtered_filesizeformat(package['size']) | ||
2059 | total_size += package['size'] | ||
2060 | |||
2061 | return {"error": "ok", | ||
2062 | "packages" : list(packages), | ||
2063 | "total" : len(packages), | ||
2064 | "total_size" : total_size, | ||
2065 | "total_size_formatted" : | ||
2066 | filtered_filesizeformat(total_size) | ||
2067 | } | ||
2068 | else: | ||
2069 | all_current_packages = recipe.get_all_packages() | ||
2070 | |||
2071 | # Dependencies for package which aren't satisfied by the | ||
2072 | # current packages in the custom image recipe | ||
2073 | deps =\ | ||
2074 | package.package_dependencies_source.for_target_or_none( | ||
2075 | recipe.name)['packages'].annotate( | ||
2076 | name=F('depends_on__name'), | ||
2077 | pk=F('depends_on__pk'), | ||
2078 | size=F('depends_on__size'), | ||
2079 | ).values("name", "pk", "size").filter( | ||
2080 | # There are two depends types we don't know why | ||
2081 | (Q(dep_type=Package_Dependency.TYPE_TRDEPENDS) | | ||
2082 | Q(dep_type=Package_Dependency.TYPE_RDEPENDS)) & | ||
2083 | ~Q(pk__in=all_current_packages) | ||
2084 | ) | ||
2085 | |||
2086 | # Reverse dependencies which are needed by packages that are | ||
2087 | # in the image. Recursive search providing all dependents, | ||
2088 | # not just immediate dependents. | ||
2089 | reverse_deps = _get_all_dependents(package_id, all_current_packages) | ||
2090 | total_size_deps = 0 | ||
2091 | total_size_reverse_deps = 0 | ||
2092 | |||
2093 | for dep in deps: | ||
2094 | dep['size_formatted'] = \ | ||
2095 | filtered_filesizeformat(dep['size']) | ||
2096 | total_size_deps += dep['size'] | ||
2097 | |||
2098 | for dep in reverse_deps: | ||
2099 | dep['size_formatted'] = \ | ||
2100 | filtered_filesizeformat(dep['size']) | ||
2101 | total_size_reverse_deps += dep['size'] | ||
2102 | |||
2103 | |||
2104 | return {"error": "ok", | ||
2105 | "id": package.pk, | ||
2106 | "name": package.name, | ||
2107 | "version": package.version, | ||
2108 | "unsatisfied_dependencies": list(deps), | ||
2109 | "unsatisfied_dependencies_size": total_size_deps, | ||
2110 | "unsatisfied_dependencies_size_formatted": | ||
2111 | filtered_filesizeformat(total_size_deps), | ||
2112 | "reverse_dependencies": list(reverse_deps), | ||
2113 | "reverse_dependencies_size": total_size_reverse_deps, | ||
2114 | "reverse_dependencies_size_formatted": | ||
2115 | filtered_filesizeformat(total_size_reverse_deps)} | ||
2116 | |||
2117 | included_packages = recipe.includes_set.values_list('pk', flat=True) | ||
2118 | |||
2119 | if request.method == 'PUT': | ||
2120 | # If we're adding back a package which used to be included in this | ||
2121 | # image all we need to do is remove it from the excludes | ||
2122 | if package.pk in included_packages: | ||
2123 | try: | ||
2124 | recipe.excludes_set.remove(package) | ||
2125 | return {"error": "ok"} | ||
2126 | except Package.DoesNotExist: | ||
2127 | return {"error": | ||
2128 | "Package %s not found in excludes but was in " | ||
2129 | "included list" % package.name} | ||
2130 | |||
2131 | else: | ||
2132 | recipe.appends_set.add(package) | ||
2133 | # Make sure that package is not in the excludes set | ||
2134 | try: | ||
2135 | recipe.excludes_set.remove(package) | ||
2136 | except: | ||
2137 | pass | ||
2138 | # Add the dependencies we think will be added to the recipe | ||
2139 | # as a result of appending this package. | ||
2140 | # TODO this should recurse down the entire deps tree | ||
2141 | for dep in package.package_dependencies_source.all_depends(): | ||
2142 | try: | ||
2143 | cust_package = CustomImagePackage.objects.get( | ||
2144 | name=dep.depends_on.name) | ||
2145 | |||
2146 | recipe.includes_set.add(cust_package) | ||
2147 | try: | ||
2148 | # When adding the pre-requisite package, make | ||
2149 | # sure it's not in the excluded list from a | ||
2150 | # prior removal. | ||
2151 | recipe.excludes_set.remove(cust_package) | ||
2152 | except Package.DoesNotExist: | ||
2153 | # Don't care if the package had never been excluded | ||
2154 | pass | ||
2155 | except: | ||
2156 | logger.warning("Could not add package's suggested" | ||
2157 | "dependencies to the list") | ||
2158 | |||
2159 | return {"error": "ok"} | ||
2160 | |||
2161 | elif request.method == 'DELETE': | ||
2162 | try: | ||
2163 | # If we're deleting a package which is included we need to | ||
2164 | # Add it to the excludes list. | ||
2165 | if package.pk in included_packages: | ||
2166 | recipe.excludes_set.add(package) | ||
2167 | else: | ||
2168 | recipe.appends_set.remove(package) | ||
2169 | all_current_packages = recipe.get_all_packages() | ||
2170 | reverse_deps_dictlist = _get_all_dependents(package.pk, all_current_packages) | ||
2171 | ids = [entry['pk'] for entry in reverse_deps_dictlist] | ||
2172 | reverse_deps = CustomImagePackage.objects.filter(id__in=ids) | ||
2173 | for r in reverse_deps: | ||
2174 | try: | ||
2175 | if r.id in included_packages: | ||
2176 | recipe.excludes_set.add(r) | ||
2177 | else: | ||
2178 | recipe.appends_set.remove(r) | ||
2179 | except: | ||
2180 | pass | ||
2181 | |||
2182 | return {"error": "ok"} | ||
2183 | except CustomImageRecipe.DoesNotExist: | ||
2184 | return {"error": "Tried to remove package that wasn't present"} | ||
2185 | |||
2186 | else: | ||
2187 | return {"error": "Method %s is not supported" % request.method} | ||
2188 | |||
2189 | def importlayer(request, pid): | 1764 | def importlayer(request, pid): |
2190 | template = "importlayer.html" | 1765 | template = "importlayer.html" |
2191 | context = { | 1766 | context = { |