From 1b636173ca88e5ccca1992f9a12367a1189fa674 Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Tue, 7 Jan 2014 13:10:42 +0000 Subject: bitbake: toaster: Toaster GUI, generic search, filter and order This patch implements table searching, filtering and ordering, in a generic mode reusable for all tables. The search operates list of fields defined in the corresponding class for each model, search_allowed_fields. The search expression and filters are sent through GET requests using a QuerySet-like input. The inputs are filtered and validated before usage to prevent inadvertent or malicious use. Filters and table headers are defined in the views for each table, and rendered by generic code which is easily modified for various tables. The Build table and Configuration table are implemented using this framework as an example of how it should be used. [YOCTO #4249] [YOCTO #4254] [YOCTO #4255] [YOCTO #4256] [YOCTO #4257] [YOCTO #4259] [YOCTO #4260] (Bitbake rev: 2ca15117e4bbda38cda07511d0ff317273f91528) Signed-off-by: Alexandru DAMIAN Signed-off-by: Richard Purdie --- bitbake/lib/toaster/orm/models.py | 10 +- .../lib/toaster/toastergui/static/css/default.css | 5 +- .../toastergui/templates/basetable_bottom.html | 13 +- .../toastergui/templates/basetable_top.html | 79 ++-- .../lib/toaster/toastergui/templates/build.html | 129 ++++--- .../toastergui/templates/configuration.html | 63 +++- .../toaster/toastergui/templates/configvars.html | 40 ++ .../toastergui/templates/filtersnippet.html | 19 + .../toaster/toastergui/templatetags/projecttags.py | 9 +- bitbake/lib/toaster/toastergui/urls.py | 1 + bitbake/lib/toaster/toastergui/views.py | 412 +++++++++++++-------- 11 files changed, 499 insertions(+), 281 deletions(-) create mode 100644 bitbake/lib/toaster/toastergui/templates/configvars.html create mode 100644 bitbake/lib/toaster/toastergui/templates/filtersnippet.html (limited to 'bitbake/lib') diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py index b30e405c0e..ff26c7d436 100644 --- a/bitbake/lib/toaster/orm/models.py +++ b/bitbake/lib/toaster/orm/models.py @@ -31,8 +31,8 @@ class Build(models.Model): (IN_PROGRESS, 'In Progress'), ) - search_allowed_fields = ['machine', - 'cooker_log_path'] + search_allowed_fields = ['machine', 'image_fstypes', + 'cooker_log_path', "target__target"] machine = models.CharField(max_length=100) image_fstypes = models.CharField(max_length=100) @@ -102,6 +102,8 @@ class Task(models.Model): (OUTCOME_NA, 'Not Available'), ) + search_allowed_fields = [ "recipe__name", "task_name" ] + build = models.ForeignKey(Build, related_name='task_build') order = models.IntegerField(null=True) task_executed = models.BooleanField(default=False) # True means Executed, False means Prebuilt @@ -217,6 +219,8 @@ class Layer_Version(models.Model): class Variable(models.Model): + search_allowed_fields = ['variable_name', 'variable_value', + 'variablehistory__file_name', "description"] build = models.ForeignKey(Build, related_name='variable_build') variable_name = models.CharField(max_length=100) variable_value = models.TextField(blank=True) @@ -225,7 +229,7 @@ class Variable(models.Model): description = models.TextField(blank=True) class VariableHistory(models.Model): - variable = models.ForeignKey(Variable) + variable = models.ForeignKey(Variable, related_name='vhistory') file_name = models.FilePathField(max_length=255) line_number = models.IntegerField(null=True) operation = models.CharField(max_length=16) diff --git a/bitbake/lib/toaster/toastergui/static/css/default.css b/bitbake/lib/toaster/toastergui/static/css/default.css index 844f6dcd56..53c50043bc 100644 --- a/bitbake/lib/toaster/toastergui/static/css/default.css +++ b/bitbake/lib/toaster/toastergui/static/css/default.css @@ -171,4 +171,7 @@ dd p {line-height:20px;} .tooltip { z-index: 2000 !important; } /* this makes tooltips work inside modal dialogs */ .tooltip code { background-color:transparent; color:#FFFFFF; font-weight:normal; border:none; font-size: 1em; } .manual { margin-top:11px;} -.heading-help { font-size:14px;} \ No newline at end of file +.heading-help { font-size:14px;} + + +.no-results { margin: 10px 0 0; } diff --git a/bitbake/lib/toaster/toastergui/templates/basetable_bottom.html b/bitbake/lib/toaster/toastergui/templates/basetable_bottom.html index 00703fe4c1..3e4b0cc5a4 100644 --- a/bitbake/lib/toaster/toastergui/templates/basetable_bottom.html +++ b/bitbake/lib/toaster/toastergui/templates/basetable_bottom.html @@ -1,3 +1,4 @@ + @@ -8,15 +9,15 @@
    {%if objects.has_previous %} -
  • «
    • +
  • «
    • {%else%}
  • «
    • {%endif%} {% for i in objects.page_range %} - {{i}} + {{i}} {% endfor %} {%if objects.has_next%} -
  • »
    • +
  • »
    • {%else%}
  • »
    • {%endif%} @@ -58,3 +59,9 @@ }); }); + + + {% for tc in tablecols %}{% if tc.filter %}{% with f=tc.filter %} + {% include "filtersnippet.html" %} + {% endwith %}{% endif %} {% endfor %} + diff --git a/bitbake/lib/toaster/toastergui/templates/basetable_top.html b/bitbake/lib/toaster/toastergui/templates/basetable_top.html index b9277b4a3d..34e0cd7210 100644 --- a/bitbake/lib/toaster/toastergui/templates/basetable_top.html +++ b/bitbake/lib/toaster/toastergui/templates/basetable_top.html @@ -21,46 +21,53 @@
    -
    -
    - - -
    -
    - - {% if tablecols %} -
    - -
      - - {% for i in tablecols %} -
    • - -
      • - {% endfor %} -
    -
    - {% endif %} - -
    - - Show rows: - + + +
    +{% if tablecols %} +
    + +
      {% for i in tablecols %} +
    • + +
      • {% endfor %} +
    +
    +{% endif %} +
    + + Show rows: + -
    -
    -
    -
    + +
    +
    + + + + + + {% for tc in tablecols %}{% endfor %} + + + diff --git a/bitbake/lib/toaster/toastergui/templates/build.html b/bitbake/lib/toaster/toastergui/templates/build.html index 43b491d558..eb7e03c951 100644 --- a/bitbake/lib/toaster/toastergui/templates/build.html +++ b/bitbake/lib/toaster/toastergui/templates/build.html @@ -7,70 +7,77 @@ {% block pagecontent %}
    -
    -

    - Recent Builds -

    -
    -{% for build in mru %} -
    -
    -
    - {%if build.outcome == build.SUCCEEDED%}{%elif build.outcome == build.FAILED%}{%else%}{%endif%} - - 1%}title="Targets: {%for target in build.target_set.all%}{{target.target}} {%endfor%}"{%endif%}>{{build.target_set.all.0.target}} {%if build.target_set.all.count > 1%}(+ {{build.target_set.all.count|add:"-1"}}){%endif%} {{build.machine}} ({{build.completed_on|naturaltime}}) - -
    -{%if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %} -
    -{% if build.errors_no %} - {{build.errors_no}} error{{build.errors_no|pluralize}} -{% endif %} -
    -
    -{% if build.warnings_no %} - {{build.warnings_no}} warning{{build.warnings_no|pluralize}} -{% endif %} -
    -
    - Build time: {{ build|timespent }} -
    -{%endif%}{%if build.outcome == build.IN_PROGRESS %} -
    -
    -
    + {%if mru.count > 0%} +
    +

    + Recent Builds +

    +
    + {% for build in mru %} +
    +
    +
    + {%if build.outcome == build.SUCCEEDED%}{%elif build.outcome == build.FAILED%}{%else%}{%endif%} + + 1%}title="Targets: {%for target in build.target_set.all%}{{target.target}} {%endfor%}"{%endif%}>{{build.target_set.all.0.target}} {%if build.target_set.all.count > 1%}(+ {{build.target_set.all.count|add:"-1"}}){%endif%} {{build.machine}} ({{build.completed_on|naturaltime}}) + +
    + {%if build.outcome == build.SUCCEEDED or build.outcome == build.FAILED %} +
    + {% if build.errors_no %} + {{build.errors_no}} error{{build.errors_no|pluralize}} + {% endif %} +
    +
    + {% if build.warnings_no %} + {{build.warnings_no}} warning{{build.warnings_no|pluralize}} + {% endif %}
    +
    + Build time: {{ build|timespent }} +
    + {%endif%}{%if build.outcome == build.IN_PROGRESS %} +
    +
    +
    +
    +
    +
    ETA: in {{build.eta|naturaltime}}
    + {%endif%}
    -
    ETA: in {{build.eta|naturaltime}}
    -{%endif%}
    -
    -{% endfor %} + {% endfor %}{%endif%} - -
    -

    - All builds +
    +

    + {% if request.GET.filter or request.GET.search and objects.ocount > 0 %} + {{objects.ocount}} build{{objects.ocount|pluralize}} found + {%elif objects.ocount == 0%} + No builds + {%else%} + All builds + {%endif%}

    -
    +

    -{% include "basetable_top.html" %} + {% if objects.ocount == 0 %} +
    +
    +
    +
    + + + +
    + +
    +
    -
    - - - - - - - - - - - - +{% else %} +{% include "basetable_top.html" %} + {% for build in objects %} @@ -78,11 +85,11 @@ - - - + + + - + @@ -91,5 +98,7 @@ {% include "basetable_bottom.html" %} - +{% endif %} + + {% endblock %} diff --git a/bitbake/lib/toaster/toastergui/templates/configuration.html b/bitbake/lib/toaster/toastergui/templates/configuration.html index e390a95ff5..467fbd02ad 100644 --- a/bitbake/lib/toaster/toastergui/templates/configuration.html +++ b/bitbake/lib/toaster/toastergui/templates/configuration.html @@ -4,25 +4,54 @@ {% endblock %} {% block buildinfomain %} + +
    +
    +

    Configuration

    +
    +
    -{% include "basetable_top.html" %} + + - - - - - + +
    +

    Build configuration

    +
    +
    BitBake version
    1.19.1
    +
    Build system
    x86_64-linux
    +
    Host distribution
    Ubuntu-12.04
    +
    Target system
    i586-poky-linux
    +
    Machine
    atom-pc
    +
    Distro
    poky
    +
    Distro version
    1.4+snapshot-20130718
    +
    Tune features
    m32 i586
    +
    Target(s)
    core-image-sato
    +
    +

    Layers

    +
    +
    + {%if tc.qhelp%}{%endif%} + {{tc.name}} + {%if tc.filter%}
    + +
    {%endif%} +
    Outcome
    		Target Machine Started on
    		Completed on
    		Failed tasks
    		Errors
    		Warnings
    		Time Log Output
    {%if build.outcome == build.SUCCEEDED%}{%elif build.outcome == build.FAILED%}{%else%}{%endif%}{{build.machine}} {{build.started_on}} {{build.completed_on}}{% if build.errors_no %}{{build.errors_no}} error{{build.errors_no|pluralize}}{%endif%}{% if build.warnings_no %}{{build.warnings_no}} warning{{build.warnings_no|pluralize}}{%endif%}{% query build.task_build outcome=4 order__gt=0 as exectask%}{% if exectask.count == 1 %}{{exectask.0.recipe.name}}.{{exectask.0.task_name}}{% elif exectask.count > 1%}{{exectask.count}}{%endif%}{% if build.errors_no %}{{build.errors_no}} error{{build.errors_no|pluralize}}{%endif%}{% if build.warnings_no %}{{build.warnings_no}} warning{{build.warnings_no|pluralize}}{%endif%} {{build|timespent}}{{build.log}}{{build.cooker_log_path}} {% if build.outcome == 0 %}{% for t in build.target_set.all %}{% if t.is_image %}{{build.image_fstypes}}{% endif %}{% endfor %}{% endif %}
    NameDescriptionDefinition historyValue
    + + + + + + + + + {% for lv in build.layer_version_build.all %} + + + {% endfor %} + +
    LayerLayer branchLayer commitLayer directory
    {{lv.layer.name}} {{lv.branch}}{{lv.commit|slice:":8"}}...{{lv.layer.local_path}}
    + + - {% for variable in objects %} - - - {{variable.variable_name}} - {% if variable.description %}{{variable.description}}{% endif %} - {% for vh in variable.variablehistory_set.all %}{{vh.operation}} in {{vh.file_name}}:{{vh.line_number}}
    {%endfor%} - {{variable.variable_value}} - {% endfor %} - -{% include "basetable_bottom.html" %} + {% endblock %} diff --git a/bitbake/lib/toaster/toastergui/templates/configvars.html b/bitbake/lib/toaster/toastergui/templates/configvars.html new file mode 100644 index 0000000000..8ce04b883d --- /dev/null +++ b/bitbake/lib/toaster/toastergui/templates/configvars.html @@ -0,0 +1,40 @@ +{% extends "basebuildpage.html" %} +{% block localbreadcrumb %} +
  • Configuration
    • +{% endblock %} + +{% block buildinfomain %} + +
    +
    +

    Configuration

    +
    +
    + + + +{% endblock %} diff --git a/bitbake/lib/toaster/toastergui/templates/filtersnippet.html b/bitbake/lib/toaster/toastergui/templates/filtersnippet.html new file mode 100644 index 0000000000..26ff67563e --- /dev/null +++ b/bitbake/lib/toaster/toastergui/templates/filtersnippet.html @@ -0,0 +1,19 @@ + + + diff --git a/bitbake/lib/toaster/toastergui/templatetags/projecttags.py b/bitbake/lib/toaster/toastergui/templatetags/projecttags.py index 1455026754..15a1757b35 100644 --- a/bitbake/lib/toaster/toastergui/templatetags/projecttags.py +++ b/bitbake/lib/toaster/toastergui/templatetags/projecttags.py @@ -16,8 +16,9 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from datetime import datetime +from datetime import datetime, timedelta from django import template +from django.utils import timezone register = template.Library() @@ -42,8 +43,14 @@ def query(qs, **kwargs): @register.filter def divide(value, arg): + if int(arg) == 0: + return -1 return int(value) / int(arg) @register.filter def multiply(value, arg): return int(value) * int(arg) + +@register.assignment_tag +def datecompute(delta, start = timezone.now()): + return start + timedelta(delta) diff --git a/bitbake/lib/toaster/toastergui/urls.py b/bitbake/lib/toaster/toastergui/urls.py index f531eb0137..585578316a 100644 --- a/bitbake/lib/toaster/toastergui/urls.py +++ b/bitbake/lib/toaster/toastergui/urls.py @@ -39,6 +39,7 @@ urlpatterns = patterns('toastergui.views', url(r'^build/(?P\d+)/target/(?P\d+)/packages$', 'tpackage', name='targetpackages'), url(r'^build/(?P\d+)/configuration$', 'configuration', name='configuration'), + url(r'^build/(?P\d+)/configvars$', 'configvars', name='configvars'), url(r'^build/(?P\d+)/buildtime$', 'buildtime', name='buildtime'), url(r'^build/(?P\d+)/cpuusage$', 'cpuusage', name='cpuusage'), url(r'^build/(?P\d+)/diskio$', 'diskio', name='diskio'), diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 7d4d710f83..09da9c2a2e 100644 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py @@ -25,7 +25,10 @@ from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File from orm.models import Target_Installed_Package from django.views.decorators.cache import cache_control from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger - +from django.http import HttpResponseBadRequest +from django.utils import timezone +from datetime import timedelta +from django.utils import formats def _build_page_range(paginator, index = 1): try: @@ -72,6 +75,109 @@ def _redirect_parameters(view, g, mandatory_parameters, *args, **kwargs): return redirect(url + "?%s" % urllib.urlencode(params), *args, **kwargs) +FIELD_SEPARATOR = ":" +VALUE_SEPARATOR = ";" +DESCENDING = "-" + +def __get_q_for_val(name, value): + if "OR" in value: + return reduce(operator.or_, map(lambda x: __get_q_for_val(name, x), [ x for x in value.split("OR") ])) + if "AND" in value: + return reduce(operator.and_, map(lambda x: __get_q_for_val(name, x), [ x for x in value.split("AND") ])) + if value.startswith("NOT"): + kwargs = { name : value.strip("NOT") } + return ~Q(**kwargs) + else: + kwargs = { name : value } + return Q(**kwargs) + +def _get_filtering_query(filter_string): + + search_terms = filter_string.split(FIELD_SEPARATOR) + keys = search_terms[0].split(VALUE_SEPARATOR) + values = search_terms[1].split(VALUE_SEPARATOR) + + querydict = dict(zip(keys, values)) + return reduce(lambda x, y: x & y, map(lambda x: __get_q_for_val(k, querydict[k]),[k for k in querydict])) + +def _get_toggle_order(request, orderkey): + return "%s:-" % orderkey if request.GET.get('orderby', "") == "%s:+" % orderkey else "%s:+" % orderkey + +# we check that the input comes in a valid form that we can recognize +def _validate_input(input, model): + + invalid = None + + if input: + input_list = input.split(FIELD_SEPARATOR) + + # Check we have only one colon + if len(input_list) != 2: + invalid = "We have an invalid number of separators" + return None, invalid + + # Check we have an equal number of terms both sides of the colon + if len(input_list[0].split(VALUE_SEPARATOR)) != len(input_list[1].split(VALUE_SEPARATOR)): + invalid = "Not all arg names got values" + return None, invalid + str(input_list) + + # Check we are looking for a valid field + valid_fields = model._meta.get_all_field_names() + for field in input_list[0].split(VALUE_SEPARATOR): + if not reduce(lambda x, y: x or y, map(lambda x: field.startswith(x), [ x for x in valid_fields ])): + return None, (field, [ x for x in valid_fields ]) + + return input, invalid + +# uses search_allowed_fields in orm/models.py to create a search query +# for these fields with the supplied input text +def _get_search_results(search_term, queryset, model): + search_objects = [] + for st in search_term.split(" "): + q_map = map(lambda x: Q(**{x+'__icontains': st}), + model.search_allowed_fields) + + search_objects.append(reduce(operator.or_, q_map)) + search_object = reduce(operator.and_, search_objects) + queryset = queryset.filter(search_object) + + return queryset + + +# function to extract the search/filter/ordering parameters from the request +# it uses the request and the model to validate input for the filter and orderby values +def _search_tuple(request, model): + ordering_string, invalid = _validate_input(request.GET.get('orderby', ''), model) + if invalid: + raise BaseException("Invalid ordering " + str(invalid)) + + filter_string, invalid = _validate_input(request.GET.get('filter', ''), model) + if invalid: + raise BaseException("Invalid filter " + str(invalid)) + + search_term = request.GET.get('search', '') + return (filter_string, search_term, ordering_string) + + +# returns a lazy-evaluated queryset for a filter/search/order combination +def _get_queryset(model, filter_string, search_term, ordering_string): + if filter_string: + filter_query = _get_filtering_query(filter_string) + queryset = model.objects.filter(filter_query) + else: + queryset = model.objects.all() + + if search_term: + queryset = _get_search_results(search_term, queryset, model) + + if ordering_string and queryset: + column, order = ordering_string.split(':') + if order.lower() == DESCENDING: + queryset = queryset.order_by('-' + column) + else: + queryset = queryset.order_by(column) + + return queryset # shows the "all builds" page def builds(request): @@ -84,16 +190,24 @@ def builds(request): if retval: return _redirect_parameters( 'all-builds', request.GET, mandatory_parameters) - # retrieve the objects that will be displayed in the table - build_info = _build_page_range(Paginator(Build.objects.exclude(outcome = Build.IN_PROGRESS).order_by("-id"), request.GET.get('count', 10)),request.GET.get('page', 1)) + # boilerplate code that takes a request for an object type and returns a queryset + # for that object type. copypasta for all needed table searches + (filter_string, search_term, ordering_string) = _search_tuple(request, Build) + queryset = _get_queryset(Build, filter_string, search_term, ordering_string) + + # retrieve the objects that will be displayed in the table; builds a paginator and gets a page range to display + build_info = _build_page_range(Paginator(queryset.exclude(outcome = Build.IN_PROGRESS), request.GET.get('count', 10)),request.GET.get('page', 1)) - # build view-specific information; this is rendered specifically in the builds page - build_mru = Build.objects.order_by("-started_on")[:3] + # build view-specific information; this is rendered specifically in the builds page, at the top of the page (i.e. Recent builds) + build_mru = Build.objects.filter(completed_on__gte=(timezone.now()-timedelta(hours=24))).order_by("-started_on")[:3] for b in [ x for x in build_mru if x.outcome == Build.IN_PROGRESS ]: tf = Task.objects.filter(build = b) b.completeper = tf.exclude(order__isnull=True).count()*100/tf.count() - from django.utils import timezone - b.eta = timezone.now() + ((timezone.now() - b.started_on)*100/b.completeper) + b.eta = timezone.now() + if b.completeper > 0: + b.eta += ((timezone.now() - b.started_on)*100/b.completeper) + else: + b.eta = 0 # send the data to the template context = { @@ -101,19 +215,78 @@ def builds(request): 'mru' : build_mru, # TODO: common objects for all table views, adapt as needed 'objects' : build_info, + # Specifies the display of columns for the table, appearance in "Edit columns" box, toggling default show/hide, and specifying filters for columns 'tablecols' : [ - {'name': 'Target ', 'clclass': 'target',}, - {'name': 'Machine ', 'clclass': 'machine'}, - {'name': 'Completed on ', 'clclass': 'completed_on'}, - {'name': 'Failed tasks ', 'clclass': 'failed_tasks'}, - {'name': 'Errors ', 'clclass': 'errors_no'}, - {'name': 'Warnings', 'clclass': 'warnings_no'}, - {'name': 'Output ', 'clclass': 'output'}, - {'name': 'Started on ', 'clclass': 'started_on', 'hidden' : 1}, - {'name': 'Time ', 'clclass': 'time', 'hidden' : 1}, - {'name': 'Output', 'clclass': 'output'}, - {'name': 'Log', 'clclass': 'log', 'hidden': 1}, - ]} + {'name': 'Outcome ', # column with a single filter + 'qhelp' : "The outcome tells you if a build completed successfully or failed", # the help button content + 'dclass' : "span2", # indication about column width; comes from the design + 'orderfield': _get_toggle_order(request, "outcome"), # adds ordering by the field value; default ascending unless clicked from ascending into descending + # filter field will set a filter on that column with the specs in the filter description + # the class field in the filter has no relation with clclass; the control different aspects of the UI + # still, it is recommended for the values to be identical for easy tracking in the generated HTML + 'filter' : {'class' : 'outcome', 'label': 'Show only', 'options' : { + 'Successful builds': 'outcome:' + str(Build.SUCCEEDED), # this is the field search expression + 'Failed builds': 'outcome:'+ str(Build.FAILED), + } + } + }, + {'name': 'Target ', # default column, disabled box, with just the name in the list + 'qhelp': "This is the build target(s): one or more recipes or image recipes", + 'orderfield': _get_toggle_order(request, "target__target"), + }, + {'name': 'Machine ', + 'qhelp': "The machine is the hardware for which you are building", + 'dclass': 'span3'}, # a slightly wider column + {'name': 'Started on ', 'clclass': 'started_on', 'hidden' : 1, # this is an unchecked box, which hides the column + 'qhelp': "The date and time you started the build", + 'filter' : {'class' : 'started_on', 'label': 'Show only builds started', 'options' : { + 'Today' : 'started_on__gte:'+timezone.now().strftime("%Y-%m-%d"), + 'Yesterday' : 'started_on__gte:'+(timezone.now()-timedelta(hours=24)).strftime("%Y-%m-%d"), + 'Within one week' : 'started_on__gte:'+(timezone.now()-timedelta(days=7)).strftime("%Y-%m-%d"), + }} + }, + {'name': 'Completed on ', + 'qhelp': "The date and time the build finished", + 'orderfield': _get_toggle_order(request, "completed_on"), + 'filter' : {'class' : 'completed_on', 'label': 'Show only builds completed', 'options' : { + 'Today' : 'completed_on__gte:'+timezone.now().strftime("%Y-%m-%d"), + 'Yesterday' : 'completed_on__gte:'+(timezone.now()-timedelta(hours=24)).strftime("%Y-%m-%d"), + 'Within one week' : 'completed_on__gte:'+(timezone.now()-timedelta(days=7)).strftime("%Y-%m-%d"), + }} + }, + {'name': 'Failed tasks ', 'clclass': 'failed_tasks', # specifing a clclass will enable the checkbox + 'qhelp': "How many tasks failed during the build", + 'filter' : {'class' : 'failed_tasks', 'label': 'Show only ', 'options' : { + 'Builds with failed tasks' : 'task_build__outcome:4', + 'Builds without failed tasks' : 'task_build__outcome:NOT4', + }} + }, + {'name': 'Errors ', 'clclass': 'errors_no', + 'qhelp': "How many errors were encountered during the build (if any)", + 'orderfield': _get_toggle_order(request, "errors_no"), + 'filter' : {'class' : 'errors_no', 'label': 'Show only ', 'options' : { + 'Builds with errors' : 'errors_no__gte:1', + 'Builds without errors' : 'errors_no:0', + }} + }, + {'name': 'Warnings', 'clclass': 'warnings_no', + 'qhelp': "How many warnigns were encountered during the build (if any)", + 'orderfield': _get_toggle_order(request, "warnings_no"), + 'filter' : {'class' : 'warnings_no', 'label': 'Show only ', 'options' : { + 'Builds with warnings' : 'warnings_no__gte:1', + 'Builds without warnings' : 'warnings_no:0', + }} + }, + {'name': 'Time ', 'clclass': 'time', 'hidden' : 1, + 'qhelp': "How long it took the build to finish",}, + {'name': 'Log', + 'dclass': "span4", + 'qhelp': "The location in disk of the build main log file", + 'clclass': 'log', 'hidden': 1}, + {'name': 'Output', 'clclass': 'output', + 'qhelp': "The root file system types produced by the build. You can find them in your /build/tmp/deploy/images/ directory"}, + ] + } return render(request, template, context) @@ -191,8 +364,10 @@ def tasks(request, build_id): retval = _verify_parameters( request.GET, mandatory_parameters ) if retval: return _redirect_parameters( 'tasks', request.GET, mandatory_parameters, build_id = build_id) + (filter_string, search_term, ordering_string) = _search_tuple(request, Task) + queryset = _get_queryset(Task, filter_string, search_term, ordering_string) - tasks = _build_page_range(Paginator(Task.objects.filter(build=build_id, order__gt=0), request.GET.get('count', 100)),request.GET.get('page', 1)) + tasks = _build_page_range(Paginator(queryset.filter(build=build_id, order__gt=0), request.GET.get('count', 100)),request.GET.get('page', 1)) for t in tasks: if t.outcome == Task.OUTCOME_COVERED: @@ -208,8 +383,10 @@ def recipes(request, build_id): retval = _verify_parameters( request.GET, mandatory_parameters ) if retval: return _redirect_parameters( 'recipes', request.GET, mandatory_parameters, build_id = build_id) + (filter_string, search_term, ordering_string) = _search_tuple(request, Recipe) + queryset = _get_queryset(Recipe, filter_string, search_term, ordering_string) - recipes = _build_page_range(Paginator(Recipe.objects.filter(layer_version__id__in=Layer_Version.objects.filter(build=build_id)), request.GET.get('count', 100)),request.GET.get('page', 1)) + recipes = _build_page_range(Paginator(queryset.filter(layer_version__id__in=Layer_Version.objects.filter(build=build_id)), request.GET.get('count', 100)),request.GET.get('page', 1)) context = {'build': Build.objects.filter(pk=build_id)[0], 'objects': recipes, } @@ -218,15 +395,63 @@ def recipes(request, build_id): def configuration(request, build_id): template = 'configuration.html' + context = {'build': Build.objects.filter(pk=build_id)[0]} + return render(request, template, context) + + +def configvars(request, build_id): + template = 'configvars.html' mandatory_parameters = { 'count': 100, 'page' : 1}; retval = _verify_parameters( request.GET, mandatory_parameters ) if retval: - return _redirect_parameters( 'configuration', request.GET, mandatory_parameters, build_id = build_id) + return _redirect_parameters( 'configvars', request.GET, mandatory_parameters, build_id = build_id) + + (filter_string, search_term, ordering_string) = _search_tuple(request, Variable) + queryset = _get_queryset(Variable, filter_string, search_term, ordering_string) + + variables = _build_page_range(Paginator(queryset.filter(build=build_id), request.GET.get('count', 50)), request.GET.get('page', 1)) + + context = { + 'build': Build.objects.filter(pk=build_id)[0], + 'objects' : variables, + # Specifies the display of columns for the table, appearance in "Edit columns" box, toggling default show/hide, and specifying filters for columns + 'tablecols' : [ + {'name': 'Variable ', + 'qhelp': "Base variable expanded name", + 'clclass' : 'variable', + 'dclass' : "span3", + 'orderfield': _get_toggle_order(request, "variable_name"), + }, + {'name': 'Value ', + 'qhelp': "The value assigned to the variable", + 'clclass': 'variable_value', + 'dclass': "span4", + 'orderfield': _get_toggle_order(request, "variable_value"), + }, + {'name': 'Configuration file(s) ', + 'qhelp': "The configuration file(s) that touched the variable value", + 'clclass': 'file', + 'dclass': "span6", + 'orderfield': _get_toggle_order(request, "variable_vhistory__file_name"), + 'filter' : { 'class': 'file', 'label' : 'Show only', 'options' : { + } + } + }, + {'name': 'Description ', + 'qhelp': "A brief explanation of a variable", + 'clclass': 'description', + 'dclass': "span5", + 'orderfield': _get_toggle_order(request, "description"), + 'filter' : { 'class' : 'description', 'label' : 'No', 'options' : { + } + }, + } + ] + } - variables = _build_page_range(Paginator(Variable.objects.filter(build=build_id), 50), request.GET.get('page', 1)) - context = {'build': Build.objects.filter(pk=build_id)[0], 'objects' : variables} return render(request, template, context) + def buildtime(request, build_id): template = "buildtime.html" if Build.objects.filter(pk=build_id).count() == 0 : @@ -263,8 +488,10 @@ def bpackage(request, build_id): retval = _verify_parameters( request.GET, mandatory_parameters ) if retval: return _redirect_parameters( 'packages', request.GET, mandatory_parameters, build_id = build_id) + (filter_string, search_term, ordering_string) = _search_tuple(request, Package) + queryset = _get_queryset(Package, filter_string, search_term, ordering_string) - packages = _build_page_range(Paginator(Package.objects.filter(build = build_id), request.GET.get('count', 100)),request.GET.get('page', 1)) + packages = _build_page_range(Paginator(queryset.filter(build = build_id), request.GET.get('count', 100)),request.GET.get('page', 1)) context = {'build': Build.objects.filter(pk=build_id)[0], 'objects' : packages} return render(request, template, context) @@ -305,139 +532,4 @@ def layer_versions_recipes(request, layerversion_id): return render(request, template, context) -#### API - -import json -from django.core import serializers -from django.http import HttpResponse, HttpResponseBadRequest - - -def model_explorer(request, model_name): - - DESCENDING = 'desc' - response_data = {} - model_mapping = { - 'build': Build, - 'target': Target, - 'task': Task, - 'task_dependency': Task_Dependency, - 'package': Package, - 'layer': Layer, - 'layerversion': Layer_Version, - 'recipe': Recipe, - 'recipe_dependency': Recipe_Dependency, - 'package': Package, - 'package_dependency': Package_Dependency, - 'build_file': Package_File, - 'variable': Variable, - 'logmessage': LogMessage, - } - - if model_name not in model_mapping.keys(): - return HttpResponseBadRequest() - - model = model_mapping[model_name] - - try: - limit = int(request.GET.get('limit', 0)) - except ValueError: - limit = 0 - - try: - offset = int(request.GET.get('offset', 0)) - except ValueError: - offset = 0 - - ordering_string, invalid = _validate_input(request.GET.get('orderby', ''), - model) - if invalid: - return HttpResponseBadRequest() - - filter_string, invalid = _validate_input(request.GET.get('filter', ''), - model) - if invalid: - return HttpResponseBadRequest() - - search_term = request.GET.get('search', '') - - if filter_string: - filter_terms = _get_filtering_terms(filter_string) - try: - queryset = model.objects.filter(**filter_terms) - except ValueError: - queryset = [] - else: - queryset = model.objects.all() - if search_term: - queryset = _get_search_results(search_term, queryset, model) - - if ordering_string and queryset: - column, order = ordering_string.split(':') - if order.lower() == DESCENDING: - queryset = queryset.order_by('-' + column) - else: - queryset = queryset.order_by(column) - - if offset and limit: - queryset = queryset[offset:(offset+limit)] - elif offset: - queryset = queryset[offset:] - elif limit: - queryset = queryset[:limit] - - if queryset: - response_data['count'] = queryset.count() - else: - response_data['count'] = 0 - response_data['list'] = serializers.serialize('json', queryset) -# response_data = serializers.serialize('json', queryset) - - return HttpResponse(json.dumps(response_data), - content_type='application/json') - -def _get_filtering_terms(filter_string): - - search_terms = filter_string.split(":") - keys = search_terms[0].split(',') - values = search_terms[1].split(',') - - return dict(zip(keys, values)) - -def _validate_input(input, model): - - invalid = 0 - - if input: - input_list = input.split(":") - - # Check we have only one colon - if len(input_list) != 2: - invalid = 1 - return None, invalid - - # Check we have an equal number of terms both sides of the colon - if len(input_list[0].split(',')) != len(input_list[1].split(',')): - invalid = 1 - return None, invalid - - # Check we are looking for a valid field - valid_fields = model._meta.get_all_field_names() - for field in input_list[0].split(','): - if field not in valid_fields: - invalid = 1 - return None, invalid - - return input, invalid - -def _get_search_results(search_term, queryset, model): - search_objects = [] - for st in search_term.split(" "): - q_map = map(lambda x: Q(**{x+'__icontains': st}), - model.search_allowed_fields) - - search_objects.append(reduce(operator.or_, q_map)) - search_object = reduce(operator.and_, search_objects) - queryset = queryset.filter(search_object) - - return queryset -- cgit v1.2.3-54-g00ecf