bitbake: toaster: bitbake cooker log saving and downloading

This patch brings in cooker log saving and proper download links.

* toasterui will now write the cooker log file if running in managed
mode
* the BuildRequest has a new state, REQ_ARCHIVE, indicating that the
build is completed, and the artifacts are ready to be grabbed
* the runbuild test execution commands will gather needed artifacts,
and save them to a storage directory selected during Toaster setup.
* the build dashboard, project builds and all builds pages have
permanent links for the cooker log

[YOCTO #7220]
[YOCTO #7206]

(Bitbake rev: fad80e36c9da663b000cdf2cb3c75440c6431d84)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

6 files changed, 118 insertions, 36 deletions

diff --git a/bitbake/lib/toaster/toastergui/templates/build.html b/bitbake/lib/toaster/toastergui/templates/build.html
index e71e38feb9..684ec65884 100644
--- a/bitbake/lib/toaster/toastergui/templates/build.html
+++ b/bitbake/lib/toaster/toastergui/templates/build.html
@@ -40,7 +40,9 @@
         <!-- Table data rows; the order needs to match the order of "tablecols" definitions; and the <td class value needs to match the tablecols clclass value for show/hide buttons to work -->
         {% for build in objects %}
         <tr class="data">
-            <td class="outcome"><a href="{% url "builddashboard" build.id %}">{%if build.outcome == build.SUCCEEDED%}<i class="icon-ok-sign success"></i>{%elif build.outcome == build.FAILED%}<i class="icon-minus-sign error"></i>{%else%}{%endif%}</a></td>
+            <td class="outcome">
+            <a href="{% url "builddashboard" build.id %}">{%if build.outcome == build.SUCCEEDED%}<i class="icon-ok-sign success"></i>{%elif build.outcome == build.FAILED%}<i class="icon-minus-sign error"></i>{%else%}{%endif%}</a> &nbsp;
+            </td>
             <td class="target">{% for t in build.target_set.all %} <a href="{% url "builddashboard" build.id %}"> {{t.target}} </a> <br />{% endfor %}</td>
             <td class="machine"><a href="{% url "builddashboard" build.id %}">{{build.machine}}</a></td>
             <td class="started_on"><a href="{% url "builddashboard" build.id %}">{{build.started_on|date:"d/m/y H:i"}}</a></td>
@@ -61,11 +63,6 @@
             <td class="errors_no">
                 {% if  build.errors_no %}
                     <a class="errors_no error" href="{% url "builddashboard" build.id %}#errors">{{build.errors_no}} error{{build.errors_no|pluralize}}</a>
-                    {% if MANAGED and build.project %}
-                        <a href="{% url 'build_artifact' build.id "cookerlog" build.id %}">
-                            <i class="icon-download-alt" title="" data-original-title="Download build log"></i>
-                        </a>
-                    {% endif %}
                 {%endif%}
             </td>
             <td class="warnings_no">{% if  build.warnings_no %}<a class="warnings_no warning" href="{% url "builddashboard" build.id %}#warnings">{{build.warnings_no}} warning{{build.warnings_no|pluralize}}</a>{%endif%}</td>
diff --git a/bitbake/lib/toaster/toastergui/templates/builddashboard.html b/bitbake/lib/toaster/toastergui/templates/builddashboard.html
index 2458cdb6d1..c0898e291d 100644
--- a/bitbake/lib/toaster/toastergui/templates/builddashboard.html
+++ b/bitbake/lib/toaster/toastergui/templates/builddashboard.html
@@ -36,7 +36,11 @@
 {% endif %}
     <span > <i class="icon-warning-sign yellow"></i><strong><a href="#warnings" class="warning show-warnings"> {{build.warnings_no}} warning{{build.warnings_no|pluralize}}</a></strong></span>
 {% endif %}
-            <span class="pull-right">Build time: <a href="{% url 'buildtime' build.pk %}">{{ build.timespent|sectohms }}</a></span>
+            <span class="pull-right">Build time: <a href="{% url 'buildtime' build.pk %}">{{ build.timespent|sectohms }}</a> &nbsp;
+      {% if MANAGED and build.project %}
+            <a class="btn {%if build.outcome == build.SUCCEEDED%}btn-success{%else%}btn-danger{%endif%} pull-right" href="{% url 'build_artifact' build.id "cookerlog" build.id %}">Download build log</a>
+      {% endif %}
+            </span>
 {%endif%}
     </div>
     {% if build.toaster_exceptions.count > 0 %}
@@ -54,10 +58,7 @@
 <div class="accordion span10 pull-right" id="errors">
   <div class="accordion-group">
     <div class="accordion-heading">
-      {% if MANAGED and build.project %}
-          <a class="btn btn-large pull-right" href="{% url 'build_artifact' build.id "cookerlog" build.id %}" style="margin:15px;">Download build log</a>
-      {% endif %}
-      <a class="accordion-toggle error toggle-errors">
+            <a class="accordion-toggle error toggle-errors">
          <h2 id="error-toggle">
            <i class="icon-minus-sign"></i>
            {{build.errors_no}} error{{build.errors_no|pluralize}}
diff --git a/bitbake/lib/toaster/toastergui/templates/managed_builds.html b/bitbake/lib/toaster/toastergui/templates/managed_builds.html
index 121ad07898..a4db55b967 100644
--- a/bitbake/lib/toaster/toastergui/templates/managed_builds.html
+++ b/bitbake/lib/toaster/toastergui/templates/managed_builds.html
@@ -46,7 +46,14 @@
         <!-- Table data rows; the order needs to match the order of "tablecols" definitions; and the <td class value needs to match the tablecols clclass value for show/hide buttons to work -->
         {% for buildrequest in objects %}{% if buildrequest.build %}  {% with build=buildrequest.build %} {# if we have a build, just display it #}
         <tr class="data">
-            <td class="outcome"><a href="{% url "builddashboard" build.id %}">{%if build.outcome == build.SUCCEEDED%}<i class="icon-ok-sign success"></i>{%elif build.outcome == build.FAILED%}<i class="icon-minus-sign error"></i>{%else%}{%endif%}</a></td>
+            <td class="outcome"><a href="{% url "builddashboard" build.id %}">{%if build.outcome == build.SUCCEEDED%}<i class="icon-ok-sign success"></i>{%elif build.outcome == build.FAILED%}<i class="icon-minus-sign error"></i>{%else%}{%endif%}</a>
+                    {% if build.project %}
+                        &nbsp; <a href="{% url 'build_artifact' build.id "cookerlog" build.id %}">
+                            <i class="icon-download-alt" title="" data-original-title="Download build log"></i>
+                        </a>
+                    {% endif %}
+
+            </td>
             <td class="target">{% for t in build.target_set.all %} <a href="{% url "builddashboard" build.id %}"> {{t.target}} </a> <br />{% endfor %}</td>
             <td class="machine"><a href="{% url "builddashboard" build.id %}">{{build.machine}}</a></td>
             <td class="started_on"><a href="{% url "builddashboard" build.id %}">{{build.started_on|date:"d/m/y H:i"}}</a></td>
diff --git a/bitbake/lib/toaster/toastergui/templates/projectbuilds.html b/bitbake/lib/toaster/toastergui/templates/projectbuilds.html
index 02d166341f..afcf5191b5 100644
--- a/bitbake/lib/toaster/toastergui/templates/projectbuilds.html
+++ b/bitbake/lib/toaster/toastergui/templates/projectbuilds.html
@@ -49,7 +49,14 @@
         <!-- Table data rows; the order needs to match the order of "tablecols" definitions; and the <td class value needs to match the tablecols clclass value for show/hide buttons to work -->
         {% for br in objects %}{% if br.build %}  {% with build=br.build %} {# if we have a build, just display it #}
         <tr class="data">
-            <td class="outcome"><a href="{% url "builddashboard" build.id %}">{%if build.outcome == build.SUCCEEDED%}<i class="icon-ok-sign success"></i>{%elif build.outcome == build.FAILED%}<i class="icon-minus-sign error"></i>{%else%}{%endif%}</a></td>
+            <td class="outcome"><a href="{% url "builddashboard" build.id %}">{%if build.outcome == build.SUCCEEDED%}<i class="icon-ok-sign success"></i>{%elif build.outcome == build.FAILED%}<i class="icon-minus-sign error"></i>{%else%}{%endif%}</a>
+                   {% if build.project %}
+                        &nbsp; <a href="{% url 'build_artifact' build.id "cookerlog" build.id %}">
+                            <i class="icon-download-alt" title="" data-original-title="Download build log"></i>
+                        </a>
+                    {% endif %}
+            </td>
+
             <td class="target">{% for t in build.target_set.all %} <a href="{% url "builddashboard" build.id %}"> {{t.target}} </a> <br />{% endfor %}</td>
             <td class="machine"><a href="{% url "builddashboard" build.id %}">{{build.machine}}</a></td>
             <td class="started_on"><a href="{% url "builddashboard" build.id %}">{{build.started_on|date:"d/m/y H:i"}}</a></td>
@@ -70,11 +77,6 @@
             <td class="errors_no">
                 {% if  build.errors_no %}
                     <a class="errors_no error" href="{% url "builddashboard" build.id %}#errors">{{build.errors_no}} error{{build.errors_no|pluralize}}</a>
-                    {% if MANAGED and build.project %}
-                        <a href="{% url 'build_artifact' build.id "cookerlog" build.id %}">
-                            <i class="icon-download-alt" title="" data-original-title="Download build log"></i>
-                        </a>
-                    {% endif %}
                 {%endif%}
             </td>
             <td class="warnings_no">{% if  build.warnings_no %}<a class="warnings_no warning" href="{% url "builddashboard" build.id %}#warnings">{{build.warnings_no}} warning{{build.warnings_no|pluralize}}</a>{%endif%}</td>
diff --git a/bitbake/lib/toaster/toastergui/templates/unavailable_artifact.html b/bitbake/lib/toaster/toastergui/templates/unavailable_artifact.html
new file mode 100644
index 0000000000..c93d4257db
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/templates/unavailable_artifact.html
@@ -0,0 +1,17 @@
+{% extends "base.html" %}
+{% load projecttags %}
+{% load humanize %}
+{% load static %}
+
+{% block pagecontent %}
+<div class="section">
+</div>
+<div class="row-fluid">
+
+    <div class="alert alert-info">
+    <p class="lead"> The artifact you are seeking to download is not available. We are sorry. You can <a href="javascript:window.history.back()">go back</a>
+    </p>
+    </div>
+</div>
+{% endblock %}
+
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py
index b5ff9b1d53..eb323ec81d 100755
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -2914,9 +2914,6 @@ if toastermain.settings.MANAGED:
         if artifact_type == "imagefile":
             file_name = Target_Image_File.objects.get(target__build = b, pk = artifact_id).file_name
 
-        elif artifact_type == "cookerlog":
-            file_name = b.cooker_log_path
-
         elif artifact_type == "buildartifact":
             file_name = BuildArtifact.objects.get(build = b, pk = artifact_id).file_name
 
@@ -2935,26 +2932,87 @@ if toastermain.settings.MANAGED:
 
 
     def build_artifact(request, build_id, artifact_type, artifact_id):
-        b = Build.objects.get(pk=build_id)
-        if b.buildrequest is None or b.buildrequest.environment is None:
-            raise Exception("Artifact not available for download (missing build request or build environment)")
+        if artifact_type in ["cookerlog"]:
+            # these artifacts are saved after building, so they are on the server itself
+            def _mimetype_for_artifact(path):
+                try:
+                    import magic
+
+                    # fair warning: this is a mess; there are multiple competing and incompatible
+                    # magic modules floating around, so we try some of the most common combinations
+
+                    try:    # we try ubuntu's python-magic 5.4
+                        m = magic.open(magic.MAGIC_MIME_TYPE)
+                        m.load()
+                        return m.file(path)
+                    except AttributeError:
+                        pass
+
+                    try:    # we try python-magic 0.4.6
+                        m = magic.Magic(magic.MAGIC_MIME)
+                        return m.from_file(path)
+                    except AttributeError:
+                        pass
+
+                    try:    # we try pip filemagic 1.6
+                        m = magic.Magic(flags=magic.MAGIC_MIME_TYPE)
+                        return m.id_filename(path)
+                    except AttributeError:
+                        pass
+
+                    return "binary/octet-stream"
+                except ImportError:
+                    return "binary/octet-stream"
+            try:
+                # match code with runbuilds.Command.archive()
+                build_artifact_storage_dir = os.path.join(ToasterSetting.objects.get(name="ARTIFACTS_STORAGE_DIR").value, "%d" % int(build_id))
+                file_name = os.path.join(build_artifact_storage_dir, "cooker_log.txt")
+
+                fsock = open(file_name, "r")
+                content_type=_mimetype_for_artifact(file_name)
 
-        file_name = _file_name_for_artifact(b, artifact_type, artifact_id)
-        fsock = None
-        content_type='application/force-download'
+                response = HttpResponse(fsock, content_type = content_type)
+
+                response['Content-Disposition'] = 'attachment; filename=' + os.path.basename(file_name)
+                return response
+            except IOError:
+                context = {
+                    'build' : Build.objects.get(pk = build_id),
+                }
+                return render(request, "unavailable_artifact.html", context)
 
-        if file_name is None:
-            raise Exception("Could not handle artifact %s id %s" % (artifact_type, artifact_id))
         else:
-            content_type = b.buildrequest.environment.get_artifact_type(file_name)
-            fsock = b.buildrequest.environment.get_artifact(file_name)
-            file_name = os.path.basename(file_name) # we assume that the build environment system has the same path conventions as host
+            # retrieve the artifact directly from the build environment
+            return _get_be_artifact(request, build_id, artifact_type, artifact_id)
 
-        response = HttpResponse(fsock, content_type = content_type)
 
-        # returns a file from the environment
-        response['Content-Disposition'] = 'attachment; filename=' + file_name
-        return response
+    def _get_be_artifact(request, build_id, artifact_type, artifact_id):
+        try:
+            b = Build.objects.get(pk=build_id)
+            if b.buildrequest is None or b.buildrequest.environment is None:
+                raise Exception("Artifact not available for download (missing build request or build environment)")
+
+            file_name = _file_name_for_artifact(b, artifact_type, artifact_id)
+            fsock = None
+            content_type='application/force-download'
+
+            if file_name is None:
+                raise Exception("Could not handle artifact %s id %s" % (artifact_type, artifact_id))
+            else:
+                content_type = b.buildrequest.environment.get_artifact_type(file_name)
+                fsock = b.buildrequest.environment.get_artifact(file_name)
+                file_name = os.path.basename(file_name) # we assume that the build environment system has the same path conventions as host
+
+            response = HttpResponse(fsock, content_type = content_type)
+
+            # returns a file from the environment
+            response['Content-Disposition'] = 'attachment; filename=' + file_name
+            return response
+        except IOError:
+            context = {
+                'build' : Build.objects.get(pk = build_id),
+            }
+            return render(request, "unavailable_artifact.html", context)