diff options
author | Elliot Smith <elliot.smith@intel.com> | 2016-07-12 15:54:48 -0700 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2016-07-19 08:56:51 +0100 |
commit | 00c2c0be5ead435601a21a676401674b7045f6f0 (patch) | |
tree | 32295fe980f44dfae3353ed7bf58691aee356ecb /bitbake/lib/toaster | |
parent | f39ae146eadf92f650ed6340158e780a65d483b1 (diff) | |
download | poky-00c2c0be5ead435601a21a676401674b7045f6f0.tar.gz |
bitbake: toaster: improve scan for SDK artifacts
SDK artifacts were previously picked up by toaster.bbclass and
notified to buildinfohelper (via toasterui). The artifacts
were then added to the Build object, so that it wasn't clear
which artifact went with which target; we were also unable
to attach SDK artifacts to a Build if they had already been
attached to a previous build.
Now, toaster.bbclass just notifies the TOOLCHAIN_OUTPUTNAME when
a populate_sdk* target completes. The scan is moved to buildinfohelper,
where we search the SDK deploy directory for files matching
TOOLCHAIN_OUTPUTNAME and attach them to targets (not builds).
If an SDK file is not produced by a target, we now look for a
similar, previously-run target which did produce artifacts.
If there is one, we clone the SDK artifacts from that target
onto the current one.
This all means that we can show SDK artifacts by target, and should
always get artifacts associated with a target, regardless of whether
it really build them.
This requires an additional model, TargetSDKFile, which tracks
the size and path of SDK artifact files with respect to Target
objects.
[YOCTO #8556]
(Bitbake rev: 5e650c611605507e1e0d1588cd5eb6535c2d34fc)
Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: bavery <brian.avery@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster')
-rw-r--r-- | bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py | 39 | ||||
-rw-r--r-- | bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py | 23 | ||||
-rw-r--r-- | bitbake/lib/toaster/orm/models.py | 112 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastergui/templates/builddashboard.html | 64 | ||||
-rwxr-xr-x | bitbake/lib/toaster/toastergui/views.py | 19 |
5 files changed, 162 insertions, 95 deletions
diff --git a/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py b/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py new file mode 100644 index 0000000000..3367582a81 --- /dev/null +++ b/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py | |||
@@ -0,0 +1,39 @@ | |||
1 | # -*- coding: utf-8 -*- | ||
2 | from __future__ import unicode_literals | ||
3 | |||
4 | from django.db import migrations, models | ||
5 | |||
6 | |||
7 | class Migration(migrations.Migration): | ||
8 | |||
9 | dependencies = [ | ||
10 | ('orm', '0007_auto_20160523_1446'), | ||
11 | ] | ||
12 | |||
13 | operations = [ | ||
14 | migrations.CreateModel( | ||
15 | name='TargetKernelFile', | ||
16 | fields=[ | ||
17 | ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)), | ||
18 | ('file_name', models.FilePathField()), | ||
19 | ('file_size', models.IntegerField()), | ||
20 | ('target', models.ForeignKey(to='orm.Target')), | ||
21 | ], | ||
22 | ), | ||
23 | migrations.CreateModel( | ||
24 | name='TargetSDKFile', | ||
25 | fields=[ | ||
26 | ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)), | ||
27 | ('file_name', models.FilePathField()), | ||
28 | ('file_size', models.IntegerField()), | ||
29 | ('target', models.ForeignKey(to='orm.Target')), | ||
30 | ], | ||
31 | ), | ||
32 | migrations.RemoveField( | ||
33 | model_name='buildartifact', | ||
34 | name='build', | ||
35 | ), | ||
36 | migrations.DeleteModel( | ||
37 | name='BuildArtifact', | ||
38 | ), | ||
39 | ] | ||
diff --git a/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py b/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py deleted file mode 100644 index 9f1d9bf4ec..0000000000 --- a/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py +++ /dev/null | |||
@@ -1,23 +0,0 @@ | |||
1 | # -*- coding: utf-8 -*- | ||
2 | from __future__ import unicode_literals | ||
3 | |||
4 | from django.db import migrations, models | ||
5 | |||
6 | |||
7 | class Migration(migrations.Migration): | ||
8 | |||
9 | dependencies = [ | ||
10 | ('orm', '0007_auto_20160523_1446'), | ||
11 | ] | ||
12 | |||
13 | operations = [ | ||
14 | migrations.CreateModel( | ||
15 | name='TargetArtifactFile', | ||
16 | fields=[ | ||
17 | ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')), | ||
18 | ('file_name', models.FilePathField()), | ||
19 | ('file_size', models.IntegerField()), | ||
20 | ('target', models.ForeignKey(to='orm.Target')), | ||
21 | ], | ||
22 | ), | ||
23 | ] | ||
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py index 9edbef3396..61f6a2072e 100644 --- a/bitbake/lib/toaster/orm/models.py +++ b/bitbake/lib/toaster/orm/models.py | |||
@@ -581,28 +581,6 @@ class Build(models.Model): | |||
581 | def __str__(self): | 581 | def __str__(self): |
582 | return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) | 582 | return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) |
583 | 583 | ||
584 | |||
585 | # an Artifact is anything that results from a Build, and may be of interest to the user, and is not stored elsewhere | ||
586 | class BuildArtifact(models.Model): | ||
587 | build = models.ForeignKey(Build) | ||
588 | file_name = models.FilePathField() | ||
589 | file_size = models.IntegerField() | ||
590 | |||
591 | def get_local_file_name(self): | ||
592 | try: | ||
593 | deploydir = Variable.objects.get(build = self.build, variable_name="DEPLOY_DIR").variable_value | ||
594 | return self.file_name[len(deploydir)+1:] | ||
595 | except: | ||
596 | raise | ||
597 | |||
598 | return self.file_name | ||
599 | |||
600 | def get_basename(self): | ||
601 | return os.path.basename(self.file_name) | ||
602 | |||
603 | def is_available(self): | ||
604 | return self.build.buildrequest.environment.has_artifact(self.file_name) | ||
605 | |||
606 | class ProjectTarget(models.Model): | 584 | class ProjectTarget(models.Model): |
607 | project = models.ForeignKey(Project) | 585 | project = models.ForeignKey(Project) |
608 | target = models.CharField(max_length=100) | 586 | target = models.CharField(max_length=100) |
@@ -625,13 +603,19 @@ class Target(models.Model): | |||
625 | 603 | ||
626 | def get_similar_targets(self): | 604 | def get_similar_targets(self): |
627 | """ | 605 | """ |
628 | Get targets for the same machine, task and target name | 606 | Get target sfor the same machine, task and target name |
629 | (e.g. 'core-image-minimal') from a successful build for this project | 607 | (e.g. 'core-image-minimal') from a successful build for this project |
630 | (but excluding this target). | 608 | (but excluding this target). |
631 | 609 | ||
632 | Note that we look for targets built by this project because projects | 610 | Note that we only look for targets built by this project because |
633 | can have different configurations from each other, and put their | 611 | projects can have different configurations from each other, and put |
634 | artifacts in different directories. | 612 | their artifacts in different directories. |
613 | |||
614 | The possibility of error when retrieving candidate targets | ||
615 | is minimised by the fact that bitbake will rebuild artifacts if MACHINE | ||
616 | (or various other variables) change. In this case, there is no need to | ||
617 | clone artifacts from another target, as those artifacts will have | ||
618 | been re-generated for this target anyway. | ||
635 | """ | 619 | """ |
636 | query = ~Q(pk=self.pk) & \ | 620 | query = ~Q(pk=self.pk) & \ |
637 | Q(target=self.target) & \ | 621 | Q(target=self.target) & \ |
@@ -649,29 +633,54 @@ class Target(models.Model): | |||
649 | similar_target = None | 633 | similar_target = None |
650 | 634 | ||
651 | candidates = self.get_similar_targets() | 635 | candidates = self.get_similar_targets() |
652 | if candidates.count() < 1: | 636 | if candidates.count() == 0: |
653 | return similar_target | 637 | return similar_target |
654 | 638 | ||
655 | task_subquery = Q(task=self.task) | 639 | task_subquery = Q(task=self.task) |
656 | 640 | ||
657 | # we can look for a 'build' task if this task is a 'populate_sdk_ext' | 641 | # we can look for a 'build' task if this task is a 'populate_sdk_ext' |
658 | # task, as it will have created images; and vice versa; note that | 642 | # task, as the latter also creates images; and vice versa; note that |
659 | # 'build' targets can have their task set to ''; | 643 | # 'build' targets can have their task set to ''; |
660 | # also note that 'populate_sdk' does not produce image files | 644 | # also note that 'populate_sdk' does not produce image files |
661 | image_tasks = [ | 645 | image_tasks = [ |
662 | '', # aka 'build' | 646 | '', # aka 'build' |
663 | 'build', | 647 | 'build', |
648 | 'image', | ||
664 | 'populate_sdk_ext' | 649 | 'populate_sdk_ext' |
665 | ] | 650 | ] |
666 | if self.task in image_tasks: | 651 | if self.task in image_tasks: |
667 | task_subquery = Q(task__in=image_tasks) | 652 | task_subquery = Q(task__in=image_tasks) |
668 | 653 | ||
654 | # annotate with the count of files, to exclude any targets which | ||
655 | # don't have associated files | ||
656 | candidates = candidates.annotate(num_files=Count('target_image_file')) | ||
657 | |||
669 | query = task_subquery & Q(num_files__gt=0) | 658 | query = task_subquery & Q(num_files__gt=0) |
670 | 659 | ||
660 | candidates = candidates.filter(query) | ||
661 | |||
662 | if candidates.count() > 0: | ||
663 | candidates.order_by('build__completed_on') | ||
664 | similar_target = candidates.last() | ||
665 | |||
666 | return similar_target | ||
667 | |||
668 | def get_similar_target_with_sdk_files(self): | ||
669 | """ | ||
670 | Get the most recent similar target with TargetSDKFiles associated | ||
671 | with it, for the purpose of cloning those files onto this target. | ||
672 | """ | ||
673 | similar_target = None | ||
674 | |||
675 | candidates = self.get_similar_targets() | ||
676 | if candidates.count() == 0: | ||
677 | return similar_target | ||
678 | |||
671 | # annotate with the count of files, to exclude any targets which | 679 | # annotate with the count of files, to exclude any targets which |
672 | # don't have associated files | 680 | # don't have associated files |
673 | candidates = candidates.annotate( | 681 | candidates = candidates.annotate(num_files=Count('targetsdkfile')) |
674 | num_files=Count('target_image_file')) | 682 | |
683 | query = Q(task=self.task) & Q(num_files__gt=0) | ||
675 | 684 | ||
676 | candidates = candidates.filter(query) | 685 | candidates = candidates.filter(query) |
677 | 686 | ||
@@ -681,11 +690,10 @@ class Target(models.Model): | |||
681 | 690 | ||
682 | return similar_target | 691 | return similar_target |
683 | 692 | ||
684 | def clone_artifacts_from(self, target): | 693 | def clone_image_artifacts_from(self, target): |
685 | """ | 694 | """ |
686 | Make clones of the BuildArtifacts, Target_Image_Files and | 695 | Make clones of the Target_Image_Files and TargetKernelFile objects |
687 | TargetArtifactFile objects associated with Target target, then | 696 | associated with Target target, then associate them with this target. |
688 | associate them with this target. | ||
689 | 697 | ||
690 | Note that for Target_Image_Files, we only want files from the previous | 698 | Note that for Target_Image_Files, we only want files from the previous |
691 | build whose suffix matches one of the suffixes defined in this | 699 | build whose suffix matches one of the suffixes defined in this |
@@ -711,18 +719,38 @@ class Target(models.Model): | |||
711 | image_file.target = self | 719 | image_file.target = self |
712 | image_file.save() | 720 | image_file.save() |
713 | 721 | ||
714 | artifact_files = target.targetartifactfile_set.all() | 722 | kernel_files = target.targetkernelfile_set.all() |
715 | for artifact_file in artifact_files: | 723 | for kernel_file in kernel_files: |
716 | artifact_file.pk = None | 724 | kernel_file.pk = None |
717 | artifact_file.target = self | 725 | kernel_file.target = self |
718 | artifact_file.save() | 726 | kernel_file.save() |
719 | 727 | ||
720 | self.license_manifest_path = target.license_manifest_path | 728 | self.license_manifest_path = target.license_manifest_path |
721 | self.save() | 729 | self.save() |
722 | 730 | ||
723 | # an Artifact is anything that results from a target being built, and may | 731 | def clone_sdk_artifacts_from(self, target): |
724 | # be of interest to the user, and is not an image file | 732 | """ |
725 | class TargetArtifactFile(models.Model): | 733 | Clone TargetSDKFile objects from target and associate them with this |
734 | target. | ||
735 | """ | ||
736 | sdk_files = target.targetsdkfile_set.all() | ||
737 | for sdk_file in sdk_files: | ||
738 | sdk_file.pk = None | ||
739 | sdk_file.target = self | ||
740 | sdk_file.save() | ||
741 | |||
742 | # kernel artifacts for a target: bzImage and modules* | ||
743 | class TargetKernelFile(models.Model): | ||
744 | target = models.ForeignKey(Target) | ||
745 | file_name = models.FilePathField() | ||
746 | file_size = models.IntegerField() | ||
747 | |||
748 | @property | ||
749 | def basename(self): | ||
750 | return os.path.basename(self.file_name) | ||
751 | |||
752 | # SDK artifacts for a target: sh and manifest files | ||
753 | class TargetSDKFile(models.Model): | ||
726 | target = models.ForeignKey(Target) | 754 | target = models.ForeignKey(Target) |
727 | file_name = models.FilePathField() | 755 | file_name = models.FilePathField() |
728 | file_size = models.IntegerField() | 756 | file_size = models.IntegerField() |
diff --git a/bitbake/lib/toaster/toastergui/templates/builddashboard.html b/bitbake/lib/toaster/toastergui/templates/builddashboard.html index f6d62b9951..e1bde21e99 100644 --- a/bitbake/lib/toaster/toastergui/templates/builddashboard.html +++ b/bitbake/lib/toaster/toastergui/templates/builddashboard.html | |||
@@ -80,29 +80,30 @@ | |||
80 | <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd> | 80 | <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd> |
81 | <dt>Total package size</dt> | 81 | <dt>Total package size</dt> |
82 | <dd>{{target.pkgsz|filtered_filesizeformat}}</dd> | 82 | <dd>{{target.pkgsz|filtered_filesizeformat}}</dd> |
83 | {% if target.targetHasNoImages %} | 83 | </dl> |
84 | </dl> | 84 | {% if target.targetHasNoImages %} |
85 | <div class="row"> | 85 | <div class="row"> |
86 | <div class="col-md-7"> | 86 | <div class="col-md-7"> |
87 | <div class="alert alert-info"> | 87 | <div class="alert alert-info"> |
88 | <p> | 88 | <p> |
89 | <strong>This build did not create any image files</strong> | 89 | <strong>This build did not create any image files</strong> |
90 | </p> | 90 | </p> |
91 | <p> | 91 | <p> |
92 | This is probably because valid image and license manifest | 92 | This is probably because valid image and license manifest |
93 | files from a previous build already exist in your | 93 | files from a previous build already exist in your |
94 | <code>build/tmp/deploy</code> | 94 | <code>build/tmp/deploy</code> |
95 | directory. You can | 95 | directory. You can |
96 | also <a href="{% url 'target' build.pk target.target.pk %}">view the | 96 | also <a href="{% url 'target' build.pk target.target.pk %}">view the |
97 | license manifest information</a> in Toaster. | 97 | license manifest information</a> in Toaster. |
98 | </p> | 98 | </p> |
99 | </div> | ||
100 | </div> | ||
101 | </div> | 99 | </div> |
102 | {% else %} | 100 | </div> |
101 | </div> | ||
102 | {% endif %} | ||
103 | {% if not target.targetHasNoImages %} | ||
104 | <dl class="dl-horizontal"> | ||
103 | <dt> | 105 | <dt> |
104 | <span class="glyphicon glyphicon-question-sign get-help" title="The location in disk of the license manifest, a document listing all packages installed in your image and their licenses"></span> | 106 | <span class="glyphicon glyphicon-question-sign get-help" title="The location in disk of the license manifest, a document listing all packages installed in your image and their licenses"></span> |
105 | |||
106 | License manifest | 107 | License manifest |
107 | </dt> | 108 | </dt> |
108 | <dd> | 109 | <dd> |
@@ -129,15 +130,32 @@ | |||
129 | </dt> | 130 | </dt> |
130 | <dd> | 131 | <dd> |
131 | <ul class="list-unstyled"> | 132 | <ul class="list-unstyled"> |
132 | {% for artifact in target.target_artifacts|dictsort:"basename" %} | 133 | {% for artifact in target.target_kernel_artifacts|dictsort:"basename" %} |
133 | <li> | 134 | <li> |
134 | <a href="{% url 'build_artifact' build.id 'targetartifactfile' artifact.id %}">{{artifact.basename}}</a> | 135 | <a href="{% url 'build_artifact' build.id 'targetkernelartifact' artifact.id %}">{{artifact.basename}}</a> |
135 | ({{artifact.file_size|filtered_filesizeformat}}) | 136 | ({{artifact.file_size|filtered_filesizeformat}}) |
136 | </li> | 137 | </li> |
137 | {% endfor %} | 138 | {% endfor %} |
138 | </ul> | 139 | </ul> |
139 | </dd> | 140 | </dd> |
140 | </dl> | 141 | </dl> |
142 | {% endif %} | ||
143 | {% if target.target_sdk_artifacts_count > 0 %} | ||
144 | <dl class="dl-horizontal"> | ||
145 | <dt> | ||
146 | SDK artifacts | ||
147 | </dt> | ||
148 | <dd> | ||
149 | <ul class="list-unstyled"> | ||
150 | {% for artifact in target.target_sdk_artifacts|dictsort:"basename" %} | ||
151 | <li> | ||
152 | <a href="{% url 'build_artifact' build.id 'targetsdkartifact' artifact.id %}">{{artifact.basename}}</a> | ||
153 | ({{artifact.file_size|filtered_filesizeformat}}) | ||
154 | </li> | ||
155 | {% endfor %} | ||
156 | </ul> | ||
157 | </dd> | ||
158 | </dl> | ||
141 | {% endif %} | 159 | {% endif %} |
142 | </div> | 160 | </div> |
143 | {% endif %} | 161 | {% endif %} |
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 0ec88d9a77..a82a261e0d 100755 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py | |||
@@ -30,8 +30,8 @@ from django.db import IntegrityError, Error | |||
30 | from django.shortcuts import render, redirect, get_object_or_404 | 30 | from django.shortcuts import render, redirect, get_object_or_404 |
31 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable | 31 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable |
32 | from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency | 32 | from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency |
33 | from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage | 33 | from orm.models import Target_Installed_Package, Target_File, Target_Image_File, CustomImagePackage |
34 | from orm.models import TargetArtifactFile | 34 | from orm.models import TargetKernelFile, TargetSDKFile |
35 | from orm.models import BitbakeVersion, CustomImageRecipe | 35 | from orm.models import BitbakeVersion, CustomImageRecipe |
36 | from bldcontrol import bbcontroller | 36 | from bldcontrol import bbcontroller |
37 | from django.views.decorators.cache import cache_control | 37 | from django.views.decorators.cache import cache_control |
@@ -510,7 +510,11 @@ def builddashboard( request, build_id ): | |||
510 | targetHasNoImages = True | 510 | targetHasNoImages = True |
511 | elem[ 'imageFiles' ] = imageFiles | 511 | elem[ 'imageFiles' ] = imageFiles |
512 | elem[ 'targetHasNoImages' ] = targetHasNoImages | 512 | elem[ 'targetHasNoImages' ] = targetHasNoImages |
513 | elem['target_artifacts'] = t.targetartifactfile_set.all() | 513 | elem['target_kernel_artifacts'] = t.targetkernelfile_set.all() |
514 | |||
515 | target_sdk_files = t.targetsdkfile_set.all() | ||
516 | elem['target_sdk_artifacts_count'] = target_sdk_files.count() | ||
517 | elem['target_sdk_artifacts'] = target_sdk_files | ||
514 | 518 | ||
515 | targets.append( elem ) | 519 | targets.append( elem ) |
516 | 520 | ||
@@ -2294,11 +2298,12 @@ if True: | |||
2294 | elif artifact_type == "imagefile": | 2298 | elif artifact_type == "imagefile": |
2295 | file_name = Target_Image_File.objects.get(target__build = build, pk = artifact_id).file_name | 2299 | file_name = Target_Image_File.objects.get(target__build = build, pk = artifact_id).file_name |
2296 | 2300 | ||
2297 | elif artifact_type == "buildartifact": | 2301 | elif artifact_type == "targetkernelartifact": |
2298 | file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name | 2302 | target = TargetKernelFile.objects.get(pk=artifact_id) |
2303 | file_name = target.file_name | ||
2299 | 2304 | ||
2300 | elif artifact_type == "targetartifactfile": | 2305 | elif artifact_type == "targetsdkartifact": |
2301 | target = TargetArtifactFile.objects.get(pk=artifact_id) | 2306 | target = TargetSDKFile.objects.get(pk=artifact_id) |
2302 | file_name = target.file_name | 2307 | file_name = target.file_name |
2303 | 2308 | ||
2304 | elif artifact_type == "licensemanifest": | 2309 | elif artifact_type == "licensemanifest": |