summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py85
-rw-r--r--bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py23
-rw-r--r--bitbake/lib/toaster/orm/models.py123
-rw-r--r--bitbake/lib/toaster/toastergui/templates/builddashboard.html15
-rwxr-xr-xbitbake/lib/toaster/toastergui/views.py7
5 files changed, 221 insertions, 32 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 8bdc9cc0a7..a5b22379aa 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\
37django.setup() 37django.setup()
38 38
39from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText 39from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
40from orm.models import Target_Image_File, BuildArtifact 40from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile
41from orm.models import Variable, VariableHistory 41from orm.models import Variable, VariableHistory
42from orm.models import Package, Package_File, Target_Installed_Package, Target_File 42from orm.models import Package, Package_File, Target_Installed_Package, Target_File
43from orm.models import Task_Dependency, Package_Dependency 43from orm.models import Task_Dependency, Package_Dependency
@@ -121,6 +121,13 @@ class ORMWrapper(object):
121 121
122 return vars(self)[dictname][key] 122 return vars(self)[dictname][key]
123 123
124 def get_similar_target_with_image_files(self, target):
125 """
126 Get a Target object "similar" to target; i.e. with the same target
127 name ('core-image-minimal' etc.) and machine.
128 """
129 return target.get_similar_target_with_image_files()
130
124 def _timestamp_to_datetime(self, secs): 131 def _timestamp_to_datetime(self, secs):
125 """ 132 """
126 Convert timestamp in seconds to Python datetime 133 Convert timestamp in seconds to Python datetime
@@ -678,27 +685,32 @@ class ORMWrapper(object):
678 file_name = file_name, 685 file_name = file_name,
679 file_size = file_size) 686 file_size = file_size)
680 687
681 def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size): 688 def save_target_artifact_file(self, target_obj, file_name, file_size):
682 """ 689 """
683 Save artifact information without checking for duplicate paths; 690 Save artifact file information for a Target target_obj.
684 this is used when we are saving data about an artifact which was 691
685 generated by a previous build but which is also relevant to this build, 692 Note that this doesn't include SDK artifacts, only images and
686 e.g. a bzImage file. 693 related files (e.g. bzImage).
687 """ 694 """
688 BuildArtifact.objects.create(build=build_obj, file_name=file_name, 695 TargetArtifactFile.objects.create(target=target_obj,
689 file_size=file_size) 696 file_name=file_name, file_size=file_size)
690 697
691 def save_artifact_information(self, build_obj, file_name, file_size): 698 def save_artifact_information(self, build_obj, file_name, file_size):
692 # we skip the image files from other builds 699 """
693 if Target_Image_File.objects.filter(file_name = file_name).count() > 0: 700 TODO this is currently used to save SDK artifacts to the database,
694 return 701 but will be replaced in future once SDK artifacts are associated with
695 702 Target objects (as they eventually should be)
703 """
696 # do not update artifacts found in other builds 704 # do not update artifacts found in other builds
697 if BuildArtifact.objects.filter(file_name = file_name).count() > 0: 705 if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
698 return 706 return
699 707
700 self.save_artifact_information_no_dedupe(self, build_obj, file_name, 708 # do not update artifact if already a target artifact file for that path
701 file_size) 709 if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0:
710 return
711
712 BuildArtifact.objects.create(build=build_obj, file_name=file_name,
713 file_size=file_size)
702 714
703 def create_logmessage(self, log_information): 715 def create_logmessage(self, log_information):
704 assert 'build' in log_information 716 assert 'build' in log_information
@@ -1496,7 +1508,7 @@ class BuildInfoHelper(object):
1496 1508
1497 self.orm_wrapper.create_logmessage(log_information) 1509 self.orm_wrapper.create_logmessage(log_information)
1498 1510
1499 def _get_files_from_image_license(self, image_license_manifest_path): 1511 def _get_filenames_from_image_license(self, image_license_manifest_path):
1500 """ 1512 """
1501 Find the FILES line in the image_license.manifest file, 1513 Find the FILES line in the image_license.manifest file,
1502 which has the basenames of the bzImage and modules files 1514 which has the basenames of the bzImage and modules files
@@ -1567,19 +1579,20 @@ class BuildInfoHelper(object):
1567 1579
1568 OR 1580 OR
1569 1581
1570 2. There are no files for the target, so copy them from a 1582 2. There are no new files for the target (they were already produced by
1571 previous build with the same target + machine. 1583 a previous build), so copy them from the most recent previous build with
1584 the same target, task and machine.
1572 """ 1585 """
1573 deploy_dir_image = \ 1586 deploy_dir_image = \
1574 self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0] 1587 self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
1575 1588
1576 # if there's no DEPLOY_DIR_IMAGE, there aren't going to be 1589 # if there's no DEPLOY_DIR_IMAGE, there aren't going to be
1577 # any build artifacts, so we can return immediately 1590 # any image artifacts, so we can return immediately
1578 if not deploy_dir_image: 1591 if not deploy_dir_image:
1579 return 1592 return
1580 1593
1581 buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0] 1594 buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
1582 machine = self.server.runCommand(['getVariable', 'MACHINE'])[0] 1595 machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
1583 image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0] 1596 image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
1584 1597
1585 # location of the image_license.manifest files for this build; 1598 # location of the image_license.manifest files for this build;
@@ -1597,7 +1610,10 @@ class BuildInfoHelper(object):
1597 image_file_extensions_unique = set(image_file_extensions.split(' ')) 1610 image_file_extensions_unique = set(image_file_extensions.split(' '))
1598 1611
1599 targets = self.internal_state['targets'] 1612 targets = self.internal_state['targets']
1613
1614 # filter out anything which isn't an image target
1600 image_targets = [target for target in targets if target.is_image] 1615 image_targets = [target for target in targets if target.is_image]
1616
1601 for target in image_targets: 1617 for target in image_targets:
1602 # this is set to True if we find at least one file relating to 1618 # this is set to True if we find at least one file relating to
1603 # this target; if this remains False after the scan, we copy the 1619 # this target; if this remains False after the scan, we copy the
@@ -1625,16 +1641,17 @@ class BuildInfoHelper(object):
1625 if os.path.isfile(image_license_manifest_path): 1641 if os.path.isfile(image_license_manifest_path):
1626 has_files = True 1642 has_files = True
1627 1643
1628 basenames = self._get_files_from_image_license( 1644 basenames = self._get_filenames_from_image_license(
1629 image_license_manifest_path) 1645 image_license_manifest_path)
1630 1646
1631 for basename in basenames: 1647 for basename in basenames:
1632 artifact_path = os.path.join(deploy_dir_image, basename) 1648 artifact_path = os.path.join(deploy_dir_image, basename)
1633 artifact_size = os.stat(artifact_path).st_size 1649 artifact_size = os.stat(artifact_path).st_size
1634 1650
1635 self.orm_wrapper.save_artifact_information_no_dedupe( 1651 # note that the artifact will only be saved against this
1636 self.internal_state['build'], artifact_path, 1652 # build if it hasn't been already
1637 artifact_size) 1653 self.orm_wrapper.save_target_artifact_file(target,
1654 artifact_path, artifact_size)
1638 1655
1639 # store the license manifest path on the target 1656 # store the license manifest path on the target
1640 # (this file is also created any time an image file is created) 1657 # (this file is also created any time an image file is created)
@@ -1648,7 +1665,10 @@ class BuildInfoHelper(object):
1648 # (via real_image_name); note that we don't have to set 1665 # (via real_image_name); note that we don't have to set
1649 # has_files = True, as searching for the license manifest file 1666 # has_files = True, as searching for the license manifest file
1650 # will already have set it to true if at least one image file was 1667 # will already have set it to true if at least one image file was
1651 # produced 1668 # produced; note that the real_image_name includes BUILDNAME, which
1669 # in turn includes a timestamp; so if no files were produced for
1670 # this timestamp (i.e. the build reused existing image files already
1671 # in the directory), no files will be recorded against this target
1652 image_files = self._get_image_files(deploy_dir_image, 1672 image_files = self._get_image_files(deploy_dir_image,
1653 real_image_name, image_file_extensions_unique) 1673 real_image_name, image_file_extensions_unique)
1654 1674
@@ -1657,11 +1677,18 @@ class BuildInfoHelper(object):
1657 target, image_file['path'], image_file['size']) 1677 target, image_file['path'], image_file['size'])
1658 1678
1659 if not has_files: 1679 if not has_files:
1660 # TODO copy artifact and image files from the 1680 # copy image files and build artifacts from the
1661 # most-recently-built Target with the same target + machine 1681 # most-recently-built Target with the
1662 # as this Target; also copy the license manifest path, 1682 # same target + machine as this Target; also copy the license
1663 # as that is treated differently 1683 # manifest path, as that is not treated as an artifact and needs
1664 pass 1684 # to be set separately
1685 most_recent = \
1686 self.orm_wrapper.get_similar_target_with_image_files(target)
1687
1688 if most_recent:
1689 logger.info('image artifacts for target %s cloned from ' \
1690 'target %s' % (target.pk, most_recent.pk))
1691 target.clone_artifacts_from(most_recent)
1665 1692
1666 def close(self, errorcode): 1693 def close(self, errorcode):
1667 if self.brbe is not None: 1694 if self.brbe is not None:
diff --git a/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py b/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py
new file mode 100644
index 0000000000..9f1d9bf4ec
--- /dev/null
+++ b/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py
@@ -0,0 +1,23 @@
1# -*- coding: utf-8 -*-
2from __future__ import unicode_literals
3
4from django.db import migrations, models
5
6
7class Migration(migrations.Migration):
8
9 dependencies = [
10 ('orm', '0007_auto_20160523_1446'),
11 ]
12
13 operations = [
14 migrations.CreateModel(
15 name='TargetArtifactFile',
16 fields=[
17 ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')),
18 ('file_name', models.FilePathField()),
19 ('file_size', models.IntegerField()),
20 ('target', models.ForeignKey(to='orm.Target')),
21 ],
22 ),
23 ]
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
index 40cdb9e5c5..9edbef3396 100644
--- a/bitbake/lib/toaster/orm/models.py
+++ b/bitbake/lib/toaster/orm/models.py
@@ -22,7 +22,7 @@
22from __future__ import unicode_literals 22from __future__ import unicode_literals
23 23
24from django.db import models, IntegrityError 24from django.db import models, IntegrityError
25from django.db.models import F, Q, Avg, Max, Sum 25from django.db.models import F, Q, Avg, Max, Sum, Count
26from django.utils import timezone 26from django.utils import timezone
27from django.utils.encoding import force_bytes 27from django.utils.encoding import force_bytes
28 28
@@ -438,7 +438,9 @@ class Build(models.Model):
438 438
439 def get_image_file_extensions(self): 439 def get_image_file_extensions(self):
440 """ 440 """
441 Get list of file name extensions for images produced by this build 441 Get list of file name extensions for images produced by this build;
442 note that this is the actual list of extensions stored on Target objects
443 for this build, and not the value of IMAGE_FSTYPES.
442 """ 444 """
443 extensions = [] 445 extensions = []
444 446
@@ -458,6 +460,15 @@ class Build(models.Model):
458 460
459 return ', '.join(extensions) 461 return ', '.join(extensions)
460 462
463 def get_image_fstypes(self):
464 """
465 Get the IMAGE_FSTYPES variable value for this build as a de-duplicated
466 list of image file suffixes.
467 """
468 image_fstypes = Variable.objects.get(
469 build=self, variable_name='IMAGE_FSTYPES').variable_value
470 return list(set(re.split(r' {1,}', image_fstypes)))
471
461 def get_sorted_target_list(self): 472 def get_sorted_target_list(self):
462 tgts = Target.objects.filter(build_id = self.id).order_by( 'target' ); 473 tgts = Target.objects.filter(build_id = self.id).order_by( 'target' );
463 return( tgts ); 474 return( tgts );
@@ -612,6 +623,114 @@ class Target(models.Model):
612 def __unicode__(self): 623 def __unicode__(self):
613 return self.target 624 return self.target
614 625
626 def get_similar_targets(self):
627 """
628 Get targets for the same machine, task and target name
629 (e.g. 'core-image-minimal') from a successful build for this project
630 (but excluding this target).
631
632 Note that we look for targets built by this project because projects
633 can have different configurations from each other, and put their
634 artifacts in different directories.
635 """
636 query = ~Q(pk=self.pk) & \
637 Q(target=self.target) & \
638 Q(build__machine=self.build.machine) & \
639 Q(build__outcome=Build.SUCCEEDED) & \
640 Q(build__project=self.build.project)
641
642 return Target.objects.filter(query)
643
644 def get_similar_target_with_image_files(self):
645 """
646 Get the most recent similar target with Target_Image_Files associated
647 with it, for the purpose of cloning those files onto this target.
648 """
649 similar_target = None
650
651 candidates = self.get_similar_targets()
652 if candidates.count() < 1:
653 return similar_target
654
655 task_subquery = Q(task=self.task)
656
657 # we can look for a 'build' task if this task is a 'populate_sdk_ext'
658 # task, as it will have created images; and vice versa; note that
659 # 'build' targets can have their task set to '';
660 # also note that 'populate_sdk' does not produce image files
661 image_tasks = [
662 '', # aka 'build'
663 'build',
664 'populate_sdk_ext'
665 ]
666 if self.task in image_tasks:
667 task_subquery = Q(task__in=image_tasks)
668
669 query = task_subquery & Q(num_files__gt=0)
670
671 # annotate with the count of files, to exclude any targets which
672 # don't have associated files
673 candidates = candidates.annotate(
674 num_files=Count('target_image_file'))
675
676 candidates = candidates.filter(query)
677
678 if candidates.count() > 0:
679 candidates.order_by('build__completed_on')
680 similar_target = candidates.last()
681
682 return similar_target
683
684 def clone_artifacts_from(self, target):
685 """
686 Make clones of the BuildArtifacts, Target_Image_Files and
687 TargetArtifactFile objects associated with Target target, then
688 associate them with this target.
689
690 Note that for Target_Image_Files, we only want files from the previous
691 build whose suffix matches one of the suffixes defined in this
692 target's build's IMAGE_FSTYPES configuration variable. This prevents the
693 Target_Image_File object for an ext4 image being associated with a
694 target for a project which didn't produce an ext4 image (for example).
695
696 Also sets the license_manifest_path of this target to the same path
697 as that of target being cloned from, as the license manifest path is
698 also a build artifact but is treated differently.
699 """
700
701 image_fstypes = self.build.get_image_fstypes()
702
703 # filter out any image files whose suffixes aren't in the
704 # IMAGE_FSTYPES suffixes variable for this target's build
705 image_files = [target_image_file \
706 for target_image_file in target.target_image_file_set.all() \
707 if target_image_file.suffix in image_fstypes]
708
709 for image_file in image_files:
710 image_file.pk = None
711 image_file.target = self
712 image_file.save()
713
714 artifact_files = target.targetartifactfile_set.all()
715 for artifact_file in artifact_files:
716 artifact_file.pk = None
717 artifact_file.target = self
718 artifact_file.save()
719
720 self.license_manifest_path = target.license_manifest_path
721 self.save()
722
723# an Artifact is anything that results from a target being built, and may
724# be of interest to the user, and is not an image file
725class TargetArtifactFile(models.Model):
726 target = models.ForeignKey(Target)
727 file_name = models.FilePathField()
728 file_size = models.IntegerField()
729
730 @property
731 def basename(self):
732 return os.path.basename(self.file_name)
733
615class Target_Image_File(models.Model): 734class Target_Image_File(models.Model):
616 # valid suffixes for image files produced by a build 735 # valid suffixes for image files produced by a build
617 SUFFIXES = { 736 SUFFIXES = {
diff --git a/bitbake/lib/toaster/toastergui/templates/builddashboard.html b/bitbake/lib/toaster/toastergui/templates/builddashboard.html
index dcda2a891f..f6d62b9951 100644
--- a/bitbake/lib/toaster/toastergui/templates/builddashboard.html
+++ b/bitbake/lib/toaster/toastergui/templates/builddashboard.html
@@ -74,7 +74,7 @@
74 {% for target in targets %} 74 {% for target in targets %}
75 {% if target.target.is_image %} 75 {% if target.target.is_image %}
76 <div class="well well-transparent dashboard-section"> 76 <div class="well well-transparent dashboard-section">
77 <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target}}</a></h3> 77 <h3><a href="{% url 'target' build.pk target.target.pk %}">{{target.target.target}}</a></h3>
78 <dl class="dl-horizontal"> 78 <dl class="dl-horizontal">
79 <dt>Packages included</dt> 79 <dt>Packages included</dt>
80 <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd> 80 <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd>
@@ -124,6 +124,19 @@
124 {% endfor %} 124 {% endfor %}
125 </ul> 125 </ul>
126 </dd> 126 </dd>
127 <dt>
128 Kernel artifacts
129 </dt>
130 <dd>
131 <ul class="list-unstyled">
132 {% for artifact in target.target_artifacts|dictsort:"basename" %}
133 <li>
134 <a href="{% url 'build_artifact' build.id 'targetartifactfile' artifact.id %}">{{artifact.basename}}</a>
135 ({{artifact.file_size|filtered_filesizeformat}})
136 </li>
137 {% endfor %}
138 </ul>
139 </dd>
127 </dl> 140 </dl>
128 {% endif %} 141 {% endif %}
129 </div> 142 </div>
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py
index ad85fafb4d..0ec88d9a77 100755
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -31,6 +31,7 @@ from django.shortcuts import render, redirect, get_object_or_404
31from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable 31from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable
32from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency 32from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency
33from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage 33from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage
34from orm.models import TargetArtifactFile
34from orm.models import BitbakeVersion, CustomImageRecipe 35from orm.models import BitbakeVersion, CustomImageRecipe
35from bldcontrol import bbcontroller 36from bldcontrol import bbcontroller
36from django.views.decorators.cache import cache_control 37from django.views.decorators.cache import cache_control
@@ -509,6 +510,8 @@ def builddashboard( request, build_id ):
509 targetHasNoImages = True 510 targetHasNoImages = True
510 elem[ 'imageFiles' ] = imageFiles 511 elem[ 'imageFiles' ] = imageFiles
511 elem[ 'targetHasNoImages' ] = targetHasNoImages 512 elem[ 'targetHasNoImages' ] = targetHasNoImages
513 elem['target_artifacts'] = t.targetartifactfile_set.all()
514
512 targets.append( elem ) 515 targets.append( elem )
513 516
514 ## 517 ##
@@ -2294,6 +2297,10 @@ if True:
2294 elif artifact_type == "buildartifact": 2297 elif artifact_type == "buildartifact":
2295 file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name 2298 file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name
2296 2299
2300 elif artifact_type == "targetartifactfile":
2301 target = TargetArtifactFile.objects.get(pk=artifact_id)
2302 file_name = target.file_name
2303
2297 elif artifact_type == "licensemanifest": 2304 elif artifact_type == "licensemanifest":
2298 file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path 2305 file_name = Target.objects.get(build = build, pk = artifact_id).license_manifest_path
2299 2306