diff options
author | Elliot Smith <elliot.smith@intel.com> | 2016-07-12 15:54:48 -0700 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2016-07-19 08:56:51 +0100 |
commit | 00c2c0be5ead435601a21a676401674b7045f6f0 (patch) | |
tree | 32295fe980f44dfae3353ed7bf58691aee356ecb /bitbake/lib | |
parent | f39ae146eadf92f650ed6340158e780a65d483b1 (diff) | |
download | poky-00c2c0be5ead435601a21a676401674b7045f6f0.tar.gz |
bitbake: toaster: improve scan for SDK artifacts
SDK artifacts were previously picked up by toaster.bbclass and
notified to buildinfohelper (via toasterui). The artifacts
were then added to the Build object, so that it wasn't clear
which artifact went with which target; we were also unable
to attach SDK artifacts to a Build if they had already been
attached to a previous build.
Now, toaster.bbclass just notifies the TOOLCHAIN_OUTPUTNAME when
a populate_sdk* target completes. The scan is moved to buildinfohelper,
where we search the SDK deploy directory for files matching
TOOLCHAIN_OUTPUTNAME and attach them to targets (not builds).
If an SDK file is not produced by a target, we now look for a
similar, previously-run target which did produce artifacts.
If there is one, we clone the SDK artifacts from that target
onto the current one.
This all means that we can show SDK artifacts by target, and should
always get artifacts associated with a target, regardless of whether
it really build them.
This requires an additional model, TargetSDKFile, which tracks
the size and path of SDK artifact files with respect to Target
objects.
[YOCTO #8556]
(Bitbake rev: 5e650c611605507e1e0d1588cd5eb6535c2d34fc)
Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: bavery <brian.avery@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r-- | bitbake/lib/bb/ui/buildinfohelper.py | 175 | ||||
-rw-r--r-- | bitbake/lib/bb/ui/toasterui.py | 7 | ||||
-rw-r--r-- | bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py | 39 | ||||
-rw-r--r-- | bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py | 23 | ||||
-rw-r--r-- | bitbake/lib/toaster/orm/models.py | 112 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastergui/templates/builddashboard.html | 64 | ||||
-rwxr-xr-x | bitbake/lib/toaster/toastergui/views.py | 19 |
7 files changed, 300 insertions, 139 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index e1b59c3e87..bf032ae835 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py | |||
@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\ | |||
37 | django.setup() | 37 | django.setup() |
38 | 38 | ||
39 | from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText | 39 | from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText |
40 | from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile | 40 | from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile |
41 | from orm.models import Variable, VariableHistory | 41 | from orm.models import Variable, VariableHistory |
42 | from orm.models import Package, Package_File, Target_Installed_Package, Target_File | 42 | from orm.models import Package, Package_File, Target_Installed_Package, Target_File |
43 | from orm.models import Task_Dependency, Package_Dependency | 43 | from orm.models import Task_Dependency, Package_Dependency |
@@ -128,6 +128,15 @@ class ORMWrapper(object): | |||
128 | """ | 128 | """ |
129 | return target.get_similar_target_with_image_files() | 129 | return target.get_similar_target_with_image_files() |
130 | 130 | ||
131 | def get_similar_target_with_sdk_files(self, target): | ||
132 | return target.get_similar_target_with_sdk_files() | ||
133 | |||
134 | def clone_image_artifacts(self, target_from, target_to): | ||
135 | target_to.clone_image_artifacts_from(target_from) | ||
136 | |||
137 | def clone_sdk_artifacts(self, target_from, target_to): | ||
138 | target_to.clone_sdk_artifacts_from(target_from) | ||
139 | |||
131 | def _timestamp_to_datetime(self, secs): | 140 | def _timestamp_to_datetime(self, secs): |
132 | """ | 141 | """ |
133 | Convert timestamp in seconds to Python datetime | 142 | Convert timestamp in seconds to Python datetime |
@@ -682,35 +691,22 @@ class ORMWrapper(object): | |||
682 | logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) | 691 | logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) |
683 | 692 | ||
684 | def save_target_image_file_information(self, target_obj, file_name, file_size): | 693 | def save_target_image_file_information(self, target_obj, file_name, file_size): |
685 | Target_Image_File.objects.create( target = target_obj, | 694 | Target_Image_File.objects.create(target=target_obj, |
686 | file_name = file_name, | 695 | file_name=file_name, file_size=file_size) |
687 | file_size = file_size) | ||
688 | 696 | ||
689 | def save_target_artifact_file(self, target_obj, file_name, file_size): | 697 | def save_target_kernel_file(self, target_obj, file_name, file_size): |
690 | """ | 698 | """ |
691 | Save artifact file information for a Target target_obj. | 699 | Save kernel file (bzImage, modules*) information for a Target target_obj. |
692 | |||
693 | Note that this doesn't include SDK artifacts, only images and | ||
694 | related files (e.g. bzImage). | ||
695 | """ | 700 | """ |
696 | TargetArtifactFile.objects.create(target=target_obj, | 701 | TargetKernelFile.objects.create(target=target_obj, |
697 | file_name=file_name, file_size=file_size) | 702 | file_name=file_name, file_size=file_size) |
698 | 703 | ||
699 | def save_artifact_information(self, build_obj, file_name, file_size): | 704 | def save_target_sdk_file(self, target_obj, file_name, file_size): |
700 | """ | 705 | """ |
701 | TODO this is currently used to save SDK artifacts to the database, | 706 | Save SDK artifacts to the database, associating them with a |
702 | but will be replaced in future once SDK artifacts are associated with | 707 | Target object. |
703 | Target objects (as they eventually should be) | ||
704 | """ | 708 | """ |
705 | # do not update artifacts found in other builds | 709 | TargetSDKFile.objects.create(target=target_obj, file_name=file_name, |
706 | if BuildArtifact.objects.filter(file_name = file_name).count() > 0: | ||
707 | return | ||
708 | |||
709 | # do not update artifact if already a target artifact file for that path | ||
710 | if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0: | ||
711 | return | ||
712 | |||
713 | BuildArtifact.objects.create(build=build_obj, file_name=file_name, | ||
714 | file_size=file_size) | 710 | file_size=file_size) |
715 | 711 | ||
716 | def create_logmessage(self, log_information): | 712 | def create_logmessage(self, log_information): |
@@ -1085,11 +1081,6 @@ class BuildInfoHelper(object): | |||
1085 | 1081 | ||
1086 | return self.brbe | 1082 | return self.brbe |
1087 | 1083 | ||
1088 | def update_artifact_image_file(self, event): | ||
1089 | evdata = BuildInfoHelper._get_data_from_event(event) | ||
1090 | for artifact_path in evdata.keys(): | ||
1091 | self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) | ||
1092 | |||
1093 | def update_build_information(self, event, errors, warnings, taskfailures): | 1084 | def update_build_information(self, event, errors, warnings, taskfailures): |
1094 | if 'build' in self.internal_state: | 1085 | if 'build' in self.internal_state: |
1095 | self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) | 1086 | self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) |
@@ -1568,9 +1559,9 @@ class BuildInfoHelper(object): | |||
1568 | 1559 | ||
1569 | return image_files | 1560 | return image_files |
1570 | 1561 | ||
1571 | def scan_build_artifacts(self): | 1562 | def scan_image_artifacts(self): |
1572 | """ | 1563 | """ |
1573 | Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them | 1564 | Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them |
1574 | with a Target object in self.internal_state['targets']. | 1565 | with a Target object in self.internal_state['targets']. |
1575 | 1566 | ||
1576 | We have two situations to handle: | 1567 | We have two situations to handle: |
@@ -1615,7 +1606,7 @@ class BuildInfoHelper(object): | |||
1615 | # filter out anything which isn't an image target | 1606 | # filter out anything which isn't an image target |
1616 | image_targets = [target for target in targets if target.is_image] | 1607 | image_targets = [target for target in targets if target.is_image] |
1617 | 1608 | ||
1618 | for target in image_targets: | 1609 | for image_target in image_targets: |
1619 | # this is set to True if we find at least one file relating to | 1610 | # this is set to True if we find at least one file relating to |
1620 | # this target; if this remains False after the scan, we copy the | 1611 | # this target; if this remains False after the scan, we copy the |
1621 | # files from the most-recent Target with the same target + machine | 1612 | # files from the most-recent Target with the same target + machine |
@@ -1627,7 +1618,7 @@ class BuildInfoHelper(object): | |||
1627 | # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; | 1618 | # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; |
1628 | # we need to change it to | 1619 | # we need to change it to |
1629 | # <TARGET>-<MACHINE>-<BUILDNAME> | 1620 | # <TARGET>-<MACHINE>-<BUILDNAME> |
1630 | real_image_name = re.sub(r'^defaultpkgname', target.target, | 1621 | real_image_name = re.sub(r'^defaultpkgname', image_target.target, |
1631 | image_name) | 1622 | image_name) |
1632 | 1623 | ||
1633 | image_license_manifest_path = os.path.join( | 1624 | image_license_manifest_path = os.path.join( |
@@ -1651,7 +1642,7 @@ class BuildInfoHelper(object): | |||
1651 | 1642 | ||
1652 | # note that the artifact will only be saved against this | 1643 | # note that the artifact will only be saved against this |
1653 | # build if it hasn't been already | 1644 | # build if it hasn't been already |
1654 | self.orm_wrapper.save_target_artifact_file(target, | 1645 | self.orm_wrapper.save_target_kernel_file(image_target, |
1655 | artifact_path, artifact_size) | 1646 | artifact_path, artifact_size) |
1656 | 1647 | ||
1657 | # store the license manifest path on the target | 1648 | # store the license manifest path on the target |
@@ -1659,8 +1650,8 @@ class BuildInfoHelper(object): | |||
1659 | license_path = os.path.join(license_directory, | 1650 | license_path = os.path.join(license_directory, |
1660 | real_image_name, 'license.manifest') | 1651 | real_image_name, 'license.manifest') |
1661 | 1652 | ||
1662 | self.orm_wrapper.update_target_set_license_manifest(target, | 1653 | self.orm_wrapper.update_target_set_license_manifest( |
1663 | license_path) | 1654 | image_target, license_path) |
1664 | 1655 | ||
1665 | # scan the directory for image files relating to this build | 1656 | # scan the directory for image files relating to this build |
1666 | # (via real_image_name); note that we don't have to set | 1657 | # (via real_image_name); note that we don't have to set |
@@ -1675,7 +1666,7 @@ class BuildInfoHelper(object): | |||
1675 | 1666 | ||
1676 | for image_file in image_files: | 1667 | for image_file in image_files: |
1677 | self.orm_wrapper.save_target_image_file_information( | 1668 | self.orm_wrapper.save_target_image_file_information( |
1678 | target, image_file['path'], image_file['size']) | 1669 | image_target, image_file['path'], image_file['size']) |
1679 | 1670 | ||
1680 | if not has_files: | 1671 | if not has_files: |
1681 | # copy image files and build artifacts from the | 1672 | # copy image files and build artifacts from the |
@@ -1683,13 +1674,115 @@ class BuildInfoHelper(object): | |||
1683 | # same target + machine as this Target; also copy the license | 1674 | # same target + machine as this Target; also copy the license |
1684 | # manifest path, as that is not treated as an artifact and needs | 1675 | # manifest path, as that is not treated as an artifact and needs |
1685 | # to be set separately | 1676 | # to be set separately |
1686 | most_recent = \ | 1677 | similar_target = \ |
1687 | self.orm_wrapper.get_similar_target_with_image_files(target) | 1678 | self.orm_wrapper.get_similar_target_with_image_files( |
1679 | image_target) | ||
1688 | 1680 | ||
1689 | if most_recent: | 1681 | if similar_target: |
1690 | logger.info('image artifacts for target %s cloned from ' \ | 1682 | logger.info('image artifacts for target %s cloned from ' \ |
1691 | 'target %s' % (target.pk, most_recent.pk)) | 1683 | 'target %s' % (image_target.pk, similar_target.pk)) |
1692 | target.clone_artifacts_from(most_recent) | 1684 | self.orm_wrapper.clone_image_artifacts(similar_target, |
1685 | image_target) | ||
1686 | |||
1687 | def _get_sdk_targets(self): | ||
1688 | """ | ||
1689 | Return targets which could generate SDK artifacts, i.e. | ||
1690 | "do_populate_sdk" and "do_populate_sdk_ext". | ||
1691 | """ | ||
1692 | return [target for target in self.internal_state['targets'] \ | ||
1693 | if target.task in ['populate_sdk', 'populate_sdk_ext']] | ||
1694 | |||
1695 | def scan_sdk_artifacts(self, event): | ||
1696 | """ | ||
1697 | Note that we have to intercept an SDKArtifactInfo event from | ||
1698 | toaster.bbclass (via toasterui) to get hold of the SDK variables we | ||
1699 | need to be able to scan for files accurately: this is because | ||
1700 | variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time | ||
1701 | BuildCompleted is fired by bitbake, so we have to get those values | ||
1702 | while the build is still in progress. | ||
1703 | |||
1704 | For populate_sdk_ext, this runs twice, with two different | ||
1705 | TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the | ||
1706 | files in the SDK output directory. | ||
1707 | """ | ||
1708 | sdk_vars = BuildInfoHelper._get_data_from_event(event) | ||
1709 | toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME'] | ||
1710 | |||
1711 | # targets which might have created SDK artifacts | ||
1712 | sdk_targets = self._get_sdk_targets() | ||
1713 | |||
1714 | # location of SDK artifacts | ||
1715 | tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0] | ||
1716 | sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk') | ||
1717 | |||
1718 | # all files in the SDK directory | ||
1719 | artifacts = [] | ||
1720 | for dir_path, _, filenames in os.walk(sdk_dir): | ||
1721 | for filename in filenames: | ||
1722 | full_path = os.path.join(dir_path, filename) | ||
1723 | if not os.path.islink(full_path): | ||
1724 | artifacts.append(full_path) | ||
1725 | |||
1726 | for sdk_target in sdk_targets: | ||
1727 | # find files in the SDK directory which haven't already been | ||
1728 | # recorded against a Target and whose basename matches | ||
1729 | # TOOLCHAIN_OUTPUTNAME | ||
1730 | for artifact_path in artifacts: | ||
1731 | basename = os.path.basename(artifact_path) | ||
1732 | |||
1733 | toolchain_match = basename.startswith(toolchain_outputname) | ||
1734 | |||
1735 | # files which match the name of the target which produced them; | ||
1736 | # for example, | ||
1737 | # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh | ||
1738 | target_match = re.search(sdk_target.target, basename) | ||
1739 | |||
1740 | # targets which produce "*-nativesdk-*" files | ||
1741 | is_ext_sdk_target = sdk_target.task in \ | ||
1742 | ['do_populate_sdk_ext', 'populate_sdk_ext'] | ||
1743 | |||
1744 | # SDK files which don't match the target name, i.e. | ||
1745 | # x86_64-nativesdk-libc.* | ||
1746 | # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot* | ||
1747 | is_ext_sdk_file = re.search('-nativesdk-', basename) | ||
1748 | |||
1749 | file_from_target = (toolchain_match and target_match) or \ | ||
1750 | (is_ext_sdk_target and is_ext_sdk_file) | ||
1751 | |||
1752 | if file_from_target: | ||
1753 | # don't record the file if it's already been added to this | ||
1754 | # target | ||
1755 | matching_files = TargetSDKFile.objects.filter( | ||
1756 | target=sdk_target, file_name=artifact_path) | ||
1757 | |||
1758 | if matching_files.count() == 0: | ||
1759 | artifact_size = os.stat(artifact_path).st_size | ||
1760 | |||
1761 | self.orm_wrapper.save_target_sdk_file( | ||
1762 | sdk_target, artifact_path, artifact_size) | ||
1763 | |||
1764 | def clone_required_sdk_artifacts(self): | ||
1765 | """ | ||
1766 | If an SDK target doesn't have any SDK artifacts, this means that | ||
1767 | the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which | ||
1768 | in turn means that the targets of this build didn't generate any new | ||
1769 | artifacts. | ||
1770 | |||
1771 | In this case, clone SDK artifacts for targets in the current build | ||
1772 | from existing targets for this build. | ||
1773 | """ | ||
1774 | sdk_targets = self._get_sdk_targets() | ||
1775 | for sdk_target in sdk_targets: | ||
1776 | # only clone for SDK targets which have no TargetSDKFiles yet | ||
1777 | if sdk_target.targetsdkfile_set.all().count() == 0: | ||
1778 | similar_target = \ | ||
1779 | self.orm_wrapper.get_similar_target_with_sdk_files( | ||
1780 | sdk_target) | ||
1781 | if similar_target: | ||
1782 | logger.info('SDK artifacts for target %s cloned from ' \ | ||
1783 | 'target %s' % (sdk_target.pk, similar_target.pk)) | ||
1784 | self.orm_wrapper.clone_sdk_artifacts(similar_target, | ||
1785 | sdk_target) | ||
1693 | 1786 | ||
1694 | def close(self, errorcode): | 1787 | def close(self, errorcode): |
1695 | if self.brbe is not None: | 1788 | if self.brbe is not None: |
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py index d8bccdb81c..f399a7d316 100644 --- a/bitbake/lib/bb/ui/toasterui.py +++ b/bitbake/lib/bb/ui/toasterui.py | |||
@@ -364,7 +364,8 @@ def main(server, eventHandler, params): | |||
364 | errorcode = 1 | 364 | errorcode = 1 |
365 | logger.error("Command execution failed: %s", event.error) | 365 | logger.error("Command execution failed: %s", event.error) |
366 | elif isinstance(event, bb.event.BuildCompleted): | 366 | elif isinstance(event, bb.event.BuildCompleted): |
367 | buildinfohelper.scan_build_artifacts() | 367 | buildinfohelper.scan_image_artifacts() |
368 | buildinfohelper.clone_required_sdk_artifacts() | ||
368 | 369 | ||
369 | # turn off logging to the current build log | 370 | # turn off logging to the current build log |
370 | _close_build_log(build_log) | 371 | _close_build_log(build_log) |
@@ -412,8 +413,8 @@ def main(server, eventHandler, params): | |||
412 | buildinfohelper.store_target_package_data(event) | 413 | buildinfohelper.store_target_package_data(event) |
413 | elif event.type == "MissedSstate": | 414 | elif event.type == "MissedSstate": |
414 | buildinfohelper.store_missed_state_tasks(event) | 415 | buildinfohelper.store_missed_state_tasks(event) |
415 | elif event.type == "ArtifactFileSize": | 416 | elif event.type == "SDKArtifactInfo": |
416 | buildinfohelper.update_artifact_image_file(event) | 417 | buildinfohelper.scan_sdk_artifacts(event) |
417 | elif event.type == "SetBRBE": | 418 | elif event.type == "SetBRBE": |
418 | buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) | 419 | buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) |
419 | elif event.type == "OSErrorException": | 420 | elif event.type == "OSErrorException": |
diff --git a/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py b/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py new file mode 100644 index 0000000000..3367582a81 --- /dev/null +++ b/bitbake/lib/toaster/orm/migrations/0008_refactor_artifact_models.py | |||
@@ -0,0 +1,39 @@ | |||
1 | # -*- coding: utf-8 -*- | ||
2 | from __future__ import unicode_literals | ||
3 | |||
4 | from django.db import migrations, models | ||
5 | |||
6 | |||
7 | class Migration(migrations.Migration): | ||
8 | |||
9 | dependencies = [ | ||
10 | ('orm', '0007_auto_20160523_1446'), | ||
11 | ] | ||
12 | |||
13 | operations = [ | ||
14 | migrations.CreateModel( | ||
15 | name='TargetKernelFile', | ||
16 | fields=[ | ||
17 | ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)), | ||
18 | ('file_name', models.FilePathField()), | ||
19 | ('file_size', models.IntegerField()), | ||
20 | ('target', models.ForeignKey(to='orm.Target')), | ||
21 | ], | ||
22 | ), | ||
23 | migrations.CreateModel( | ||
24 | name='TargetSDKFile', | ||
25 | fields=[ | ||
26 | ('id', models.AutoField(auto_created=True, primary_key=True, verbose_name='ID', serialize=False)), | ||
27 | ('file_name', models.FilePathField()), | ||
28 | ('file_size', models.IntegerField()), | ||
29 | ('target', models.ForeignKey(to='orm.Target')), | ||
30 | ], | ||
31 | ), | ||
32 | migrations.RemoveField( | ||
33 | model_name='buildartifact', | ||
34 | name='build', | ||
35 | ), | ||
36 | migrations.DeleteModel( | ||
37 | name='BuildArtifact', | ||
38 | ), | ||
39 | ] | ||
diff --git a/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py b/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py deleted file mode 100644 index 9f1d9bf4ec..0000000000 --- a/bitbake/lib/toaster/orm/migrations/0008_targetartifactfile.py +++ /dev/null | |||
@@ -1,23 +0,0 @@ | |||
1 | # -*- coding: utf-8 -*- | ||
2 | from __future__ import unicode_literals | ||
3 | |||
4 | from django.db import migrations, models | ||
5 | |||
6 | |||
7 | class Migration(migrations.Migration): | ||
8 | |||
9 | dependencies = [ | ||
10 | ('orm', '0007_auto_20160523_1446'), | ||
11 | ] | ||
12 | |||
13 | operations = [ | ||
14 | migrations.CreateModel( | ||
15 | name='TargetArtifactFile', | ||
16 | fields=[ | ||
17 | ('id', models.AutoField(primary_key=True, serialize=False, auto_created=True, verbose_name='ID')), | ||
18 | ('file_name', models.FilePathField()), | ||
19 | ('file_size', models.IntegerField()), | ||
20 | ('target', models.ForeignKey(to='orm.Target')), | ||
21 | ], | ||
22 | ), | ||
23 | ] | ||
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py index 9edbef3396..61f6a2072e 100644 --- a/bitbake/lib/toaster/orm/models.py +++ b/bitbake/lib/toaster/orm/models.py | |||
@@ -581,28 +581,6 @@ class Build(models.Model): | |||
581 | def __str__(self): | 581 | def __str__(self): |
582 | return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) | 582 | return "%d %s %s" % (self.id, self.project, ",".join([t.target for t in self.target_set.all()])) |
583 | 583 | ||
584 | |||
585 | # an Artifact is anything that results from a Build, and may be of interest to the user, and is not stored elsewhere | ||
586 | class BuildArtifact(models.Model): | ||
587 | build = models.ForeignKey(Build) | ||
588 | file_name = models.FilePathField() | ||
589 | file_size = models.IntegerField() | ||
590 | |||
591 | def get_local_file_name(self): | ||
592 | try: | ||
593 | deploydir = Variable.objects.get(build = self.build, variable_name="DEPLOY_DIR").variable_value | ||
594 | return self.file_name[len(deploydir)+1:] | ||
595 | except: | ||
596 | raise | ||
597 | |||
598 | return self.file_name | ||
599 | |||
600 | def get_basename(self): | ||
601 | return os.path.basename(self.file_name) | ||
602 | |||
603 | def is_available(self): | ||
604 | return self.build.buildrequest.environment.has_artifact(self.file_name) | ||
605 | |||
606 | class ProjectTarget(models.Model): | 584 | class ProjectTarget(models.Model): |
607 | project = models.ForeignKey(Project) | 585 | project = models.ForeignKey(Project) |
608 | target = models.CharField(max_length=100) | 586 | target = models.CharField(max_length=100) |
@@ -625,13 +603,19 @@ class Target(models.Model): | |||
625 | 603 | ||
626 | def get_similar_targets(self): | 604 | def get_similar_targets(self): |
627 | """ | 605 | """ |
628 | Get targets for the same machine, task and target name | 606 | Get target sfor the same machine, task and target name |
629 | (e.g. 'core-image-minimal') from a successful build for this project | 607 | (e.g. 'core-image-minimal') from a successful build for this project |
630 | (but excluding this target). | 608 | (but excluding this target). |
631 | 609 | ||
632 | Note that we look for targets built by this project because projects | 610 | Note that we only look for targets built by this project because |
633 | can have different configurations from each other, and put their | 611 | projects can have different configurations from each other, and put |
634 | artifacts in different directories. | 612 | their artifacts in different directories. |
613 | |||
614 | The possibility of error when retrieving candidate targets | ||
615 | is minimised by the fact that bitbake will rebuild artifacts if MACHINE | ||
616 | (or various other variables) change. In this case, there is no need to | ||
617 | clone artifacts from another target, as those artifacts will have | ||
618 | been re-generated for this target anyway. | ||
635 | """ | 619 | """ |
636 | query = ~Q(pk=self.pk) & \ | 620 | query = ~Q(pk=self.pk) & \ |
637 | Q(target=self.target) & \ | 621 | Q(target=self.target) & \ |
@@ -649,29 +633,54 @@ class Target(models.Model): | |||
649 | similar_target = None | 633 | similar_target = None |
650 | 634 | ||
651 | candidates = self.get_similar_targets() | 635 | candidates = self.get_similar_targets() |
652 | if candidates.count() < 1: | 636 | if candidates.count() == 0: |
653 | return similar_target | 637 | return similar_target |
654 | 638 | ||
655 | task_subquery = Q(task=self.task) | 639 | task_subquery = Q(task=self.task) |
656 | 640 | ||
657 | # we can look for a 'build' task if this task is a 'populate_sdk_ext' | 641 | # we can look for a 'build' task if this task is a 'populate_sdk_ext' |
658 | # task, as it will have created images; and vice versa; note that | 642 | # task, as the latter also creates images; and vice versa; note that |
659 | # 'build' targets can have their task set to ''; | 643 | # 'build' targets can have their task set to ''; |
660 | # also note that 'populate_sdk' does not produce image files | 644 | # also note that 'populate_sdk' does not produce image files |
661 | image_tasks = [ | 645 | image_tasks = [ |
662 | '', # aka 'build' | 646 | '', # aka 'build' |
663 | 'build', | 647 | 'build', |
648 | 'image', | ||
664 | 'populate_sdk_ext' | 649 | 'populate_sdk_ext' |
665 | ] | 650 | ] |
666 | if self.task in image_tasks: | 651 | if self.task in image_tasks: |
667 | task_subquery = Q(task__in=image_tasks) | 652 | task_subquery = Q(task__in=image_tasks) |
668 | 653 | ||
654 | # annotate with the count of files, to exclude any targets which | ||
655 | # don't have associated files | ||
656 | candidates = candidates.annotate(num_files=Count('target_image_file')) | ||
657 | |||
669 | query = task_subquery & Q(num_files__gt=0) | 658 | query = task_subquery & Q(num_files__gt=0) |
670 | 659 | ||
660 | candidates = candidates.filter(query) | ||
661 | |||
662 | if candidates.count() > 0: | ||
663 | candidates.order_by('build__completed_on') | ||
664 | similar_target = candidates.last() | ||
665 | |||
666 | return similar_target | ||
667 | |||
668 | def get_similar_target_with_sdk_files(self): | ||
669 | """ | ||
670 | Get the most recent similar target with TargetSDKFiles associated | ||
671 | with it, for the purpose of cloning those files onto this target. | ||
672 | """ | ||
673 | similar_target = None | ||
674 | |||
675 | candidates = self.get_similar_targets() | ||
676 | if candidates.count() == 0: | ||
677 | return similar_target | ||
678 | |||
671 | # annotate with the count of files, to exclude any targets which | 679 | # annotate with the count of files, to exclude any targets which |
672 | # don't have associated files | 680 | # don't have associated files |
673 | candidates = candidates.annotate( | 681 | candidates = candidates.annotate(num_files=Count('targetsdkfile')) |
674 | num_files=Count('target_image_file')) | 682 | |
683 | query = Q(task=self.task) & Q(num_files__gt=0) | ||
675 | 684 | ||
676 | candidates = candidates.filter(query) | 685 | candidates = candidates.filter(query) |
677 | 686 | ||
@@ -681,11 +690,10 @@ class Target(models.Model): | |||
681 | 690 | ||
682 | return similar_target | 691 | return similar_target |
683 | 692 | ||
684 | def clone_artifacts_from(self, target): | 693 | def clone_image_artifacts_from(self, target): |
685 | """ | 694 | """ |
686 | Make clones of the BuildArtifacts, Target_Image_Files and | 695 | Make clones of the Target_Image_Files and TargetKernelFile objects |
687 | TargetArtifactFile objects associated with Target target, then | 696 | associated with Target target, then associate them with this target. |
688 | associate them with this target. | ||
689 | 697 | ||
690 | Note that for Target_Image_Files, we only want files from the previous | 698 | Note that for Target_Image_Files, we only want files from the previous |
691 | build whose suffix matches one of the suffixes defined in this | 699 | build whose suffix matches one of the suffixes defined in this |
@@ -711,18 +719,38 @@ class Target(models.Model): | |||
711 | image_file.target = self | 719 | image_file.target = self |
712 | image_file.save() | 720 | image_file.save() |
713 | 721 | ||
714 | artifact_files = target.targetartifactfile_set.all() | 722 | kernel_files = target.targetkernelfile_set.all() |
715 | for artifact_file in artifact_files: | 723 | for kernel_file in kernel_files: |
716 | artifact_file.pk = None | 724 | kernel_file.pk = None |
717 | artifact_file.target = self | 725 | kernel_file.target = self |
718 | artifact_file.save() | 726 | kernel_file.save() |
719 | 727 | ||
720 | self.license_manifest_path = target.license_manifest_path | 728 | self.license_manifest_path = target.license_manifest_path |
721 | self.save() | 729 | self.save() |
722 | 730 | ||
723 | # an Artifact is anything that results from a target being built, and may | 731 | def clone_sdk_artifacts_from(self, target): |
724 | # be of interest to the user, and is not an image file | 732 | """ |
725 | class TargetArtifactFile(models.Model): | 733 | Clone TargetSDKFile objects from target and associate them with this |
734 | target. | ||
735 | """ | ||
736 | sdk_files = target.targetsdkfile_set.all() | ||
737 | for sdk_file in sdk_files: | ||
738 | sdk_file.pk = None | ||
739 | sdk_file.target = self | ||
740 | sdk_file.save() | ||
741 | |||
742 | # kernel artifacts for a target: bzImage and modules* | ||
743 | class TargetKernelFile(models.Model): | ||
744 | target = models.ForeignKey(Target) | ||
745 | file_name = models.FilePathField() | ||
746 | file_size = models.IntegerField() | ||
747 | |||
748 | @property | ||
749 | def basename(self): | ||
750 | return os.path.basename(self.file_name) | ||
751 | |||
752 | # SDK artifacts for a target: sh and manifest files | ||
753 | class TargetSDKFile(models.Model): | ||
726 | target = models.ForeignKey(Target) | 754 | target = models.ForeignKey(Target) |
727 | file_name = models.FilePathField() | 755 | file_name = models.FilePathField() |
728 | file_size = models.IntegerField() | 756 | file_size = models.IntegerField() |
diff --git a/bitbake/lib/toaster/toastergui/templates/builddashboard.html b/bitbake/lib/toaster/toastergui/templates/builddashboard.html index f6d62b9951..e1bde21e99 100644 --- a/bitbake/lib/toaster/toastergui/templates/builddashboard.html +++ b/bitbake/lib/toaster/toastergui/templates/builddashboard.html | |||
@@ -80,29 +80,30 @@ | |||
80 | <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd> | 80 | <dd><a href="{% url 'target' build.pk target.target.pk %}">{{target.npkg}}</a></dd> |
81 | <dt>Total package size</dt> | 81 | <dt>Total package size</dt> |
82 | <dd>{{target.pkgsz|filtered_filesizeformat}}</dd> | 82 | <dd>{{target.pkgsz|filtered_filesizeformat}}</dd> |
83 | {% if target.targetHasNoImages %} | 83 | </dl> |
84 | </dl> | 84 | {% if target.targetHasNoImages %} |
85 | <div class="row"> | 85 | <div class="row"> |
86 | <div class="col-md-7"> | 86 | <div class="col-md-7"> |
87 | <div class="alert alert-info"> | 87 | <div class="alert alert-info"> |
88 | <p> | 88 | <p> |
89 | <strong>This build did not create any image files</strong> | 89 | <strong>This build did not create any image files</strong> |
90 | </p> | 90 | </p> |
91 | <p> | 91 | <p> |
92 | This is probably because valid image and license manifest | 92 | This is probably because valid image and license manifest |
93 | files from a previous build already exist in your | 93 | files from a previous build already exist in your |
94 | <code>build/tmp/deploy</code> | 94 | <code>build/tmp/deploy</code> |
95 | directory. You can | 95 | directory. You can |
96 | also <a href="{% url 'target' build.pk target.target.pk %}">view the | 96 | also <a href="{% url 'target' build.pk target.target.pk %}">view the |
97 | license manifest information</a> in Toaster. | 97 | license manifest information</a> in Toaster. |
98 | </p> | 98 | </p> |
99 | </div> | ||
100 | </div> | ||
101 | </div> | 99 | </div> |
102 | {% else %} | 100 | </div> |
101 | </div> | ||
102 | {% endif %} | ||
103 | {% if not target.targetHasNoImages %} | ||
104 | <dl class="dl-horizontal"> | ||
103 | <dt> | 105 | <dt> |
104 | <span class="glyphicon glyphicon-question-sign get-help" title="The location in disk of the license manifest, a document listing all packages installed in your image and their licenses"></span> | 106 | <span class="glyphicon glyphicon-question-sign get-help" title="The location in disk of the license manifest, a document listing all packages installed in your image and their licenses"></span> |
105 | |||
106 | License manifest | 107 | License manifest |
107 | </dt> | 108 | </dt> |
108 | <dd> | 109 | <dd> |
@@ -129,15 +130,32 @@ | |||
129 | </dt> | 130 | </dt> |
130 | <dd> | 131 | <dd> |
131 | <ul class="list-unstyled"> | 132 | <ul class="list-unstyled"> |
132 | {% for artifact in target.target_artifacts|dictsort:"basename" %} | 133 | {% for artifact in target.target_kernel_artifacts|dictsort:"basename" %} |
133 | <li> | 134 | <li> |
134 | <a href="{% url 'build_artifact' build.id 'targetartifactfile' artifact.id %}">{{artifact.basename}}</a> | 135 | <a href="{% url 'build_artifact' build.id 'targetkernelartifact' artifact.id %}">{{artifact.basename}}</a> |
135 | ({{artifact.file_size|filtered_filesizeformat}}) | 136 | ({{artifact.file_size|filtered_filesizeformat}}) |
136 | </li> | 137 | </li> |
137 | {% endfor %} | 138 | {% endfor %} |
138 | </ul> | 139 | </ul> |
139 | </dd> | 140 | </dd> |
140 | </dl> | 141 | </dl> |
142 | {% endif %} | ||
143 | {% if target.target_sdk_artifacts_count > 0 %} | ||
144 | <dl class="dl-horizontal"> | ||
145 | <dt> | ||
146 | SDK artifacts | ||
147 | </dt> | ||
148 | <dd> | ||
149 | <ul class="list-unstyled"> | ||
150 | {% for artifact in target.target_sdk_artifacts|dictsort:"basename" %} | ||
151 | <li> | ||
152 | <a href="{% url 'build_artifact' build.id 'targetsdkartifact' artifact.id %}">{{artifact.basename}}</a> | ||
153 | ({{artifact.file_size|filtered_filesizeformat}}) | ||
154 | </li> | ||
155 | {% endfor %} | ||
156 | </ul> | ||
157 | </dd> | ||
158 | </dl> | ||
141 | {% endif %} | 159 | {% endif %} |
142 | </div> | 160 | </div> |
143 | {% endif %} | 161 | {% endif %} |
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py index 0ec88d9a77..a82a261e0d 100755 --- a/bitbake/lib/toaster/toastergui/views.py +++ b/bitbake/lib/toaster/toastergui/views.py | |||
@@ -30,8 +30,8 @@ from django.db import IntegrityError, Error | |||
30 | from django.shortcuts import render, redirect, get_object_or_404 | 30 | from django.shortcuts import render, redirect, get_object_or_404 |
31 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable | 31 | from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe, LogMessage, Variable |
32 | from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency | 32 | from orm.models import Task_Dependency, Recipe_Dependency, Package, Package_File, Package_Dependency |
33 | from orm.models import Target_Installed_Package, Target_File, Target_Image_File, BuildArtifact, CustomImagePackage | 33 | from orm.models import Target_Installed_Package, Target_File, Target_Image_File, CustomImagePackage |
34 | from orm.models import TargetArtifactFile | 34 | from orm.models import TargetKernelFile, TargetSDKFile |
35 | from orm.models import BitbakeVersion, CustomImageRecipe | 35 | from orm.models import BitbakeVersion, CustomImageRecipe |
36 | from bldcontrol import bbcontroller | 36 | from bldcontrol import bbcontroller |
37 | from django.views.decorators.cache import cache_control | 37 | from django.views.decorators.cache import cache_control |
@@ -510,7 +510,11 @@ def builddashboard( request, build_id ): | |||
510 | targetHasNoImages = True | 510 | targetHasNoImages = True |
511 | elem[ 'imageFiles' ] = imageFiles | 511 | elem[ 'imageFiles' ] = imageFiles |
512 | elem[ 'targetHasNoImages' ] = targetHasNoImages | 512 | elem[ 'targetHasNoImages' ] = targetHasNoImages |
513 | elem['target_artifacts'] = t.targetartifactfile_set.all() | 513 | elem['target_kernel_artifacts'] = t.targetkernelfile_set.all() |
514 | |||
515 | target_sdk_files = t.targetsdkfile_set.all() | ||
516 | elem['target_sdk_artifacts_count'] = target_sdk_files.count() | ||
517 | elem['target_sdk_artifacts'] = target_sdk_files | ||
514 | 518 | ||
515 | targets.append( elem ) | 519 | targets.append( elem ) |
516 | 520 | ||
@@ -2294,11 +2298,12 @@ if True: | |||
2294 | elif artifact_type == "imagefile": | 2298 | elif artifact_type == "imagefile": |
2295 | file_name = Target_Image_File.objects.get(target__build = build, pk = artifact_id).file_name | 2299 | file_name = Target_Image_File.objects.get(target__build = build, pk = artifact_id).file_name |
2296 | 2300 | ||
2297 | elif artifact_type == "buildartifact": | 2301 | elif artifact_type == "targetkernelartifact": |
2298 | file_name = BuildArtifact.objects.get(build = build, pk = artifact_id).file_name | 2302 | target = TargetKernelFile.objects.get(pk=artifact_id) |
2303 | file_name = target.file_name | ||
2299 | 2304 | ||
2300 | elif artifact_type == "targetartifactfile": | 2305 | elif artifact_type == "targetsdkartifact": |
2301 | target = TargetArtifactFile.objects.get(pk=artifact_id) | 2306 | target = TargetSDKFile.objects.get(pk=artifact_id) |
2302 | file_name = target.file_name | 2307 | file_name = target.file_name |
2303 | 2308 | ||
2304 | elif artifact_type == "licensemanifest": | 2309 | elif artifact_type == "licensemanifest": |