diff options
Diffstat (limited to 'bitbake/lib/bb/ui/buildinfohelper.py')
-rw-r--r-- | bitbake/lib/bb/ui/buildinfohelper.py | 175 |
1 files changed, 134 insertions, 41 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py index e1b59c3e87..bf032ae835 100644 --- a/bitbake/lib/bb/ui/buildinfohelper.py +++ b/bitbake/lib/bb/ui/buildinfohelper.py | |||
@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\ | |||
37 | django.setup() | 37 | django.setup() |
38 | 38 | ||
39 | from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText | 39 | from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText |
40 | from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile | 40 | from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile |
41 | from orm.models import Variable, VariableHistory | 41 | from orm.models import Variable, VariableHistory |
42 | from orm.models import Package, Package_File, Target_Installed_Package, Target_File | 42 | from orm.models import Package, Package_File, Target_Installed_Package, Target_File |
43 | from orm.models import Task_Dependency, Package_Dependency | 43 | from orm.models import Task_Dependency, Package_Dependency |
@@ -128,6 +128,15 @@ class ORMWrapper(object): | |||
128 | """ | 128 | """ |
129 | return target.get_similar_target_with_image_files() | 129 | return target.get_similar_target_with_image_files() |
130 | 130 | ||
131 | def get_similar_target_with_sdk_files(self, target): | ||
132 | return target.get_similar_target_with_sdk_files() | ||
133 | |||
134 | def clone_image_artifacts(self, target_from, target_to): | ||
135 | target_to.clone_image_artifacts_from(target_from) | ||
136 | |||
137 | def clone_sdk_artifacts(self, target_from, target_to): | ||
138 | target_to.clone_sdk_artifacts_from(target_from) | ||
139 | |||
131 | def _timestamp_to_datetime(self, secs): | 140 | def _timestamp_to_datetime(self, secs): |
132 | """ | 141 | """ |
133 | Convert timestamp in seconds to Python datetime | 142 | Convert timestamp in seconds to Python datetime |
@@ -682,35 +691,22 @@ class ORMWrapper(object): | |||
682 | logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) | 691 | logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) |
683 | 692 | ||
684 | def save_target_image_file_information(self, target_obj, file_name, file_size): | 693 | def save_target_image_file_information(self, target_obj, file_name, file_size): |
685 | Target_Image_File.objects.create( target = target_obj, | 694 | Target_Image_File.objects.create(target=target_obj, |
686 | file_name = file_name, | 695 | file_name=file_name, file_size=file_size) |
687 | file_size = file_size) | ||
688 | 696 | ||
689 | def save_target_artifact_file(self, target_obj, file_name, file_size): | 697 | def save_target_kernel_file(self, target_obj, file_name, file_size): |
690 | """ | 698 | """ |
691 | Save artifact file information for a Target target_obj. | 699 | Save kernel file (bzImage, modules*) information for a Target target_obj. |
692 | |||
693 | Note that this doesn't include SDK artifacts, only images and | ||
694 | related files (e.g. bzImage). | ||
695 | """ | 700 | """ |
696 | TargetArtifactFile.objects.create(target=target_obj, | 701 | TargetKernelFile.objects.create(target=target_obj, |
697 | file_name=file_name, file_size=file_size) | 702 | file_name=file_name, file_size=file_size) |
698 | 703 | ||
699 | def save_artifact_information(self, build_obj, file_name, file_size): | 704 | def save_target_sdk_file(self, target_obj, file_name, file_size): |
700 | """ | 705 | """ |
701 | TODO this is currently used to save SDK artifacts to the database, | 706 | Save SDK artifacts to the database, associating them with a |
702 | but will be replaced in future once SDK artifacts are associated with | 707 | Target object. |
703 | Target objects (as they eventually should be) | ||
704 | """ | 708 | """ |
705 | # do not update artifacts found in other builds | 709 | TargetSDKFile.objects.create(target=target_obj, file_name=file_name, |
706 | if BuildArtifact.objects.filter(file_name = file_name).count() > 0: | ||
707 | return | ||
708 | |||
709 | # do not update artifact if already a target artifact file for that path | ||
710 | if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0: | ||
711 | return | ||
712 | |||
713 | BuildArtifact.objects.create(build=build_obj, file_name=file_name, | ||
714 | file_size=file_size) | 710 | file_size=file_size) |
715 | 711 | ||
716 | def create_logmessage(self, log_information): | 712 | def create_logmessage(self, log_information): |
@@ -1085,11 +1081,6 @@ class BuildInfoHelper(object): | |||
1085 | 1081 | ||
1086 | return self.brbe | 1082 | return self.brbe |
1087 | 1083 | ||
1088 | def update_artifact_image_file(self, event): | ||
1089 | evdata = BuildInfoHelper._get_data_from_event(event) | ||
1090 | for artifact_path in evdata.keys(): | ||
1091 | self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path]) | ||
1092 | |||
1093 | def update_build_information(self, event, errors, warnings, taskfailures): | 1084 | def update_build_information(self, event, errors, warnings, taskfailures): |
1094 | if 'build' in self.internal_state: | 1085 | if 'build' in self.internal_state: |
1095 | self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) | 1086 | self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) |
@@ -1568,9 +1559,9 @@ class BuildInfoHelper(object): | |||
1568 | 1559 | ||
1569 | return image_files | 1560 | return image_files |
1570 | 1561 | ||
1571 | def scan_build_artifacts(self): | 1562 | def scan_image_artifacts(self): |
1572 | """ | 1563 | """ |
1573 | Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them | 1564 | Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them |
1574 | with a Target object in self.internal_state['targets']. | 1565 | with a Target object in self.internal_state['targets']. |
1575 | 1566 | ||
1576 | We have two situations to handle: | 1567 | We have two situations to handle: |
@@ -1615,7 +1606,7 @@ class BuildInfoHelper(object): | |||
1615 | # filter out anything which isn't an image target | 1606 | # filter out anything which isn't an image target |
1616 | image_targets = [target for target in targets if target.is_image] | 1607 | image_targets = [target for target in targets if target.is_image] |
1617 | 1608 | ||
1618 | for target in image_targets: | 1609 | for image_target in image_targets: |
1619 | # this is set to True if we find at least one file relating to | 1610 | # this is set to True if we find at least one file relating to |
1620 | # this target; if this remains False after the scan, we copy the | 1611 | # this target; if this remains False after the scan, we copy the |
1621 | # files from the most-recent Target with the same target + machine | 1612 | # files from the most-recent Target with the same target + machine |
@@ -1627,7 +1618,7 @@ class BuildInfoHelper(object): | |||
1627 | # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; | 1618 | # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; |
1628 | # we need to change it to | 1619 | # we need to change it to |
1629 | # <TARGET>-<MACHINE>-<BUILDNAME> | 1620 | # <TARGET>-<MACHINE>-<BUILDNAME> |
1630 | real_image_name = re.sub(r'^defaultpkgname', target.target, | 1621 | real_image_name = re.sub(r'^defaultpkgname', image_target.target, |
1631 | image_name) | 1622 | image_name) |
1632 | 1623 | ||
1633 | image_license_manifest_path = os.path.join( | 1624 | image_license_manifest_path = os.path.join( |
@@ -1651,7 +1642,7 @@ class BuildInfoHelper(object): | |||
1651 | 1642 | ||
1652 | # note that the artifact will only be saved against this | 1643 | # note that the artifact will only be saved against this |
1653 | # build if it hasn't been already | 1644 | # build if it hasn't been already |
1654 | self.orm_wrapper.save_target_artifact_file(target, | 1645 | self.orm_wrapper.save_target_kernel_file(image_target, |
1655 | artifact_path, artifact_size) | 1646 | artifact_path, artifact_size) |
1656 | 1647 | ||
1657 | # store the license manifest path on the target | 1648 | # store the license manifest path on the target |
@@ -1659,8 +1650,8 @@ class BuildInfoHelper(object): | |||
1659 | license_path = os.path.join(license_directory, | 1650 | license_path = os.path.join(license_directory, |
1660 | real_image_name, 'license.manifest') | 1651 | real_image_name, 'license.manifest') |
1661 | 1652 | ||
1662 | self.orm_wrapper.update_target_set_license_manifest(target, | 1653 | self.orm_wrapper.update_target_set_license_manifest( |
1663 | license_path) | 1654 | image_target, license_path) |
1664 | 1655 | ||
1665 | # scan the directory for image files relating to this build | 1656 | # scan the directory for image files relating to this build |
1666 | # (via real_image_name); note that we don't have to set | 1657 | # (via real_image_name); note that we don't have to set |
@@ -1675,7 +1666,7 @@ class BuildInfoHelper(object): | |||
1675 | 1666 | ||
1676 | for image_file in image_files: | 1667 | for image_file in image_files: |
1677 | self.orm_wrapper.save_target_image_file_information( | 1668 | self.orm_wrapper.save_target_image_file_information( |
1678 | target, image_file['path'], image_file['size']) | 1669 | image_target, image_file['path'], image_file['size']) |
1679 | 1670 | ||
1680 | if not has_files: | 1671 | if not has_files: |
1681 | # copy image files and build artifacts from the | 1672 | # copy image files and build artifacts from the |
@@ -1683,13 +1674,115 @@ class BuildInfoHelper(object): | |||
1683 | # same target + machine as this Target; also copy the license | 1674 | # same target + machine as this Target; also copy the license |
1684 | # manifest path, as that is not treated as an artifact and needs | 1675 | # manifest path, as that is not treated as an artifact and needs |
1685 | # to be set separately | 1676 | # to be set separately |
1686 | most_recent = \ | 1677 | similar_target = \ |
1687 | self.orm_wrapper.get_similar_target_with_image_files(target) | 1678 | self.orm_wrapper.get_similar_target_with_image_files( |
1679 | image_target) | ||
1688 | 1680 | ||
1689 | if most_recent: | 1681 | if similar_target: |
1690 | logger.info('image artifacts for target %s cloned from ' \ | 1682 | logger.info('image artifacts for target %s cloned from ' \ |
1691 | 'target %s' % (target.pk, most_recent.pk)) | 1683 | 'target %s' % (image_target.pk, similar_target.pk)) |
1692 | target.clone_artifacts_from(most_recent) | 1684 | self.orm_wrapper.clone_image_artifacts(similar_target, |
1685 | image_target) | ||
1686 | |||
1687 | def _get_sdk_targets(self): | ||
1688 | """ | ||
1689 | Return targets which could generate SDK artifacts, i.e. | ||
1690 | "do_populate_sdk" and "do_populate_sdk_ext". | ||
1691 | """ | ||
1692 | return [target for target in self.internal_state['targets'] \ | ||
1693 | if target.task in ['populate_sdk', 'populate_sdk_ext']] | ||
1694 | |||
1695 | def scan_sdk_artifacts(self, event): | ||
1696 | """ | ||
1697 | Note that we have to intercept an SDKArtifactInfo event from | ||
1698 | toaster.bbclass (via toasterui) to get hold of the SDK variables we | ||
1699 | need to be able to scan for files accurately: this is because | ||
1700 | variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time | ||
1701 | BuildCompleted is fired by bitbake, so we have to get those values | ||
1702 | while the build is still in progress. | ||
1703 | |||
1704 | For populate_sdk_ext, this runs twice, with two different | ||
1705 | TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the | ||
1706 | files in the SDK output directory. | ||
1707 | """ | ||
1708 | sdk_vars = BuildInfoHelper._get_data_from_event(event) | ||
1709 | toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME'] | ||
1710 | |||
1711 | # targets which might have created SDK artifacts | ||
1712 | sdk_targets = self._get_sdk_targets() | ||
1713 | |||
1714 | # location of SDK artifacts | ||
1715 | tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0] | ||
1716 | sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk') | ||
1717 | |||
1718 | # all files in the SDK directory | ||
1719 | artifacts = [] | ||
1720 | for dir_path, _, filenames in os.walk(sdk_dir): | ||
1721 | for filename in filenames: | ||
1722 | full_path = os.path.join(dir_path, filename) | ||
1723 | if not os.path.islink(full_path): | ||
1724 | artifacts.append(full_path) | ||
1725 | |||
1726 | for sdk_target in sdk_targets: | ||
1727 | # find files in the SDK directory which haven't already been | ||
1728 | # recorded against a Target and whose basename matches | ||
1729 | # TOOLCHAIN_OUTPUTNAME | ||
1730 | for artifact_path in artifacts: | ||
1731 | basename = os.path.basename(artifact_path) | ||
1732 | |||
1733 | toolchain_match = basename.startswith(toolchain_outputname) | ||
1734 | |||
1735 | # files which match the name of the target which produced them; | ||
1736 | # for example, | ||
1737 | # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh | ||
1738 | target_match = re.search(sdk_target.target, basename) | ||
1739 | |||
1740 | # targets which produce "*-nativesdk-*" files | ||
1741 | is_ext_sdk_target = sdk_target.task in \ | ||
1742 | ['do_populate_sdk_ext', 'populate_sdk_ext'] | ||
1743 | |||
1744 | # SDK files which don't match the target name, i.e. | ||
1745 | # x86_64-nativesdk-libc.* | ||
1746 | # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot* | ||
1747 | is_ext_sdk_file = re.search('-nativesdk-', basename) | ||
1748 | |||
1749 | file_from_target = (toolchain_match and target_match) or \ | ||
1750 | (is_ext_sdk_target and is_ext_sdk_file) | ||
1751 | |||
1752 | if file_from_target: | ||
1753 | # don't record the file if it's already been added to this | ||
1754 | # target | ||
1755 | matching_files = TargetSDKFile.objects.filter( | ||
1756 | target=sdk_target, file_name=artifact_path) | ||
1757 | |||
1758 | if matching_files.count() == 0: | ||
1759 | artifact_size = os.stat(artifact_path).st_size | ||
1760 | |||
1761 | self.orm_wrapper.save_target_sdk_file( | ||
1762 | sdk_target, artifact_path, artifact_size) | ||
1763 | |||
1764 | def clone_required_sdk_artifacts(self): | ||
1765 | """ | ||
1766 | If an SDK target doesn't have any SDK artifacts, this means that | ||
1767 | the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which | ||
1768 | in turn means that the targets of this build didn't generate any new | ||
1769 | artifacts. | ||
1770 | |||
1771 | In this case, clone SDK artifacts for targets in the current build | ||
1772 | from existing targets for this build. | ||
1773 | """ | ||
1774 | sdk_targets = self._get_sdk_targets() | ||
1775 | for sdk_target in sdk_targets: | ||
1776 | # only clone for SDK targets which have no TargetSDKFiles yet | ||
1777 | if sdk_target.targetsdkfile_set.all().count() == 0: | ||
1778 | similar_target = \ | ||
1779 | self.orm_wrapper.get_similar_target_with_sdk_files( | ||
1780 | sdk_target) | ||
1781 | if similar_target: | ||
1782 | logger.info('SDK artifacts for target %s cloned from ' \ | ||
1783 | 'target %s' % (sdk_target.pk, similar_target.pk)) | ||
1784 | self.orm_wrapper.clone_sdk_artifacts(similar_target, | ||
1785 | sdk_target) | ||
1693 | 1786 | ||
1694 | def close(self, errorcode): | 1787 | def close(self, errorcode): |
1695 | if self.brbe is not None: | 1788 | if self.brbe is not None: |