summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorElliot Smith <elliot.smith@intel.com>2016-07-12 15:54:48 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-19 08:56:51 +0100
commit00c2c0be5ead435601a21a676401674b7045f6f0 (patch)
tree32295fe980f44dfae3353ed7bf58691aee356ecb /bitbake/lib/bb
parentf39ae146eadf92f650ed6340158e780a65d483b1 (diff)
downloadpoky-00c2c0be5ead435601a21a676401674b7045f6f0.tar.gz
bitbake: toaster: improve scan for SDK artifacts
SDK artifacts were previously picked up by toaster.bbclass and notified to buildinfohelper (via toasterui). The artifacts were then added to the Build object, so that it wasn't clear which artifact went with which target; we were also unable to attach SDK artifacts to a Build if they had already been attached to a previous build. Now, toaster.bbclass just notifies the TOOLCHAIN_OUTPUTNAME when a populate_sdk* target completes. The scan is moved to buildinfohelper, where we search the SDK deploy directory for files matching TOOLCHAIN_OUTPUTNAME and attach them to targets (not builds). If an SDK file is not produced by a target, we now look for a similar, previously-run target which did produce artifacts. If there is one, we clone the SDK artifacts from that target onto the current one. This all means that we can show SDK artifacts by target, and should always get artifacts associated with a target, regardless of whether it really build them. This requires an additional model, TargetSDKFile, which tracks the size and path of SDK artifact files with respect to Target objects. [YOCTO #8556] (Bitbake rev: 5e650c611605507e1e0d1588cd5eb6535c2d34fc) Signed-off-by: Elliot Smith <elliot.smith@intel.com> Signed-off-by: bavery <brian.avery@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/ui/buildinfohelper.py175
-rw-r--r--bitbake/lib/bb/ui/toasterui.py7
2 files changed, 138 insertions, 44 deletions
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index e1b59c3e87..bf032ae835 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\
37django.setup() 37django.setup()
38 38
39from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText 39from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
40from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile 40from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile
41from orm.models import Variable, VariableHistory 41from orm.models import Variable, VariableHistory
42from orm.models import Package, Package_File, Target_Installed_Package, Target_File 42from orm.models import Package, Package_File, Target_Installed_Package, Target_File
43from orm.models import Task_Dependency, Package_Dependency 43from orm.models import Task_Dependency, Package_Dependency
@@ -128,6 +128,15 @@ class ORMWrapper(object):
128 """ 128 """
129 return target.get_similar_target_with_image_files() 129 return target.get_similar_target_with_image_files()
130 130
131 def get_similar_target_with_sdk_files(self, target):
132 return target.get_similar_target_with_sdk_files()
133
134 def clone_image_artifacts(self, target_from, target_to):
135 target_to.clone_image_artifacts_from(target_from)
136
137 def clone_sdk_artifacts(self, target_from, target_to):
138 target_to.clone_sdk_artifacts_from(target_from)
139
131 def _timestamp_to_datetime(self, secs): 140 def _timestamp_to_datetime(self, secs):
132 """ 141 """
133 Convert timestamp in seconds to Python datetime 142 Convert timestamp in seconds to Python datetime
@@ -682,35 +691,22 @@ class ORMWrapper(object):
682 logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg) 691 logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
683 692
684 def save_target_image_file_information(self, target_obj, file_name, file_size): 693 def save_target_image_file_information(self, target_obj, file_name, file_size):
685 Target_Image_File.objects.create( target = target_obj, 694 Target_Image_File.objects.create(target=target_obj,
686 file_name = file_name, 695 file_name=file_name, file_size=file_size)
687 file_size = file_size)
688 696
689 def save_target_artifact_file(self, target_obj, file_name, file_size): 697 def save_target_kernel_file(self, target_obj, file_name, file_size):
690 """ 698 """
691 Save artifact file information for a Target target_obj. 699 Save kernel file (bzImage, modules*) information for a Target target_obj.
692
693 Note that this doesn't include SDK artifacts, only images and
694 related files (e.g. bzImage).
695 """ 700 """
696 TargetArtifactFile.objects.create(target=target_obj, 701 TargetKernelFile.objects.create(target=target_obj,
697 file_name=file_name, file_size=file_size) 702 file_name=file_name, file_size=file_size)
698 703
699 def save_artifact_information(self, build_obj, file_name, file_size): 704 def save_target_sdk_file(self, target_obj, file_name, file_size):
700 """ 705 """
701 TODO this is currently used to save SDK artifacts to the database, 706 Save SDK artifacts to the database, associating them with a
702 but will be replaced in future once SDK artifacts are associated with 707 Target object.
703 Target objects (as they eventually should be)
704 """ 708 """
705 # do not update artifacts found in other builds 709 TargetSDKFile.objects.create(target=target_obj, file_name=file_name,
706 if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
707 return
708
709 # do not update artifact if already a target artifact file for that path
710 if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0:
711 return
712
713 BuildArtifact.objects.create(build=build_obj, file_name=file_name,
714 file_size=file_size) 710 file_size=file_size)
715 711
716 def create_logmessage(self, log_information): 712 def create_logmessage(self, log_information):
@@ -1085,11 +1081,6 @@ class BuildInfoHelper(object):
1085 1081
1086 return self.brbe 1082 return self.brbe
1087 1083
1088 def update_artifact_image_file(self, event):
1089 evdata = BuildInfoHelper._get_data_from_event(event)
1090 for artifact_path in evdata.keys():
1091 self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path])
1092
1093 def update_build_information(self, event, errors, warnings, taskfailures): 1084 def update_build_information(self, event, errors, warnings, taskfailures):
1094 if 'build' in self.internal_state: 1085 if 'build' in self.internal_state:
1095 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures) 1086 self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
@@ -1568,9 +1559,9 @@ class BuildInfoHelper(object):
1568 1559
1569 return image_files 1560 return image_files
1570 1561
1571 def scan_build_artifacts(self): 1562 def scan_image_artifacts(self):
1572 """ 1563 """
1573 Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them 1564 Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them
1574 with a Target object in self.internal_state['targets']. 1565 with a Target object in self.internal_state['targets'].
1575 1566
1576 We have two situations to handle: 1567 We have two situations to handle:
@@ -1615,7 +1606,7 @@ class BuildInfoHelper(object):
1615 # filter out anything which isn't an image target 1606 # filter out anything which isn't an image target
1616 image_targets = [target for target in targets if target.is_image] 1607 image_targets = [target for target in targets if target.is_image]
1617 1608
1618 for target in image_targets: 1609 for image_target in image_targets:
1619 # this is set to True if we find at least one file relating to 1610 # this is set to True if we find at least one file relating to
1620 # this target; if this remains False after the scan, we copy the 1611 # this target; if this remains False after the scan, we copy the
1621 # files from the most-recent Target with the same target + machine 1612 # files from the most-recent Target with the same target + machine
@@ -1627,7 +1618,7 @@ class BuildInfoHelper(object):
1627 # 'defaultpkgname-<MACHINE>-<BUILDNAME>'; 1618 # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
1628 # we need to change it to 1619 # we need to change it to
1629 # <TARGET>-<MACHINE>-<BUILDNAME> 1620 # <TARGET>-<MACHINE>-<BUILDNAME>
1630 real_image_name = re.sub(r'^defaultpkgname', target.target, 1621 real_image_name = re.sub(r'^defaultpkgname', image_target.target,
1631 image_name) 1622 image_name)
1632 1623
1633 image_license_manifest_path = os.path.join( 1624 image_license_manifest_path = os.path.join(
@@ -1651,7 +1642,7 @@ class BuildInfoHelper(object):
1651 1642
1652 # note that the artifact will only be saved against this 1643 # note that the artifact will only be saved against this
1653 # build if it hasn't been already 1644 # build if it hasn't been already
1654 self.orm_wrapper.save_target_artifact_file(target, 1645 self.orm_wrapper.save_target_kernel_file(image_target,
1655 artifact_path, artifact_size) 1646 artifact_path, artifact_size)
1656 1647
1657 # store the license manifest path on the target 1648 # store the license manifest path on the target
@@ -1659,8 +1650,8 @@ class BuildInfoHelper(object):
1659 license_path = os.path.join(license_directory, 1650 license_path = os.path.join(license_directory,
1660 real_image_name, 'license.manifest') 1651 real_image_name, 'license.manifest')
1661 1652
1662 self.orm_wrapper.update_target_set_license_manifest(target, 1653 self.orm_wrapper.update_target_set_license_manifest(
1663 license_path) 1654 image_target, license_path)
1664 1655
1665 # scan the directory for image files relating to this build 1656 # scan the directory for image files relating to this build
1666 # (via real_image_name); note that we don't have to set 1657 # (via real_image_name); note that we don't have to set
@@ -1675,7 +1666,7 @@ class BuildInfoHelper(object):
1675 1666
1676 for image_file in image_files: 1667 for image_file in image_files:
1677 self.orm_wrapper.save_target_image_file_information( 1668 self.orm_wrapper.save_target_image_file_information(
1678 target, image_file['path'], image_file['size']) 1669 image_target, image_file['path'], image_file['size'])
1679 1670
1680 if not has_files: 1671 if not has_files:
1681 # copy image files and build artifacts from the 1672 # copy image files and build artifacts from the
@@ -1683,13 +1674,115 @@ class BuildInfoHelper(object):
1683 # same target + machine as this Target; also copy the license 1674 # same target + machine as this Target; also copy the license
1684 # manifest path, as that is not treated as an artifact and needs 1675 # manifest path, as that is not treated as an artifact and needs
1685 # to be set separately 1676 # to be set separately
1686 most_recent = \ 1677 similar_target = \
1687 self.orm_wrapper.get_similar_target_with_image_files(target) 1678 self.orm_wrapper.get_similar_target_with_image_files(
1679 image_target)
1688 1680
1689 if most_recent: 1681 if similar_target:
1690 logger.info('image artifacts for target %s cloned from ' \ 1682 logger.info('image artifacts for target %s cloned from ' \
1691 'target %s' % (target.pk, most_recent.pk)) 1683 'target %s' % (image_target.pk, similar_target.pk))
1692 target.clone_artifacts_from(most_recent) 1684 self.orm_wrapper.clone_image_artifacts(similar_target,
1685 image_target)
1686
1687 def _get_sdk_targets(self):
1688 """
1689 Return targets which could generate SDK artifacts, i.e.
1690 "do_populate_sdk" and "do_populate_sdk_ext".
1691 """
1692 return [target for target in self.internal_state['targets'] \
1693 if target.task in ['populate_sdk', 'populate_sdk_ext']]
1694
1695 def scan_sdk_artifacts(self, event):
1696 """
1697 Note that we have to intercept an SDKArtifactInfo event from
1698 toaster.bbclass (via toasterui) to get hold of the SDK variables we
1699 need to be able to scan for files accurately: this is because
1700 variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time
1701 BuildCompleted is fired by bitbake, so we have to get those values
1702 while the build is still in progress.
1703
1704 For populate_sdk_ext, this runs twice, with two different
1705 TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the
1706 files in the SDK output directory.
1707 """
1708 sdk_vars = BuildInfoHelper._get_data_from_event(event)
1709 toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME']
1710
1711 # targets which might have created SDK artifacts
1712 sdk_targets = self._get_sdk_targets()
1713
1714 # location of SDK artifacts
1715 tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0]
1716 sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk')
1717
1718 # all files in the SDK directory
1719 artifacts = []
1720 for dir_path, _, filenames in os.walk(sdk_dir):
1721 for filename in filenames:
1722 full_path = os.path.join(dir_path, filename)
1723 if not os.path.islink(full_path):
1724 artifacts.append(full_path)
1725
1726 for sdk_target in sdk_targets:
1727 # find files in the SDK directory which haven't already been
1728 # recorded against a Target and whose basename matches
1729 # TOOLCHAIN_OUTPUTNAME
1730 for artifact_path in artifacts:
1731 basename = os.path.basename(artifact_path)
1732
1733 toolchain_match = basename.startswith(toolchain_outputname)
1734
1735 # files which match the name of the target which produced them;
1736 # for example,
1737 # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh
1738 target_match = re.search(sdk_target.target, basename)
1739
1740 # targets which produce "*-nativesdk-*" files
1741 is_ext_sdk_target = sdk_target.task in \
1742 ['do_populate_sdk_ext', 'populate_sdk_ext']
1743
1744 # SDK files which don't match the target name, i.e.
1745 # x86_64-nativesdk-libc.*
1746 # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot*
1747 is_ext_sdk_file = re.search('-nativesdk-', basename)
1748
1749 file_from_target = (toolchain_match and target_match) or \
1750 (is_ext_sdk_target and is_ext_sdk_file)
1751
1752 if file_from_target:
1753 # don't record the file if it's already been added to this
1754 # target
1755 matching_files = TargetSDKFile.objects.filter(
1756 target=sdk_target, file_name=artifact_path)
1757
1758 if matching_files.count() == 0:
1759 artifact_size = os.stat(artifact_path).st_size
1760
1761 self.orm_wrapper.save_target_sdk_file(
1762 sdk_target, artifact_path, artifact_size)
1763
1764 def clone_required_sdk_artifacts(self):
1765 """
1766 If an SDK target doesn't have any SDK artifacts, this means that
1767 the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which
1768 in turn means that the targets of this build didn't generate any new
1769 artifacts.
1770
1771 In this case, clone SDK artifacts for targets in the current build
1772 from existing targets for this build.
1773 """
1774 sdk_targets = self._get_sdk_targets()
1775 for sdk_target in sdk_targets:
1776 # only clone for SDK targets which have no TargetSDKFiles yet
1777 if sdk_target.targetsdkfile_set.all().count() == 0:
1778 similar_target = \
1779 self.orm_wrapper.get_similar_target_with_sdk_files(
1780 sdk_target)
1781 if similar_target:
1782 logger.info('SDK artifacts for target %s cloned from ' \
1783 'target %s' % (sdk_target.pk, similar_target.pk))
1784 self.orm_wrapper.clone_sdk_artifacts(similar_target,
1785 sdk_target)
1693 1786
1694 def close(self, errorcode): 1787 def close(self, errorcode):
1695 if self.brbe is not None: 1788 if self.brbe is not None:
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index d8bccdb81c..f399a7d316 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -364,7 +364,8 @@ def main(server, eventHandler, params):
364 errorcode = 1 364 errorcode = 1
365 logger.error("Command execution failed: %s", event.error) 365 logger.error("Command execution failed: %s", event.error)
366 elif isinstance(event, bb.event.BuildCompleted): 366 elif isinstance(event, bb.event.BuildCompleted):
367 buildinfohelper.scan_build_artifacts() 367 buildinfohelper.scan_image_artifacts()
368 buildinfohelper.clone_required_sdk_artifacts()
368 369
369 # turn off logging to the current build log 370 # turn off logging to the current build log
370 _close_build_log(build_log) 371 _close_build_log(build_log)
@@ -412,8 +413,8 @@ def main(server, eventHandler, params):
412 buildinfohelper.store_target_package_data(event) 413 buildinfohelper.store_target_package_data(event)
413 elif event.type == "MissedSstate": 414 elif event.type == "MissedSstate":
414 buildinfohelper.store_missed_state_tasks(event) 415 buildinfohelper.store_missed_state_tasks(event)
415 elif event.type == "ArtifactFileSize": 416 elif event.type == "SDKArtifactInfo":
416 buildinfohelper.update_artifact_image_file(event) 417 buildinfohelper.scan_sdk_artifacts(event)
417 elif event.type == "SetBRBE": 418 elif event.type == "SetBRBE":
418 buildinfohelper.brbe = buildinfohelper._get_data_from_event(event) 419 buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
419 elif event.type == "OSErrorException": 420 elif event.type == "OSErrorException":