bitbake: toaster: improve scan for SDK artifacts

SDK artifacts were previously picked up by toaster.bbclass and
notified to buildinfohelper (via toasterui). The artifacts
were then added to the Build object, so that it wasn't clear
which artifact went with which target; we were also unable
to attach SDK artifacts to a Build if they had already been
attached to a previous build.

Now, toaster.bbclass just notifies the TOOLCHAIN_OUTPUTNAME when
a populate_sdk* target completes. The scan is moved to buildinfohelper,
where we search the SDK deploy directory for files matching
TOOLCHAIN_OUTPUTNAME and attach them to targets (not builds).

If an SDK file is not produced by a target, we now look for a
similar, previously-run target which did produce artifacts.
If there is one, we clone the SDK artifacts from that target
onto the current one.

This all means that we can show SDK artifacts by target, and should
always get artifacts associated with a target, regardless of whether
it really build them.

This requires an additional model, TargetSDKFile, which tracks
the size and path of SDK artifact files with respect to Target
objects.

[YOCTO #8556]

(Bitbake rev: 5e650c611605507e1e0d1588cd5eb6535c2d34fc)

Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: bavery <brian.avery@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 138 insertions, 44 deletions

diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index e1b59c3e87..bf032ae835 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -37,7 +37,7 @@ os.environ["DJANGO_SETTINGS_MODULE"] =\
 django.setup()
 
 from orm.models import Build, Task, Recipe, Layer_Version, Layer, Target, LogMessage, HelpText
-from orm.models import Target_Image_File, BuildArtifact, TargetArtifactFile
+from orm.models import Target_Image_File, TargetKernelFile, TargetSDKFile
 from orm.models import Variable, VariableHistory
 from orm.models import Package, Package_File, Target_Installed_Package, Target_File
 from orm.models import Task_Dependency, Package_Dependency
@@ -128,6 +128,15 @@ class ORMWrapper(object):
         """
         return target.get_similar_target_with_image_files()
 
+    def get_similar_target_with_sdk_files(self, target):
+        return target.get_similar_target_with_sdk_files()
+
+    def clone_image_artifacts(self, target_from, target_to):
+        target_to.clone_image_artifacts_from(target_from)
+
+    def clone_sdk_artifacts(self, target_from, target_to):
+        target_to.clone_sdk_artifacts_from(target_from)
+
     def _timestamp_to_datetime(self, secs):
         """
         Convert timestamp in seconds to Python datetime
@@ -682,35 +691,22 @@ class ORMWrapper(object):
             logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
 
     def save_target_image_file_information(self, target_obj, file_name, file_size):
-        Target_Image_File.objects.create( target = target_obj,
-                            file_name = file_name,
-                            file_size = file_size)
+        Target_Image_File.objects.create(target=target_obj,
+            file_name=file_name, file_size=file_size)
 
-    def save_target_artifact_file(self, target_obj, file_name, file_size):
+    def save_target_kernel_file(self, target_obj, file_name, file_size):
         """
-        Save artifact file information for a Target target_obj.
-
-        Note that this doesn't include SDK artifacts, only images and
-        related files (e.g. bzImage).
+        Save kernel file (bzImage, modules*) information for a Target target_obj.
         """
-        TargetArtifactFile.objects.create(target=target_obj,
+        TargetKernelFile.objects.create(target=target_obj,
             file_name=file_name, file_size=file_size)
 
-    def save_artifact_information(self, build_obj, file_name, file_size):
+    def save_target_sdk_file(self, target_obj, file_name, file_size):
         """
-        TODO this is currently used to save SDK artifacts to the database,
-        but will be replaced in future once SDK artifacts are associated with
-        Target objects (as they eventually should be)
+        Save SDK artifacts to the database, associating them with a
+        Target object.
         """
-        # do not update artifacts found in other builds
-        if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
-            return
-
-        # do not update artifact if already a target artifact file for that path
-        if TargetArtifactFile.objects.filter(file_name = file_name).count() > 0:
-            return
-
-        BuildArtifact.objects.create(build=build_obj, file_name=file_name,
+        TargetSDKFile.objects.create(target=target_obj, file_name=file_name,
             file_size=file_size)
 
     def create_logmessage(self, log_information):
@@ -1085,11 +1081,6 @@ class BuildInfoHelper(object):
 
         return self.brbe
 
-    def update_artifact_image_file(self, event):
-        evdata = BuildInfoHelper._get_data_from_event(event)
-        for artifact_path in evdata.keys():
-            self.orm_wrapper.save_artifact_information(self.internal_state['build'], artifact_path, evdata[artifact_path])
-
     def update_build_information(self, event, errors, warnings, taskfailures):
         if 'build' in self.internal_state:
             self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
@@ -1568,9 +1559,9 @@ class BuildInfoHelper(object):
 
         return image_files
 
-    def scan_build_artifacts(self):
+    def scan_image_artifacts(self):
         """
-        Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them
+        Scan for built image artifacts in DEPLOY_DIR_IMAGE and associate them
         with a Target object in self.internal_state['targets'].
 
         We have two situations to handle:
@@ -1615,7 +1606,7 @@ class BuildInfoHelper(object):
         # filter out anything which isn't an image target
         image_targets = [target for target in targets if target.is_image]
 
-        for target in image_targets:
+        for image_target in image_targets:
             # this is set to True if we find at least one file relating to
             # this target; if this remains False after the scan, we copy the
             # files from the most-recent Target with the same target + machine
@@ -1627,7 +1618,7 @@ class BuildInfoHelper(object):
             # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
             # we need to change it to
             # <TARGET>-<MACHINE>-<BUILDNAME>
-            real_image_name = re.sub(r'^defaultpkgname', target.target,
+            real_image_name = re.sub(r'^defaultpkgname', image_target.target,
                 image_name)
 
             image_license_manifest_path = os.path.join(
@@ -1651,7 +1642,7 @@ class BuildInfoHelper(object):
 
                     # note that the artifact will only be saved against this
                     # build if it hasn't been already
-                    self.orm_wrapper.save_target_artifact_file(target,
+                    self.orm_wrapper.save_target_kernel_file(image_target,
                         artifact_path, artifact_size)
 
                 # store the license manifest path on the target
@@ -1659,8 +1650,8 @@ class BuildInfoHelper(object):
                 license_path = os.path.join(license_directory,
                     real_image_name, 'license.manifest')
 
-                self.orm_wrapper.update_target_set_license_manifest(target,
-                    license_path)
+                self.orm_wrapper.update_target_set_license_manifest(
+                    image_target, license_path)
 
             # scan the directory for image files relating to this build
             # (via real_image_name); note that we don't have to set
@@ -1675,7 +1666,7 @@ class BuildInfoHelper(object):
 
             for image_file in image_files:
                 self.orm_wrapper.save_target_image_file_information(
-                    target, image_file['path'], image_file['size'])
+                    image_target, image_file['path'], image_file['size'])
 
             if not has_files:
                 # copy image files and build artifacts from the
@@ -1683,13 +1674,115 @@ class BuildInfoHelper(object):
                 # same target + machine as this Target; also copy the license
                 # manifest path, as that is not treated as an artifact and needs
                 # to be set separately
-                most_recent = \
-                    self.orm_wrapper.get_similar_target_with_image_files(target)
+                similar_target = \
+                    self.orm_wrapper.get_similar_target_with_image_files(
+                        image_target)
 
-                if most_recent:
+                if similar_target:
                     logger.info('image artifacts for target %s cloned from ' \
-                        'target %s' % (target.pk, most_recent.pk))
-                    target.clone_artifacts_from(most_recent)
+                        'target %s' % (image_target.pk, similar_target.pk))
+                    self.orm_wrapper.clone_image_artifacts(similar_target,
+                        image_target)
+
+    def _get_sdk_targets(self):
+        """
+        Return targets which could generate SDK artifacts, i.e.
+        "do_populate_sdk" and "do_populate_sdk_ext".
+        """
+        return [target for target in self.internal_state['targets'] \
+            if target.task in ['populate_sdk', 'populate_sdk_ext']]
+
+    def scan_sdk_artifacts(self, event):
+        """
+        Note that we have to intercept an SDKArtifactInfo event from
+        toaster.bbclass (via toasterui) to get hold of the SDK variables we
+        need to be able to scan for files accurately: this is because
+        variables like TOOLCHAIN_OUTPUTNAME have reset to None by the time
+        BuildCompleted is fired by bitbake, so we have to get those values
+        while the build is still in progress.
+
+        For populate_sdk_ext, this runs twice, with two different
+        TOOLCHAIN_OUTPUTNAME settings, each of which will capture some of the
+        files in the SDK output directory.
+        """
+        sdk_vars = BuildInfoHelper._get_data_from_event(event)
+        toolchain_outputname = sdk_vars['TOOLCHAIN_OUTPUTNAME']
+
+        # targets which might have created SDK artifacts
+        sdk_targets = self._get_sdk_targets()
+
+        # location of SDK artifacts
+        tmpdir = self.server.runCommand(['getVariable', 'TMPDIR'])[0]
+        sdk_dir = os.path.join(tmpdir, 'deploy', 'sdk')
+
+        # all files in the SDK directory
+        artifacts = []
+        for dir_path, _, filenames in os.walk(sdk_dir):
+            for filename in filenames:
+                full_path = os.path.join(dir_path, filename)
+                if not os.path.islink(full_path):
+                    artifacts.append(full_path)
+
+        for sdk_target in sdk_targets:
+            # find files in the SDK directory which haven't already been
+            # recorded against a Target and whose basename matches
+            # TOOLCHAIN_OUTPUTNAME
+            for artifact_path in artifacts:
+                basename = os.path.basename(artifact_path)
+
+                toolchain_match = basename.startswith(toolchain_outputname)
+
+                # files which match the name of the target which produced them;
+                # for example,
+                # poky-glibc-x86_64-core-image-sato-i586-toolchain-ext-2.1+snapshot.sh
+                target_match = re.search(sdk_target.target, basename)
+
+                # targets which produce "*-nativesdk-*" files
+                is_ext_sdk_target = sdk_target.task in \
+                    ['do_populate_sdk_ext', 'populate_sdk_ext']
+
+                # SDK files which don't match the target name, i.e.
+                # x86_64-nativesdk-libc.*
+                # poky-glibc-x86_64-buildtools-tarball-i586-buildtools-nativesdk-standalone-2.1+snapshot*
+                is_ext_sdk_file = re.search('-nativesdk-', basename)
+
+                file_from_target = (toolchain_match and target_match) or \
+                    (is_ext_sdk_target and is_ext_sdk_file)
+
+                if file_from_target:
+                    # don't record the file if it's already been added to this
+                    # target
+                    matching_files = TargetSDKFile.objects.filter(
+                        target=sdk_target, file_name=artifact_path)
+
+                    if matching_files.count() == 0:
+                        artifact_size = os.stat(artifact_path).st_size
+
+                        self.orm_wrapper.save_target_sdk_file(
+                            sdk_target, artifact_path, artifact_size)
+
+    def clone_required_sdk_artifacts(self):
+        """
+        If an SDK target doesn't have any SDK artifacts, this means that
+        the postfuncs of populate_sdk or populate_sdk_ext didn't fire, which
+        in turn means that the targets of this build didn't generate any new
+        artifacts.
+
+        In this case, clone SDK artifacts for targets in the current build
+        from existing targets for this build.
+        """
+        sdk_targets = self._get_sdk_targets()
+        for sdk_target in sdk_targets:
+            # only clone for SDK targets which have no TargetSDKFiles yet
+            if sdk_target.targetsdkfile_set.all().count() == 0:
+                similar_target = \
+                    self.orm_wrapper.get_similar_target_with_sdk_files(
+                        sdk_target)
+                if similar_target:
+                    logger.info('SDK artifacts for target %s cloned from ' \
+                        'target %s' % (sdk_target.pk, similar_target.pk))
+                    self.orm_wrapper.clone_sdk_artifacts(similar_target,
+                        sdk_target)
 
     def close(self, errorcode):
         if self.brbe is not None:
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index d8bccdb81c..f399a7d316 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -364,7 +364,8 @@ def main(server, eventHandler, params):
                     errorcode = 1
                     logger.error("Command execution failed: %s", event.error)
                 elif isinstance(event, bb.event.BuildCompleted):
-                    buildinfohelper.scan_build_artifacts()
+                    buildinfohelper.scan_image_artifacts()
+                    buildinfohelper.clone_required_sdk_artifacts()
 
                 # turn off logging to the current build log
                 _close_build_log(build_log)
@@ -412,8 +413,8 @@ def main(server, eventHandler, params):
                     buildinfohelper.store_target_package_data(event)
                 elif event.type == "MissedSstate":
                     buildinfohelper.store_missed_state_tasks(event)
-                elif event.type == "ArtifactFileSize":
-                    buildinfohelper.update_artifact_image_file(event)
+                elif event.type == "SDKArtifactInfo":
+                    buildinfohelper.scan_sdk_artifacts(event)
                 elif event.type == "SetBRBE":
                     buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
                 elif event.type == "OSErrorException":