bitbake: toaster: do image and artifact scan on BuildCompleted

Move the image and artifact scan code from toaster.bbclass and
consolidate its logic with the existing logic in buildinfohelper.

Remove handler setup for events which used to be fired from
toaster.bbclass but which are now handled directly by buildinfohelper.

[YOCTO #8556]

(Bitbake rev: f0085cd554604cfff4a3f40a34825fbb6878004f)

Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: bavery <brian.avery@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 181 insertions, 26 deletions

diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 447670cb8b..8bdc9cc0a7 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -678,6 +678,16 @@ class ORMWrapper(object):
                             file_name = file_name,
                             file_size = file_size)
 
+    def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size):
+        """
+        Save artifact information without checking for duplicate paths;
+        this is used when we are saving data about an artifact which was
+        generated by a previous build but which is also relevant to this build,
+        e.g. a bzImage file.
+        """
+        BuildArtifact.objects.create(build=build_obj, file_name=file_name,
+            file_size=file_size)
+
     def save_artifact_information(self, build_obj, file_name, file_size):
         # we skip the image files from other builds
         if Target_Image_File.objects.filter(file_name = file_name).count() > 0:
@@ -687,7 +697,8 @@ class ORMWrapper(object):
         if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
             return
 
-        BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size)
+        self.save_artifact_information_no_dedupe(self, build_obj, file_name,
+            file_size)
 
     def create_logmessage(self, log_information):
         assert 'build' in log_information
@@ -1061,17 +1072,6 @@ class BuildInfoHelper(object):
 
         return self.brbe
 
-
-    def update_target_image_file(self, event):
-        evdata = BuildInfoHelper._get_data_from_event(event)
-
-        for t in self.internal_state['targets']:
-            if t.is_image == True:
-                output_files = list(evdata.keys())
-                for output in output_files:
-                    if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
-                        self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
-
     def update_artifact_image_file(self, event):
         evdata = BuildInfoHelper._get_data_from_event(event)
         for artifact_path in evdata.keys():
@@ -1081,16 +1081,6 @@ class BuildInfoHelper(object):
         if 'build' in self.internal_state:
             self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
 
-
-    def store_license_manifest_path(self, event):
-        deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir']
-        image_name = BuildInfoHelper._get_data_from_event(event)['image_name']
-        path = deploy_dir + "/licenses/" + image_name + "/license.manifest"
-        for target in self.internal_state['targets']:
-            if target.target in image_name:
-                self.orm_wrapper.update_target_set_license_manifest(target, path)
-
-
     def store_started_task(self, event):
         assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
         assert 'taskfile' in vars(event)
@@ -1506,6 +1496,173 @@ class BuildInfoHelper(object):
 
         self.orm_wrapper.create_logmessage(log_information)
 
+    def _get_files_from_image_license(self, image_license_manifest_path):
+        """
+        Find the FILES line in the image_license.manifest file,
+        which has the basenames of the bzImage and modules files
+        in this format:
+        FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz
+        """
+        files = []
+        with open(image_license_manifest_path) as image_license:
+            for line in image_license:
+                if line.startswith('FILES'):
+                    files_str = line.split(':')[1].strip()
+                    files_str = re.sub(r' {2,}', ' ', files_str)
+                    files = files_str.split(' ')
+        return files
+
+    def _endswith(self, str_to_test, endings):
+        """
+        Returns True if str ends with one of the strings in the list
+        endings, False otherwise
+        """
+        endswith = False
+        for ending in endings:
+            if str_to_test.endswith(ending):
+                endswith = True
+                break
+        return endswith
+
+    def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions):
+        """
+        Find files in deploy_dir_image whose basename starts with the
+        string image_name and ends with one of the strings in
+        image_file_extensions.
+
+        Returns a list of file dictionaries like
+
+        [
+            {
+                'path': '/path/to/image/file',
+                'size': <file size in bytes>
+            }
+        ]
+        """
+        image_files = []
+
+        for dirpath, _, filenames in os.walk(deploy_dir_image):
+            for filename in filenames:
+                if filename.startswith(image_name) and \
+                self._endswith(filename, image_file_extensions):
+                    image_file_path = os.path.join(dirpath, filename)
+                    image_file_size = os.stat(image_file_path).st_size
+
+                    image_files.append({
+                        'path': image_file_path,
+                        'size': image_file_size
+                    })
+
+        return image_files
+
+    def scan_build_artifacts(self):
+        """
+        Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them
+        with a Target object in self.internal_state['targets'].
+
+        We have two situations to handle:
+
+        1. This is the first time a target + machine has been built, so
+        add files from the DEPLOY_DIR_IMAGE to the target.
+
+        OR
+
+        2. There are no files for the target, so copy them from a
+        previous build with the same target + machine.
+        """
+        deploy_dir_image = \
+            self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
+
+        # if there's no DEPLOY_DIR_IMAGE, there aren't going to be
+        # any build artifacts, so we can return immediately
+        if not deploy_dir_image:
+            return
+
+        buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
+        machine =  self.server.runCommand(['getVariable', 'MACHINE'])[0]
+        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+
+        # location of the image_license.manifest files for this build;
+        # note that this file is only produced if an image is produced
+        license_directory = \
+            self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0]
+
+        # file name extensions for image files
+        image_file_extensions_unique = {}
+        image_fstypes = self.server.runCommand(
+            ['getVariable', 'IMAGE_FSTYPES'])[0]
+        if image_fstypes != None:
+            image_types_str = image_fstypes.strip()
+            image_file_extensions = re.sub(r' {2,}', ' ', image_types_str)
+            image_file_extensions_unique = set(image_file_extensions.split(' '))
+
+        targets = self.internal_state['targets']
+        image_targets = [target for target in targets if target.is_image]
+        for target in image_targets:
+            # this is set to True if we find at least one file relating to
+            # this target; if this remains False after the scan, we copy the
+            # files from the most-recent Target with the same target + machine
+            # onto this Target instead
+            has_files = False
+
+            # we construct this because by the time we reach
+            # BuildCompleted, this has reset to
+            # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
+            # we need to change it to
+            # <TARGET>-<MACHINE>-<BUILDNAME>
+            real_image_name = re.sub(r'^defaultpkgname', target.target,
+                image_name)
+
+            image_license_manifest_path = os.path.join(
+                license_directory,
+                real_image_name,
+                'image_license.manifest')
+
+            # if image_license.manifest exists, we can read the names of bzImage
+            # and modules files for this build from it, then look for them
+            # in the DEPLOY_DIR_IMAGE; note that this file is only produced
+            # if an image file was produced
+            if os.path.isfile(image_license_manifest_path):
+                has_files = True
+
+                basenames = self._get_files_from_image_license(
+                    image_license_manifest_path)
+
+                for basename in basenames:
+                    artifact_path = os.path.join(deploy_dir_image, basename)
+                    artifact_size = os.stat(artifact_path).st_size
+
+                    self.orm_wrapper.save_artifact_information_no_dedupe(
+                        self.internal_state['build'], artifact_path,
+                        artifact_size)
+
+                # store the license manifest path on the target
+                # (this file is also created any time an image file is created)
+                license_path = os.path.join(license_directory,
+                    real_image_name, 'license.manifest')
+
+                self.orm_wrapper.update_target_set_license_manifest(target,
+                    license_path)
+
+            # scan the directory for image files relating to this build
+            # (via real_image_name); note that we don't have to set
+            # has_files = True, as searching for the license manifest file
+            # will already have set it to true if at least one image file was
+            # produced
+            image_files = self._get_image_files(deploy_dir_image,
+                real_image_name, image_file_extensions_unique)
+
+            for image_file in image_files:
+                self.orm_wrapper.save_target_image_file_information(
+                    target, image_file['path'], image_file['size'])
+
+            if not has_files:
+                # TODO copy artifact and image files from the
+                # most-recently-built Target with the same target + machine
+                # as this Target; also copy the license manifest path,
+                # as that is treated differently
+                pass
+
     def close(self, errorcode):
         if self.brbe is not None:
             self._store_build_done(errorcode)
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index 5382935f82..d8bccdb81c 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -363,6 +363,8 @@ def main(server, eventHandler, params):
                     errors += 1
                     errorcode = 1
                     logger.error("Command execution failed: %s", event.error)
+                elif isinstance(event, bb.event.BuildCompleted):
+                    buildinfohelper.scan_build_artifacts()
 
                 # turn off logging to the current build log
                 _close_build_log(build_log)
@@ -410,12 +412,8 @@ def main(server, eventHandler, params):
                     buildinfohelper.store_target_package_data(event)
                 elif event.type == "MissedSstate":
                     buildinfohelper.store_missed_state_tasks(event)
-                elif event.type == "ImageFileSize":
-                    buildinfohelper.update_target_image_file(event)
                 elif event.type == "ArtifactFileSize":
                     buildinfohelper.update_artifact_image_file(event)
-                elif event.type == "LicenseManifestPath":
-                    buildinfohelper.store_license_manifest_path(event)
                 elif event.type == "SetBRBE":
                     buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
                 elif event.type == "OSErrorException":