2 files changed, 181 insertions, 26 deletions

diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 447670cb8b..8bdc9cc0a7 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -678,6 +678,16 @@ class ORMWrapper(object):
                             file_name = file_name,
                             file_size = file_size)
 
+    def save_artifact_information_no_dedupe(self, build_obj, file_name, file_size):
+        """
+        Save artifact information without checking for duplicate paths;
+        this is used when we are saving data about an artifact which was
+        generated by a previous build but which is also relevant to this build,
+        e.g. a bzImage file.
+        """
+        BuildArtifact.objects.create(build=build_obj, file_name=file_name,
+            file_size=file_size)
+
     def save_artifact_information(self, build_obj, file_name, file_size):
         # we skip the image files from other builds
         if Target_Image_File.objects.filter(file_name = file_name).count() > 0:
@@ -687,7 +697,8 @@ class ORMWrapper(object):
         if BuildArtifact.objects.filter(file_name = file_name).count() > 0:
             return
 
-        BuildArtifact.objects.create(build = build_obj, file_name = file_name, file_size = file_size)
+        self.save_artifact_information_no_dedupe(self, build_obj, file_name,
+            file_size)
 
     def create_logmessage(self, log_information):
         assert 'build' in log_information
@@ -1061,17 +1072,6 @@ class BuildInfoHelper(object):
 
         return self.brbe
 
-
-    def update_target_image_file(self, event):
-        evdata = BuildInfoHelper._get_data_from_event(event)
-
-        for t in self.internal_state['targets']:
-            if t.is_image == True:
-                output_files = list(evdata.keys())
-                for output in output_files:
-                    if t.target in output and 'rootfs' in output and not output.endswith(".manifest"):
-                        self.orm_wrapper.save_target_image_file_information(t, output, evdata[output])
-
     def update_artifact_image_file(self, event):
         evdata = BuildInfoHelper._get_data_from_event(event)
         for artifact_path in evdata.keys():
@@ -1081,16 +1081,6 @@ class BuildInfoHelper(object):
         if 'build' in self.internal_state:
             self.orm_wrapper.update_build_object(self.internal_state['build'], errors, warnings, taskfailures)
 
-
-    def store_license_manifest_path(self, event):
-        deploy_dir = BuildInfoHelper._get_data_from_event(event)['deploy_dir']
-        image_name = BuildInfoHelper._get_data_from_event(event)['image_name']
-        path = deploy_dir + "/licenses/" + image_name + "/license.manifest"
-        for target in self.internal_state['targets']:
-            if target.target in image_name:
-                self.orm_wrapper.update_target_set_license_manifest(target, path)
-
-
     def store_started_task(self, event):
         assert isinstance(event, (bb.runqueue.sceneQueueTaskStarted, bb.runqueue.runQueueTaskStarted, bb.runqueue.runQueueTaskSkipped))
         assert 'taskfile' in vars(event)
@@ -1506,6 +1496,173 @@ class BuildInfoHelper(object):
 
         self.orm_wrapper.create_logmessage(log_information)
 
+    def _get_files_from_image_license(self, image_license_manifest_path):
+        """
+        Find the FILES line in the image_license.manifest file,
+        which has the basenames of the bzImage and modules files
+        in this format:
+        FILES: bzImage--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.bin modules--4.4.11+git0+3a5f494784_53e84104c5-r0-qemux86-20160603165040.tgz
+        """
+        files = []
+        with open(image_license_manifest_path) as image_license:
+            for line in image_license:
+                if line.startswith('FILES'):
+                    files_str = line.split(':')[1].strip()
+                    files_str = re.sub(r' {2,}', ' ', files_str)
+                    files = files_str.split(' ')
+        return files
+
+    def _endswith(self, str_to_test, endings):
+        """
+        Returns True if str ends with one of the strings in the list
+        endings, False otherwise
+        """
+        endswith = False
+        for ending in endings:
+            if str_to_test.endswith(ending):
+                endswith = True
+                break
+        return endswith
+
+    def _get_image_files(self, deploy_dir_image, image_name, image_file_extensions):
+        """
+        Find files in deploy_dir_image whose basename starts with the
+        string image_name and ends with one of the strings in
+        image_file_extensions.
+
+        Returns a list of file dictionaries like
+
+        [
+            {
+                'path': '/path/to/image/file',
+                'size': <file size in bytes>
+            }
+        ]
+        """
+        image_files = []
+
+        for dirpath, _, filenames in os.walk(deploy_dir_image):
+            for filename in filenames:
+                if filename.startswith(image_name) and \
+                self._endswith(filename, image_file_extensions):
+                    image_file_path = os.path.join(dirpath, filename)
+                    image_file_size = os.stat(image_file_path).st_size
+
+                    image_files.append({
+                        'path': image_file_path,
+                        'size': image_file_size
+                    })
+
+        return image_files
+
+    def scan_build_artifacts(self):
+        """
+        Scan for build artifacts in DEPLOY_DIR_IMAGE and associate them
+        with a Target object in self.internal_state['targets'].
+
+        We have two situations to handle:
+
+        1. This is the first time a target + machine has been built, so
+        add files from the DEPLOY_DIR_IMAGE to the target.
+
+        OR
+
+        2. There are no files for the target, so copy them from a
+        previous build with the same target + machine.
+        """
+        deploy_dir_image = \
+            self.server.runCommand(['getVariable', 'DEPLOY_DIR_IMAGE'])[0]
+
+        # if there's no DEPLOY_DIR_IMAGE, there aren't going to be
+        # any build artifacts, so we can return immediately
+        if not deploy_dir_image:
+            return
+
+        buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
+        machine =  self.server.runCommand(['getVariable', 'MACHINE'])[0]
+        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+
+        # location of the image_license.manifest files for this build;
+        # note that this file is only produced if an image is produced
+        license_directory = \
+            self.server.runCommand(['getVariable', 'LICENSE_DIRECTORY'])[0]
+
+        # file name extensions for image files
+        image_file_extensions_unique = {}
+        image_fstypes = self.server.runCommand(
+            ['getVariable', 'IMAGE_FSTYPES'])[0]
+        if image_fstypes != None:
+            image_types_str = image_fstypes.strip()
+            image_file_extensions = re.sub(r' {2,}', ' ', image_types_str)
+            image_file_extensions_unique = set(image_file_extensions.split(' '))
+
+        targets = self.internal_state['targets']
+        image_targets = [target for target in targets if target.is_image]
+        for target in image_targets:
+            # this is set to True if we find at least one file relating to
+            # this target; if this remains False after the scan, we copy the
+            # files from the most-recent Target with the same target + machine
+            # onto this Target instead
+            has_files = False
+
+            # we construct this because by the time we reach
+            # BuildCompleted, this has reset to
+            # 'defaultpkgname-<MACHINE>-<BUILDNAME>';
+            # we need to change it to
+            # <TARGET>-<MACHINE>-<BUILDNAME>
+            real_image_name = re.sub(r'^defaultpkgname', target.target,
+                image_name)
+
+            image_license_manifest_path = os.path.join(
+                license_directory,
+                real_image_name,
+                'image_license.manifest')
+
+            # if image_license.manifest exists, we can read the names of bzImage
+            # and modules files for this build from it, then look for them
+            # in the DEPLOY_DIR_IMAGE; note that this file is only produced
+            # if an image file was produced
+            if os.path.isfile(image_license_manifest_path):
+                has_files = True
+
+                basenames = self._get_files_from_image_license(
+                    image_license_manifest_path)
+
+                for basename in basenames:
+                    artifact_path = os.path.join(deploy_dir_image, basename)
+                    artifact_size = os.stat(artifact_path).st_size
+
+                    self.orm_wrapper.save_artifact_information_no_dedupe(
+                        self.internal_state['build'], artifact_path,
+                        artifact_size)
+
+                # store the license manifest path on the target
+                # (this file is also created any time an image file is created)
+                license_path = os.path.join(license_directory,
+                    real_image_name, 'license.manifest')
+
+                self.orm_wrapper.update_target_set_license_manifest(target,
+                    license_path)
+
+            # scan the directory for image files relating to this build
+            # (via real_image_name); note that we don't have to set
+            # has_files = True, as searching for the license manifest file
+            # will already have set it to true if at least one image file was
+            # produced
+            image_files = self._get_image_files(deploy_dir_image,
+                real_image_name, image_file_extensions_unique)
+
+            for image_file in image_files:
+                self.orm_wrapper.save_target_image_file_information(
+                    target, image_file['path'], image_file['size'])
+
+            if not has_files:
+                # TODO copy artifact and image files from the
+                # most-recently-built Target with the same target + machine
+                # as this Target; also copy the license manifest path,
+                # as that is treated differently
+                pass
+
     def close(self, errorcode):
         if self.brbe is not None:
             self._store_build_done(errorcode)
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index 5382935f82..d8bccdb81c 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -363,6 +363,8 @@ def main(server, eventHandler, params):
                     errors += 1
                     errorcode = 1
                     logger.error("Command execution failed: %s", event.error)
+                elif isinstance(event, bb.event.BuildCompleted):
+                    buildinfohelper.scan_build_artifacts()
 
                 # turn off logging to the current build log
                 _close_build_log(build_log)
@@ -410,12 +412,8 @@ def main(server, eventHandler, params):
                     buildinfohelper.store_target_package_data(event)
                 elif event.type == "MissedSstate":
                     buildinfohelper.store_missed_state_tasks(event)
-                elif event.type == "ImageFileSize":
-                    buildinfohelper.update_target_image_file(event)
                 elif event.type == "ArtifactFileSize":
                     buildinfohelper.update_artifact_image_file(event)
-                elif event.type == "LicenseManifestPath":
-                    buildinfohelper.store_license_manifest_path(event)
                 elif event.type == "SetBRBE":
                     buildinfohelper.brbe = buildinfohelper._get_data_from_event(event)
                 elif event.type == "OSErrorException":