bitbake: toaster logger: refactor recipe and layer file paths

This refactoring brings the "local_path" of the
layer from the Layer object to the Layer_Version object, which
is more appropriate as different checkouts of the same
Layer may live in different directories.

This enables us to store Recipe file paths relative to a Layer_Version
at all times, aleviating the need to store full file paths in the
database. We also turn the prefix of the path (e.g. virtual:native
path name space) into a pathflag field.

In turn, this solves the problem of mis-identification of tasks based
on the recipe file paths, since we can also match the namespace of the
file paths on the recipe files.

[YOCTO #7594]

(Bitbake rev: ec43dc569e370767c709dec225cbee0c99151c19)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 32 insertions, 15 deletions

diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 647d663cb5..6812a52b7b 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -221,12 +221,12 @@ class ORMWrapper(object):
     def get_update_recipe_object(self, recipe_information, must_exist = False):
         assert 'layer_version' in recipe_information
         assert 'file_path' in recipe_information
+        assert 'pathflags' in recipe_information
 
-        if recipe_information['file_path'].startswith(recipe_information['layer_version'].layer.local_path):
-            recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].layer.local_path):].lstrip("/")
+        assert not recipe_information['file_path'].startswith("/")      # we should have layer-relative paths at all times
 
         recipe_object, created = self._cached_get_or_create(Recipe, layer_version=recipe_information['layer_version'],
-                                     file_path=recipe_information['file_path'])
+                                     file_path=recipe_information['file_path'], pathflags = recipe_information['pathflags'])
         if created and must_exist:
             raise NotExisting("Recipe object created when expected to exist", recipe_information)
 
@@ -247,13 +247,15 @@ class ORMWrapper(object):
         assert 'branch' in layer_version_information
         assert 'commit' in layer_version_information
         assert 'priority' in layer_version_information
+        assert 'local_path' in layer_version_information
 
         layer_version_object, created = Layer_Version.objects.get_or_create(
                                     build = build_obj,
                                     layer = layer_obj,
                                     branch = layer_version_information['branch'],
                                     commit = layer_version_information['commit'],
-                                    priority = layer_version_information['priority']
+                                    priority = layer_version_information['priority'],
+                                    local_path = layer_version_information['local_path'],
                                     )
 
         self.layer_version_objects.append(layer_version_object)
@@ -262,13 +264,11 @@ class ORMWrapper(object):
 
     def get_update_layer_object(self, layer_information, brbe):
         assert 'name' in layer_information
-        assert 'local_path' in layer_information
         assert 'layer_index_url' in layer_information
 
         if brbe is None:
             layer_object, created = Layer.objects.get_or_create(
                                 name=layer_information['name'],
-                                local_path=layer_information['local_path'],
                                 layer_index_url=layer_information['layer_index_url'])
             return layer_object
         else:
@@ -297,7 +297,6 @@ class ORMWrapper(object):
                     for pl in buildrequest.project.projectlayer_set.filter(layercommit__layer__name = brl.name):
                         if pl.layercommit.layer.vcs_url == brl.giturl :
                             layer = pl.layercommit.layer
-                            layer.local_path = layer_information['local_path']
                             layer.save()
                             return layer
 
@@ -687,12 +686,12 @@ class BuildInfoHelper(object):
         if self.brbe is None:
             def _slkey_interactive(layer_version):
                 assert isinstance(layer_version, Layer_Version)
-                return len(layer_version.layer.local_path)
+                return len(layer_version.local_path)
 
             # Heuristics: we always match recipe to the deepest layer path in the discovered layers
             for lvo in sorted(self.orm_wrapper.layer_version_objects, reverse=True, key=_slkey_interactive):
                 # we can match to the recipe file path
-                if path.startswith(lvo.layer.local_path):
+                if path.startswith(lvo.local_path):
                     return lvo
 
         else:
@@ -721,7 +720,7 @@ class BuildInfoHelper(object):
         logger.warn("Could not match layer version for recipe path %s : %s" % (path, self.orm_wrapper.layer_version_objects))
 
         #mockup the new layer
-        unknown_layer, created = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", local_path="/", layer_index_url="")
+        unknown_layer, created = Layer.objects.get_or_create(name="__FIXME__unidentified_layer", layer_index_url="")
         unknown_layer_version_obj, created = Layer_Version.objects.get_or_create(layer = unknown_layer, build = self.internal_state['build'])
 
         # append it so we don't run into this error again and again
@@ -731,11 +730,20 @@ class BuildInfoHelper(object):
 
     def _get_recipe_information_from_taskfile(self, taskfile):
         localfilepath = taskfile.split(":")[-1]
+        filepath_flags = ":".join(sorted(taskfile.split(":")[:-1]))
         layer_version_obj = self._get_layer_version_for_path(localfilepath)
 
+
+
         recipe_info = {}
         recipe_info['layer_version'] = layer_version_obj
-        recipe_info['file_path'] = taskfile
+        recipe_info['file_path'] = localfilepath
+        recipe_info['pathflags'] = filepath_flags
+
+        if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path):
+            recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/")
+        else:
+            raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path))
 
         return recipe_info
 
@@ -787,6 +795,7 @@ class BuildInfoHelper(object):
         for layer in layerinfos:
             try:
                 self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)] = layerinfos[layer]['version']
+                self.internal_state['lvs'][self.orm_wrapper.get_update_layer_object(layerinfos[layer], self.brbe)]['local_path'] = layerinfos[layer]['local_path']
             except NotExisting as nee:
                 logger.warn("buildinfohelper: cannot identify layer exception:%s " % nee)
 
@@ -899,8 +908,8 @@ class BuildInfoHelper(object):
 
             recipe_information = self._get_recipe_information_from_taskfile(taskfile)
             try:
-                if recipe_information['file_path'].startswith(recipe_information['layer_version'].layer.local_path):
-                    recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].layer.local_path):].lstrip("/")
+                if recipe_information['file_path'].startswith(recipe_information['layer_version'].local_path):
+                    recipe_information['file_path'] = recipe_information['file_path'][len(recipe_information['layer_version'].local_path):].lstrip("/")
 
                 recipe_object = Recipe.objects.get(layer_version = recipe_information['layer_version'],
                             file_path__endswith = recipe_information['file_path'],
@@ -1051,8 +1060,9 @@ class BuildInfoHelper(object):
         self.internal_state['recipes'] = {}
         for pn in event._depgraph['pn']:
 
-            file_name = event._depgraph['pn'][pn]['filename']
-            layer_version_obj = self._get_layer_version_for_path(file_name.split(":")[-1])
+            file_name = event._depgraph['pn'][pn]['filename'].split(":")[-1]
+            pathflags = ":".join(sorted(event._depgraph['pn'][pn]['filename'].split(":")[:-1]))
+            layer_version_obj = self._get_layer_version_for_path(file_name)
 
             assert layer_version_obj is not None
 
@@ -1082,6 +1092,13 @@ class BuildInfoHelper(object):
                 recipe_info['bugtracker'] = event._depgraph['pn'][pn]['bugtracker']
 
             recipe_info['file_path'] = file_name
+            recipe_info['pathflags'] = pathflags
+
+            if recipe_info['file_path'].startswith(recipe_info['layer_version'].local_path):
+                recipe_info['file_path'] = recipe_info['file_path'][len(recipe_info['layer_version'].local_path):].lstrip("/")
+            else:
+                raise RuntimeError("Recipe file path %s is not under layer version at %s" % (recipe_info['file_path'], recipe_info['layer_version'].local_path))
+
             recipe = self.orm_wrapper.get_update_recipe_object(recipe_info)
             recipe.is_image = False
             if 'inherits' in event._depgraph['pn'][pn].keys():