4 files changed, 138 insertions, 17 deletions

diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py
index fe545607bb..86aadf39a6 100644
--- a/scripts/lib/checklayer/__init__.py
+++ b/scripts/lib/checklayer/__init__.py
@@ -16,6 +16,7 @@ class LayerType(Enum):
     BSP = 0
     DISTRO = 1
     SOFTWARE = 2
+    CORE = 3
     ERROR_NO_LAYER_CONF = 98
     ERROR_BSP_DISTRO = 99
 
@@ -43,7 +44,7 @@ def _get_layer_collections(layer_path, lconf=None, data=None):
 
     ldata.setVar('LAYERDIR', layer_path)
     try:
-        ldata = bb.parse.handle(lconf, ldata, include=True)
+        ldata = bb.parse.handle(lconf, ldata, include=True, baseconfig=True)
     except:
         raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path)
     ldata.expandVarref('LAYERDIR')
@@ -106,7 +107,13 @@ def _detect_layer(layer_path):
         if distros:
             is_distro = True
 
-    if is_bsp and is_distro:
+    layer['collections'] = _get_layer_collections(layer['path'])
+
+    if layer_name == "meta" and "core" in layer['collections']:
+        layer['type'] = LayerType.CORE
+        layer['conf']['machines'] = machines
+        layer['conf']['distros'] = distros
+    elif is_bsp and is_distro:
         layer['type'] = LayerType.ERROR_BSP_DISTRO
     elif is_bsp:
         layer['type'] = LayerType.BSP
@@ -117,8 +124,6 @@ def _detect_layer(layer_path):
     else:
         layer['type'] = LayerType.SOFTWARE
 
-    layer['collections'] = _get_layer_collections(layer['path'])
-
     return layer
 
 def detect_layers(layer_directories, no_auto):
@@ -146,7 +151,7 @@ def detect_layers(layer_directories, no_auto):
 
     return layers
 
-def _find_layer_depends(depend, layers):
+def _find_layer(depend, layers):
     for layer in layers:
         if 'collections' not in layer:
             continue
@@ -156,7 +161,28 @@ def _find_layer_depends(depend, layers):
                 return layer
     return None
 
-def add_layer_dependencies(bblayersconf, layer, layers, logger):
+def sanity_check_layers(layers, logger):
+    """
+    Check that we didn't find duplicate collection names, as the layer that will
+    be used is non-deterministic. The precise check is duplicate collections
+    with different patterns, as the same pattern being repeated won't cause
+    problems.
+    """
+    import collections
+
+    passed = True
+    seen = collections.defaultdict(set)
+    for layer in layers:
+        for name, data in layer.get("collections", {}).items():
+            seen[name].add(data["pattern"])
+
+    for name, patterns in seen.items():
+        if len(patterns) > 1:
+            passed = False
+            logger.error("Collection %s found multiple times: %s" % (name, ", ".join(patterns)))
+    return passed
+
+def get_layer_dependencies(layer, layers, logger):
     def recurse_dependencies(depends, layer, layers, logger, ret = []):
         logger.debug('Processing dependencies %s for layer %s.' % \
                     (depends, layer['name']))
@@ -166,7 +192,7 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger):
             if depend == 'core':
                 continue
 
-            layer_depend = _find_layer_depends(depend, layers)
+            layer_depend = _find_layer(depend, layers)
             if not layer_depend:
                 logger.error('Layer %s depends on %s and isn\'t found.' % \
                         (layer['name'], depend))
@@ -203,6 +229,11 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger):
         layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
 
     # Note: [] (empty) is allowed, None is not!
+    return layer_depends
+
+def add_layer_dependencies(bblayersconf, layer, layers, logger):
+
+    layer_depends = get_layer_dependencies(layer, layers, logger)
     if layer_depends is None:
         return False
     else:
@@ -256,7 +287,7 @@ def check_command(error_msg, cmd, cwd=None):
         raise RuntimeError(msg)
     return output
 
-def get_signatures(builddir, failsafe=False, machine=None):
+def get_signatures(builddir, failsafe=False, machine=None, extravars=None):
     import re
 
     # some recipes needs to be excluded like meta-world-pkgdata
@@ -267,13 +298,16 @@ def get_signatures(builddir, failsafe=False, machine=None):
     sigs = {}
     tune2tasks = {}
 
-    cmd = 'BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" '
+    cmd = 'BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" '
+    if extravars:
+        cmd += extravars
+        cmd += ' '
     if machine:
         cmd += 'MACHINE=%s ' % machine
     cmd += 'bitbake '
     if failsafe:
         cmd += '-k '
-    cmd += '-S none world'
+    cmd += '-S lockedsigs world'
     sigs_file = os.path.join(builddir, 'locked-sigs.inc')
     if os.path.exists(sigs_file):
         os.unlink(sigs_file)
@@ -290,8 +324,8 @@ def get_signatures(builddir, failsafe=False, machine=None):
         else:
             raise
 
-    sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$")
-    tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
+    sig_regex = re.compile(r"^(?P<task>.*:.*):(?P<hash>.*) .$")
+    tune_regex = re.compile(r"(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
     current_tune = None
     with open(sigs_file, 'r') as f:
         for line in f.readlines():
@@ -418,3 +452,15 @@ def compare_signatures(old_sigs, curr_sigs):
             msg.extend(['      ' + line for line in output.splitlines()])
             msg.append('')
     return '\n'.join(msg)
+
+
+def get_git_toplevel(directory):
+    """
+    Try and find the top of the git repository that directory might be in.
+    Returns the top-level directory, or None.
+    """
+    cmd = ["git", "-C", directory, "rev-parse", "--show-toplevel"]
+    try:
+        return subprocess.check_output(cmd, text=True).strip()
+    except:
+        return None
diff --git a/scripts/lib/checklayer/cases/bsp.py b/scripts/lib/checklayer/cases/bsp.py
index 7fd56f5d36..b76163fb56 100644
--- a/scripts/lib/checklayer/cases/bsp.py
+++ b/scripts/lib/checklayer/cases/bsp.py
@@ -11,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase
 class BSPCheckLayer(OECheckLayerTestCase):
     @classmethod
     def setUpClass(self):
-        if self.tc.layer['type'] != LayerType.BSP:
+        if self.tc.layer['type'] not in (LayerType.BSP, LayerType.CORE):
             raise unittest.SkipTest("BSPCheckLayer: Layer %s isn't BSP one." %\
                 self.tc.layer['name'])
 
@@ -153,7 +153,7 @@ class BSPCheckLayer(OECheckLayerTestCase):
                 # do_build can be ignored: it is know to have
                 # different signatures in some cases, for example in
                 # the allarch ca-certificates due to RDEPENDS=openssl.
-                # That particular dependency is whitelisted via
+                # That particular dependency is marked via
                 # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up
                 # in the sstate signature hash because filtering it
                 # out would be hard and running do_build multiple
diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py
index b82304e361..ddead69a7b 100644
--- a/scripts/lib/checklayer/cases/common.py
+++ b/scripts/lib/checklayer/cases/common.py
@@ -6,15 +6,19 @@
 import glob
 import os
 import unittest
-from checklayer import get_signatures, LayerType, check_command, get_depgraph, compare_signatures
+import re
+from checklayer import get_signatures, LayerType, check_command, compare_signatures, get_git_toplevel
 from checklayer.case import OECheckLayerTestCase
 
 class CommonCheckLayer(OECheckLayerTestCase):
     def test_readme(self):
+        if self.tc.layer['type'] == LayerType.CORE:
+            raise unittest.SkipTest("Core layer's README is top level")
+
         # The top-level README file may have a suffix (like README.rst or README.txt).
         readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*'))
         self.assertTrue(len(readme_files) > 0,
-                        msg="Layer doesn't contains README file.")
+                        msg="Layer doesn't contain a README file.")
 
         # There might be more than one file matching the file pattern above
         # (for example, README.rst and README-COPYING.rst). The one with the shortest
@@ -26,6 +30,48 @@ class CommonCheckLayer(OECheckLayerTestCase):
         self.assertTrue(data,
                 msg="Layer contains a README file but it is empty.")
 
+        # If a layer's README references another README, then the checks below are not valid
+        if re.search('README', data, re.IGNORECASE):
+            return
+
+        self.assertIn('maintainer', data.lower())
+        self.assertIn('patch', data.lower())
+        # Check that there is an email address in the README
+        email_regex = re.compile(r"[^@]+@[^@]+")
+        self.assertTrue(email_regex.match(data))
+
+    def find_file_by_name(self, globs):
+        """
+        Utility function to find a file that matches the specified list of
+        globs, in either the layer directory itself or the repository top-level
+        directory.
+        """
+        directories = [self.tc.layer["path"]]
+        toplevel = get_git_toplevel(directories[0])
+        if toplevel:
+            directories.append(toplevel)
+
+        for path in directories:
+            for name in globs:
+                files = glob.glob(os.path.join(path, name))
+                if files:
+                    return sorted(files)[0]
+        return None
+
+    def test_security(self):
+        """
+        Test that the layer has a SECURITY.md (or similar) file, either in the
+        layer itself or at the top of the containing git repository.
+        """
+        if self.tc.layer["type"] == LayerType.CORE:
+            raise unittest.SkipTest("Core layer's SECURITY is top level")
+
+        filename = self.find_file_by_name(("SECURITY", "SECURITY.*"))
+        self.assertTrue(filename, msg="Layer doesn't contain a SECURITY.md file.")
+
+        size = os.path.getsize(filename)
+        self.assertGreater(size, 0, msg=f"{filename} has no content.")
+
     def test_parse(self):
         check_command('Layer %s failed to parse.' % self.tc.layer['name'],
                       'bitbake -p')
@@ -43,6 +89,35 @@ class CommonCheckLayer(OECheckLayerTestCase):
         '''
         get_signatures(self.td['builddir'], failsafe=False)
 
+    def test_world_inherit_class(self):
+        '''
+        This also does "bitbake -S none world" along with inheriting "yocto-check-layer"
+        class, which can do additional per-recipe test cases.
+        '''
+        msg = []
+        try:
+            get_signatures(self.td['builddir'], failsafe=False, machine=None, extravars='BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS INHERIT" INHERIT="yocto-check-layer"')
+        except RuntimeError as ex:
+            msg.append(str(ex))
+        if msg:
+            msg.insert(0, 'Layer %s failed additional checks from yocto-check-layer.bbclass\nSee below log for specific recipe parsing errors:\n' % \
+                self.tc.layer['name'])
+            self.fail('\n'.join(msg))
+
+    def test_patches_upstream_status(self):
+        import sys
+        sys.path.append(os.path.join(sys.path[0], '../../../../meta/lib/'))
+        import oe.qa
+        patches = []
+        for dirpath, dirs, files in os.walk(self.tc.layer['path']):
+            for filename in files:
+                if filename.endswith(".patch"):
+                    ppath = os.path.join(dirpath, filename)
+                    if oe.qa.check_upstream_status(ppath):
+                        patches.append(ppath)
+        self.assertEqual(len(patches), 0 , \
+                msg="Found following patches with malformed or missing upstream status:\n%s" % '\n'.join([str(patch) for patch in patches]))
+
     def test_signatures(self):
         if self.tc.layer['type'] == LayerType.SOFTWARE and \
            not self.tc.test_software_layer_signatures:
diff --git a/scripts/lib/checklayer/cases/distro.py b/scripts/lib/checklayer/cases/distro.py
index f0bee5493c..a35332451c 100644
--- a/scripts/lib/checklayer/cases/distro.py
+++ b/scripts/lib/checklayer/cases/distro.py
@@ -11,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase
 class DistroCheckLayer(OECheckLayerTestCase):
     @classmethod
     def setUpClass(self):
-        if self.tc.layer['type'] != LayerType.DISTRO:
+        if self.tc.layer['type'] not in (LayerType.DISTRO, LayerType.CORE):
             raise unittest.SkipTest("DistroCheckLayer: Layer %s isn't Distro one." %\
                 self.tc.layer['name'])