oeqa/sdk: Simplify test specification and discovery

Simplify how tests are specified and discovered for different SDK configurations
to allow per-layer customization.

* Introduce `TESTSDK_CASE_DIRS` variable to specify test directory types,
  replacing the need to modify the default_cases class member

* Discover tests from configured layers using a common discovery pattern
  (`<LAYER_DIR>/lib/oeqa/<dirname>/cases`) where `<dirname>` is specified in `TESTSDK_CASE_DIRS`

* The buildtools directories were renamed to follow the common discovery pattern
(`<LAYER_DIR>/lib/oeqa/<dirname>/cases`) for consistency across all SDK configurations.

  meta/lib/oeqa/
  ├── sdk/cases/              # Standard SDK: dirname="sdk"
  ├── buildtools/cases/       # Buildtools: dirname="buildtools"
  └── buildtools-docs/cases/  # Buildtools-docs: dirname="buildtools-docs"

  meta-mingw/lib/oeqa/
  └── sdkmingw/cases/         # MinGW: dirname="sdkmingw"

  meta-foo/lib/oeqa/
  └── sdk/cases/              # Standard SDK: dirname="sdk"

Tested by:

1. Adding new tests using the default discovery pattern `<LAYER_DIR>/lib/oeqa/sdk/cases` and
   verifying they are discovered and executed.

2. Verifying existing SDK configuration tests work (requires -c populate_sdk first):
   * Standard SDK: `bitbake core-image-minimal -c testsdk`
   * Buildtools tarball: `bitbake buildtools-tarball -c testsdk`
   * Buildtools docs tarball: `bitbake buildtools-docs-tarball -c testsdk`
   * Mingw SDK: (SDKMACHINE = "x86_64-mingw32") `bitbake core-image-minimal -c testsdk`

(From OE-Core rev: bde94c128c0b4e7e1ebea40f582b4dd6dcc965ff)

Signed-off-by: Thune Tran <thune.a.tran@boeing.com>
Signed-off-by: Chuck Wolber <chuck.wolber@boeing.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

8 files changed, 23 insertions, 133 deletions

diff --git a/meta/lib/oeqa/sdk/buildtools-cases/README b/meta/lib/oeqa/sdk/buildtools-cases/README
deleted file mode 100644
index d4f20faa9f..0000000000
--- a/meta/lib/oeqa/sdk/buildtools-cases/README
+++ /dev/null
@@ -1,2 +0,0 @@
-These test cases are used by buildtools-tarball, and are not used by the testsdk
-class.
diff --git a/meta/lib/oeqa/sdk/buildtools-cases/build.py b/meta/lib/oeqa/sdk/buildtools-cases/build.py
deleted file mode 100644
index c85c32496b..0000000000
--- a/meta/lib/oeqa/sdk/buildtools-cases/build.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Copyright OpenEmbedded Contributors
-#
-# SPDX-License-Identifier: MIT
-#
-
-import os, tempfile
-import time
-from oeqa.sdk.case import OESDKTestCase
-from oeqa.utils.subprocesstweak import errors_have_output
-errors_have_output()
-
-class BuildTests(OESDKTestCase):
-    """
-    Verify that bitbake can build virtual/libc inside the buildtools.
-    """
-    def test_libc(self):
-        with tempfile.TemporaryDirectory(prefix='bitbake-build-', dir=self.tc.sdk_dir) as testdir:
-            corebase = self.td['COREBASE']
-
-            self._run('. %s/oe-init-build-env %s' % (corebase, testdir))
-            with open(os.path.join(testdir, 'conf', 'local.conf'), 'ta') as conf:
-                conf.write('\n')
-                conf.write('DL_DIR = "%s"\n' % self.td['DL_DIR'])
-
-            try:
-                self._run('. %s/oe-init-build-env %s && bitbake virtual/libc' % (corebase, testdir))
-            finally:
-                delay = 10
-                while delay and (os.path.exists(testdir + "/bitbake.lock") or os.path.exists(testdir + "/cache/hashserv.db-wal")):
-                    time.sleep(1)
-                    delay = delay - 1
diff --git a/meta/lib/oeqa/sdk/buildtools-cases/gcc.py b/meta/lib/oeqa/sdk/buildtools-cases/gcc.py
deleted file mode 100644
index a62c4d0bc4..0000000000
--- a/meta/lib/oeqa/sdk/buildtools-cases/gcc.py
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Copyright OpenEmbedded Contributors
-#
-# SPDX-License-Identifier: MIT
-#
-
-import os.path
-from oeqa.sdk.case import OESDKTestCase
-
-class GccTests(OESDKTestCase):
-    def test_verify_specs(self):
-        """
-        Verify that the compiler has been relocated successfully and isn't
-        looking in the hard-coded prefix.
-        """
-        # Canonicalise the SDK root
-        sdk_base = os.path.realpath(self.tc.sdk_dir)
-        # Canonicalise the location of GCC
-        gcc_path = os.path.realpath(self._run("command -v gcc").strip())
-        # Skip the test if the GCC didn't come from the buildtools, as it only
-        # comes with buildtools-extended-tarball.
-        if os.path.commonprefix((sdk_base, gcc_path)) != sdk_base:
-            self.skipTest("Buildtools does not provide GCC")
-
-        # This is the prefix that GCC is build with, and should be replaced at
-        # installation time.
-        sdkpath = self.td.get("SDKPATH")
-        self.assertTrue(sdkpath)
-
-        for line in self._run('gcc -dumpspecs').splitlines():
-            self.assertNotIn(sdkpath, line)
diff --git a/meta/lib/oeqa/sdk/buildtools-cases/https.py b/meta/lib/oeqa/sdk/buildtools-cases/https.py
deleted file mode 100644
index 4525e3d758..0000000000
--- a/meta/lib/oeqa/sdk/buildtools-cases/https.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Copyright OpenEmbedded Contributors
-#
-# SPDX-License-Identifier: MIT
-#
-
-from oeqa.sdk.case import OESDKTestCase
-from oeqa.utils.subprocesstweak import errors_have_output
-errors_have_output()
-
-class HTTPTests(OESDKTestCase):
-    """
-    Verify that HTTPS certificates are working correctly, as this depends on
-    environment variables being set correctly.
-    """
-
-    def test_wget(self):
-        self._run('env -i wget --debug --output-document /dev/null https://yoctoproject.org/connectivity.html')
-
-    def test_python(self):
-        # urlopen() returns a file-like object on success and throws an exception otherwise
-        self._run('python3 -c \'import urllib.request; urllib.request.urlopen("https://yoctoproject.org/connectivity.html")\'')
diff --git a/meta/lib/oeqa/sdk/buildtools-cases/sanity.py b/meta/lib/oeqa/sdk/buildtools-cases/sanity.py
deleted file mode 100644
index a55d456656..0000000000
--- a/meta/lib/oeqa/sdk/buildtools-cases/sanity.py
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Copyright OpenEmbedded Contributors
-#
-# SPDX-License-Identifier: MIT
-#
-
-import shutil
-import os.path
-from oeqa.sdk.case import OESDKTestCase
-
-class SanityTests(OESDKTestCase):
-    def test_tools(self):
-        """
-        Test that wget and tar come from the buildtools, not the host. This
-        verifies that the buildtools have installed correctly. We can't check
-        for gcc as that is only installed by buildtools-extended.
-        """
-        for command in ("tar", "wget"):
-            # Canonicalise the SDK root
-            sdk_base = os.path.realpath(self.tc.sdk_dir)
-            # Canonicalise the location of this command
-            tool_path = os.path.realpath(self._run("command -v %s" % command).strip())
-            # Assert that the tool was found inside the SDK root
-            self.assertEqual(os.path.commonprefix((sdk_base, tool_path)), sdk_base)
diff --git a/meta/lib/oeqa/sdk/buildtools-docs-cases/README b/meta/lib/oeqa/sdk/buildtools-docs-cases/README
deleted file mode 100644
index f8edbc7dad..0000000000
--- a/meta/lib/oeqa/sdk/buildtools-docs-cases/README
+++ /dev/null
@@ -1,2 +0,0 @@
-These test cases are used by build-docs-tarball, and are not used by the testsdk
-class.
diff --git a/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py b/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py
deleted file mode 100644
index 6e3ee94292..0000000000
--- a/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Copyright OpenEmbedded Contributors
-#
-# SPDX-License-Identifier: MIT
-#
-
-import tempfile
-from oeqa.sdk.case import OESDKTestCase
-from oeqa.utils.subprocesstweak import errors_have_output
-errors_have_output()
-
-class BuildTests(OESDKTestCase):
-    """
-    Verify that our docs can build using our docs tools tarball.
-    """
-    def test_docs_build(self):
-        with tempfile.TemporaryDirectory(prefix='docs-tarball-build-', dir=self.tc.sdk_dir) as testdir:
-            self._run('git clone git://git.yoctoproject.org/yocto-docs %s' % testdir)
-            self._run('cd %s/documentation && make html' % testdir)
diff --git a/meta/lib/oeqa/sdk/testsdk.py b/meta/lib/oeqa/sdk/testsdk.py
index 52b702b6a2..cffcf9f49a 100644
--- a/meta/lib/oeqa/sdk/testsdk.py
+++ b/meta/lib/oeqa/sdk/testsdk.py
@@ -31,6 +31,28 @@ class TestSDK(TestSDKBase):
     context_class = OESDKTestContext
     test_type = 'sdk'
 
+    def sdk_dir_names(self, d):
+        """Return list from TESTSDK_CASE_DIRS."""
+        testdirs = d.getVar("TESTSDK_CASE_DIRS")
+        if testdirs:
+            return testdirs.split()
+
+        bb.fatal("TESTSDK_CASE_DIRS unset, can't find SDK test directories.")
+
+    def get_sdk_paths(self, d):
+        """
+        Return a list of paths where SDK test cases reside.
+
+        SDK tests are expected in <LAYER_DIR>/lib/oeqa/<dirname>/cases
+        """
+        paths = []
+        for layer in d.getVar("BBLAYERS").split():
+            for dirname in self.sdk_dir_names(d):
+                case_path = os.path.join(layer, "lib", "oeqa", dirname, "cases")
+                if os.path.isdir(case_path):
+                    paths.append(case_path)
+        return paths
+
     def get_tcname(self, d):
         """
         Get the name of the SDK file
@@ -115,7 +137,7 @@ class TestSDK(TestSDKBase):
 
             try:
                 modules = (d.getVar("TESTSDK_SUITES") or "").split()
-                tc.loadTests(self.context_executor_class.default_cases, modules)
+                tc.loadTests(self.get_sdk_paths(d), modules)
             except Exception as e:
                 import traceback
                 bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())