From 041ba867db947c9336c47747c621d2a1719d5c4c Mon Sep 17 00:00:00 2001 From: Thune Tran Date: Thu, 26 Jun 2025 20:57:21 +0000 Subject: oeqa/sdk: Simplify test specification and discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplify how tests are specified and discovered for different SDK configurations to allow per-layer customization. * Introduce `TESTSDK_CASE_DIRS` variable to specify test directory types, replacing the need to modify the default_cases class member * Discover tests from configured layers using a common discovery pattern (`/lib/oeqa//cases`) where `` is specified in `TESTSDK_CASE_DIRS` * The buildtools directories were renamed to follow the common discovery pattern (`/lib/oeqa//cases`) for consistency across all SDK configurations. meta/lib/oeqa/ ├── sdk/cases/ # Standard SDK: dirname="sdk" ├── buildtools/cases/ # Buildtools: dirname="buildtools" └── buildtools-docs/cases/ # Buildtools-docs: dirname="buildtools-docs" meta-mingw/lib/oeqa/ └── sdkmingw/cases/ # MinGW: dirname="sdkmingw" meta-foo/lib/oeqa/ └── sdk/cases/ # Standard SDK: dirname="sdk" Tested by: 1. Adding new tests using the default discovery pattern `/lib/oeqa/sdk/cases` and verifying they are discovered and executed. 2. Verifying existing SDK configuration tests work (requires -c populate_sdk first): * Standard SDK: `bitbake core-image-minimal -c testsdk` * Buildtools tarball: `bitbake buildtools-tarball -c testsdk` * Buildtools docs tarball: `bitbake buildtools-docs-tarball -c testsdk` * Mingw SDK: (SDKMACHINE = "x86_64-mingw32") `bitbake core-image-minimal -c testsdk` (From OE-Core rev: bde94c128c0b4e7e1ebea40f582b4dd6dcc965ff) Signed-off-by: Thune Tran Signed-off-by: Chuck Wolber Signed-off-by: Richard Purdie --- meta/classes-recipe/testsdk.bbclass | 1 + meta/lib/oeqa/buildtools-docs/cases/README | 2 ++ meta/lib/oeqa/buildtools-docs/cases/build.py | 19 ++++++++++++++ meta/lib/oeqa/buildtools/cases/README | 2 ++ meta/lib/oeqa/buildtools/cases/build.py | 32 +++++++++++++++++++++++ meta/lib/oeqa/buildtools/cases/gcc.py | 31 ++++++++++++++++++++++ meta/lib/oeqa/buildtools/cases/https.py | 22 ++++++++++++++++ meta/lib/oeqa/buildtools/cases/sanity.py | 24 +++++++++++++++++ meta/lib/oeqa/sdk/buildtools-cases/README | 2 -- meta/lib/oeqa/sdk/buildtools-cases/build.py | 32 ----------------------- meta/lib/oeqa/sdk/buildtools-cases/gcc.py | 31 ---------------------- meta/lib/oeqa/sdk/buildtools-cases/https.py | 22 ---------------- meta/lib/oeqa/sdk/buildtools-cases/sanity.py | 24 ----------------- meta/lib/oeqa/sdk/buildtools-docs-cases/README | 2 -- meta/lib/oeqa/sdk/buildtools-docs-cases/build.py | 19 -------------- meta/lib/oeqa/sdk/testsdk.py | 24 ++++++++++++++++- meta/recipes-core/meta/buildtools-docs-tarball.bb | 3 ++- meta/recipes-core/meta/buildtools-tarball.bb | 21 +++------------ 18 files changed, 161 insertions(+), 152 deletions(-) create mode 100644 meta/lib/oeqa/buildtools-docs/cases/README create mode 100644 meta/lib/oeqa/buildtools-docs/cases/build.py create mode 100644 meta/lib/oeqa/buildtools/cases/README create mode 100644 meta/lib/oeqa/buildtools/cases/build.py create mode 100644 meta/lib/oeqa/buildtools/cases/gcc.py create mode 100644 meta/lib/oeqa/buildtools/cases/https.py create mode 100644 meta/lib/oeqa/buildtools/cases/sanity.py delete mode 100644 meta/lib/oeqa/sdk/buildtools-cases/README delete mode 100644 meta/lib/oeqa/sdk/buildtools-cases/build.py delete mode 100644 meta/lib/oeqa/sdk/buildtools-cases/gcc.py delete mode 100644 meta/lib/oeqa/sdk/buildtools-cases/https.py delete mode 100644 meta/lib/oeqa/sdk/buildtools-cases/sanity.py delete mode 100644 meta/lib/oeqa/sdk/buildtools-docs-cases/README delete mode 100644 meta/lib/oeqa/sdk/buildtools-docs-cases/build.py diff --git a/meta/classes-recipe/testsdk.bbclass b/meta/classes-recipe/testsdk.bbclass index 59d2834c99..b1c4fa67e6 100644 --- a/meta/classes-recipe/testsdk.bbclass +++ b/meta/classes-recipe/testsdk.bbclass @@ -19,6 +19,7 @@ TESTSDK_SUITES ?= "" TESTSDK_CLASS_NAME ?= "oeqa.sdk.testsdk.TestSDK" TESTSDKEXT_CLASS_NAME ?= "oeqa.sdkext.testsdk.TestSDKExt" +TESTSDK_CASE_DIRS ?= "sdk" def import_and_run(name, d): import importlib diff --git a/meta/lib/oeqa/buildtools-docs/cases/README b/meta/lib/oeqa/buildtools-docs/cases/README new file mode 100644 index 0000000000..f8edbc7dad --- /dev/null +++ b/meta/lib/oeqa/buildtools-docs/cases/README @@ -0,0 +1,2 @@ +These test cases are used by build-docs-tarball, and are not used by the testsdk +class. diff --git a/meta/lib/oeqa/buildtools-docs/cases/build.py b/meta/lib/oeqa/buildtools-docs/cases/build.py new file mode 100644 index 0000000000..6e3ee94292 --- /dev/null +++ b/meta/lib/oeqa/buildtools-docs/cases/build.py @@ -0,0 +1,19 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import tempfile +from oeqa.sdk.case import OESDKTestCase +from oeqa.utils.subprocesstweak import errors_have_output +errors_have_output() + +class BuildTests(OESDKTestCase): + """ + Verify that our docs can build using our docs tools tarball. + """ + def test_docs_build(self): + with tempfile.TemporaryDirectory(prefix='docs-tarball-build-', dir=self.tc.sdk_dir) as testdir: + self._run('git clone git://git.yoctoproject.org/yocto-docs %s' % testdir) + self._run('cd %s/documentation && make html' % testdir) diff --git a/meta/lib/oeqa/buildtools/cases/README b/meta/lib/oeqa/buildtools/cases/README new file mode 100644 index 0000000000..d4f20faa9f --- /dev/null +++ b/meta/lib/oeqa/buildtools/cases/README @@ -0,0 +1,2 @@ +These test cases are used by buildtools-tarball, and are not used by the testsdk +class. diff --git a/meta/lib/oeqa/buildtools/cases/build.py b/meta/lib/oeqa/buildtools/cases/build.py new file mode 100644 index 0000000000..c85c32496b --- /dev/null +++ b/meta/lib/oeqa/buildtools/cases/build.py @@ -0,0 +1,32 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os, tempfile +import time +from oeqa.sdk.case import OESDKTestCase +from oeqa.utils.subprocesstweak import errors_have_output +errors_have_output() + +class BuildTests(OESDKTestCase): + """ + Verify that bitbake can build virtual/libc inside the buildtools. + """ + def test_libc(self): + with tempfile.TemporaryDirectory(prefix='bitbake-build-', dir=self.tc.sdk_dir) as testdir: + corebase = self.td['COREBASE'] + + self._run('. %s/oe-init-build-env %s' % (corebase, testdir)) + with open(os.path.join(testdir, 'conf', 'local.conf'), 'ta') as conf: + conf.write('\n') + conf.write('DL_DIR = "%s"\n' % self.td['DL_DIR']) + + try: + self._run('. %s/oe-init-build-env %s && bitbake virtual/libc' % (corebase, testdir)) + finally: + delay = 10 + while delay and (os.path.exists(testdir + "/bitbake.lock") or os.path.exists(testdir + "/cache/hashserv.db-wal")): + time.sleep(1) + delay = delay - 1 diff --git a/meta/lib/oeqa/buildtools/cases/gcc.py b/meta/lib/oeqa/buildtools/cases/gcc.py new file mode 100644 index 0000000000..a62c4d0bc4 --- /dev/null +++ b/meta/lib/oeqa/buildtools/cases/gcc.py @@ -0,0 +1,31 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import os.path +from oeqa.sdk.case import OESDKTestCase + +class GccTests(OESDKTestCase): + def test_verify_specs(self): + """ + Verify that the compiler has been relocated successfully and isn't + looking in the hard-coded prefix. + """ + # Canonicalise the SDK root + sdk_base = os.path.realpath(self.tc.sdk_dir) + # Canonicalise the location of GCC + gcc_path = os.path.realpath(self._run("command -v gcc").strip()) + # Skip the test if the GCC didn't come from the buildtools, as it only + # comes with buildtools-extended-tarball. + if os.path.commonprefix((sdk_base, gcc_path)) != sdk_base: + self.skipTest("Buildtools does not provide GCC") + + # This is the prefix that GCC is build with, and should be replaced at + # installation time. + sdkpath = self.td.get("SDKPATH") + self.assertTrue(sdkpath) + + for line in self._run('gcc -dumpspecs').splitlines(): + self.assertNotIn(sdkpath, line) diff --git a/meta/lib/oeqa/buildtools/cases/https.py b/meta/lib/oeqa/buildtools/cases/https.py new file mode 100644 index 0000000000..4525e3d758 --- /dev/null +++ b/meta/lib/oeqa/buildtools/cases/https.py @@ -0,0 +1,22 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +from oeqa.sdk.case import OESDKTestCase +from oeqa.utils.subprocesstweak import errors_have_output +errors_have_output() + +class HTTPTests(OESDKTestCase): + """ + Verify that HTTPS certificates are working correctly, as this depends on + environment variables being set correctly. + """ + + def test_wget(self): + self._run('env -i wget --debug --output-document /dev/null https://yoctoproject.org/connectivity.html') + + def test_python(self): + # urlopen() returns a file-like object on success and throws an exception otherwise + self._run('python3 -c \'import urllib.request; urllib.request.urlopen("https://yoctoproject.org/connectivity.html")\'') diff --git a/meta/lib/oeqa/buildtools/cases/sanity.py b/meta/lib/oeqa/buildtools/cases/sanity.py new file mode 100644 index 0000000000..a55d456656 --- /dev/null +++ b/meta/lib/oeqa/buildtools/cases/sanity.py @@ -0,0 +1,24 @@ +# +# Copyright OpenEmbedded Contributors +# +# SPDX-License-Identifier: MIT +# + +import shutil +import os.path +from oeqa.sdk.case import OESDKTestCase + +class SanityTests(OESDKTestCase): + def test_tools(self): + """ + Test that wget and tar come from the buildtools, not the host. This + verifies that the buildtools have installed correctly. We can't check + for gcc as that is only installed by buildtools-extended. + """ + for command in ("tar", "wget"): + # Canonicalise the SDK root + sdk_base = os.path.realpath(self.tc.sdk_dir) + # Canonicalise the location of this command + tool_path = os.path.realpath(self._run("command -v %s" % command).strip()) + # Assert that the tool was found inside the SDK root + self.assertEqual(os.path.commonprefix((sdk_base, tool_path)), sdk_base) diff --git a/meta/lib/oeqa/sdk/buildtools-cases/README b/meta/lib/oeqa/sdk/buildtools-cases/README deleted file mode 100644 index d4f20faa9f..0000000000 --- a/meta/lib/oeqa/sdk/buildtools-cases/README +++ /dev/null @@ -1,2 +0,0 @@ -These test cases are used by buildtools-tarball, and are not used by the testsdk -class. diff --git a/meta/lib/oeqa/sdk/buildtools-cases/build.py b/meta/lib/oeqa/sdk/buildtools-cases/build.py deleted file mode 100644 index c85c32496b..0000000000 --- a/meta/lib/oeqa/sdk/buildtools-cases/build.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Copyright OpenEmbedded Contributors -# -# SPDX-License-Identifier: MIT -# - -import os, tempfile -import time -from oeqa.sdk.case import OESDKTestCase -from oeqa.utils.subprocesstweak import errors_have_output -errors_have_output() - -class BuildTests(OESDKTestCase): - """ - Verify that bitbake can build virtual/libc inside the buildtools. - """ - def test_libc(self): - with tempfile.TemporaryDirectory(prefix='bitbake-build-', dir=self.tc.sdk_dir) as testdir: - corebase = self.td['COREBASE'] - - self._run('. %s/oe-init-build-env %s' % (corebase, testdir)) - with open(os.path.join(testdir, 'conf', 'local.conf'), 'ta') as conf: - conf.write('\n') - conf.write('DL_DIR = "%s"\n' % self.td['DL_DIR']) - - try: - self._run('. %s/oe-init-build-env %s && bitbake virtual/libc' % (corebase, testdir)) - finally: - delay = 10 - while delay and (os.path.exists(testdir + "/bitbake.lock") or os.path.exists(testdir + "/cache/hashserv.db-wal")): - time.sleep(1) - delay = delay - 1 diff --git a/meta/lib/oeqa/sdk/buildtools-cases/gcc.py b/meta/lib/oeqa/sdk/buildtools-cases/gcc.py deleted file mode 100644 index a62c4d0bc4..0000000000 --- a/meta/lib/oeqa/sdk/buildtools-cases/gcc.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright OpenEmbedded Contributors -# -# SPDX-License-Identifier: MIT -# - -import os.path -from oeqa.sdk.case import OESDKTestCase - -class GccTests(OESDKTestCase): - def test_verify_specs(self): - """ - Verify that the compiler has been relocated successfully and isn't - looking in the hard-coded prefix. - """ - # Canonicalise the SDK root - sdk_base = os.path.realpath(self.tc.sdk_dir) - # Canonicalise the location of GCC - gcc_path = os.path.realpath(self._run("command -v gcc").strip()) - # Skip the test if the GCC didn't come from the buildtools, as it only - # comes with buildtools-extended-tarball. - if os.path.commonprefix((sdk_base, gcc_path)) != sdk_base: - self.skipTest("Buildtools does not provide GCC") - - # This is the prefix that GCC is build with, and should be replaced at - # installation time. - sdkpath = self.td.get("SDKPATH") - self.assertTrue(sdkpath) - - for line in self._run('gcc -dumpspecs').splitlines(): - self.assertNotIn(sdkpath, line) diff --git a/meta/lib/oeqa/sdk/buildtools-cases/https.py b/meta/lib/oeqa/sdk/buildtools-cases/https.py deleted file mode 100644 index 4525e3d758..0000000000 --- a/meta/lib/oeqa/sdk/buildtools-cases/https.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright OpenEmbedded Contributors -# -# SPDX-License-Identifier: MIT -# - -from oeqa.sdk.case import OESDKTestCase -from oeqa.utils.subprocesstweak import errors_have_output -errors_have_output() - -class HTTPTests(OESDKTestCase): - """ - Verify that HTTPS certificates are working correctly, as this depends on - environment variables being set correctly. - """ - - def test_wget(self): - self._run('env -i wget --debug --output-document /dev/null https://yoctoproject.org/connectivity.html') - - def test_python(self): - # urlopen() returns a file-like object on success and throws an exception otherwise - self._run('python3 -c \'import urllib.request; urllib.request.urlopen("https://yoctoproject.org/connectivity.html")\'') diff --git a/meta/lib/oeqa/sdk/buildtools-cases/sanity.py b/meta/lib/oeqa/sdk/buildtools-cases/sanity.py deleted file mode 100644 index a55d456656..0000000000 --- a/meta/lib/oeqa/sdk/buildtools-cases/sanity.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright OpenEmbedded Contributors -# -# SPDX-License-Identifier: MIT -# - -import shutil -import os.path -from oeqa.sdk.case import OESDKTestCase - -class SanityTests(OESDKTestCase): - def test_tools(self): - """ - Test that wget and tar come from the buildtools, not the host. This - verifies that the buildtools have installed correctly. We can't check - for gcc as that is only installed by buildtools-extended. - """ - for command in ("tar", "wget"): - # Canonicalise the SDK root - sdk_base = os.path.realpath(self.tc.sdk_dir) - # Canonicalise the location of this command - tool_path = os.path.realpath(self._run("command -v %s" % command).strip()) - # Assert that the tool was found inside the SDK root - self.assertEqual(os.path.commonprefix((sdk_base, tool_path)), sdk_base) diff --git a/meta/lib/oeqa/sdk/buildtools-docs-cases/README b/meta/lib/oeqa/sdk/buildtools-docs-cases/README deleted file mode 100644 index f8edbc7dad..0000000000 --- a/meta/lib/oeqa/sdk/buildtools-docs-cases/README +++ /dev/null @@ -1,2 +0,0 @@ -These test cases are used by build-docs-tarball, and are not used by the testsdk -class. diff --git a/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py b/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py deleted file mode 100644 index 6e3ee94292..0000000000 --- a/meta/lib/oeqa/sdk/buildtools-docs-cases/build.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright OpenEmbedded Contributors -# -# SPDX-License-Identifier: MIT -# - -import tempfile -from oeqa.sdk.case import OESDKTestCase -from oeqa.utils.subprocesstweak import errors_have_output -errors_have_output() - -class BuildTests(OESDKTestCase): - """ - Verify that our docs can build using our docs tools tarball. - """ - def test_docs_build(self): - with tempfile.TemporaryDirectory(prefix='docs-tarball-build-', dir=self.tc.sdk_dir) as testdir: - self._run('git clone git://git.yoctoproject.org/yocto-docs %s' % testdir) - self._run('cd %s/documentation && make html' % testdir) diff --git a/meta/lib/oeqa/sdk/testsdk.py b/meta/lib/oeqa/sdk/testsdk.py index 52b702b6a2..cffcf9f49a 100644 --- a/meta/lib/oeqa/sdk/testsdk.py +++ b/meta/lib/oeqa/sdk/testsdk.py @@ -31,6 +31,28 @@ class TestSDK(TestSDKBase): context_class = OESDKTestContext test_type = 'sdk' + def sdk_dir_names(self, d): + """Return list from TESTSDK_CASE_DIRS.""" + testdirs = d.getVar("TESTSDK_CASE_DIRS") + if testdirs: + return testdirs.split() + + bb.fatal("TESTSDK_CASE_DIRS unset, can't find SDK test directories.") + + def get_sdk_paths(self, d): + """ + Return a list of paths where SDK test cases reside. + + SDK tests are expected in /lib/oeqa//cases + """ + paths = [] + for layer in d.getVar("BBLAYERS").split(): + for dirname in self.sdk_dir_names(d): + case_path = os.path.join(layer, "lib", "oeqa", dirname, "cases") + if os.path.isdir(case_path): + paths.append(case_path) + return paths + def get_tcname(self, d): """ Get the name of the SDK file @@ -115,7 +137,7 @@ class TestSDK(TestSDKBase): try: modules = (d.getVar("TESTSDK_SUITES") or "").split() - tc.loadTests(self.context_executor_class.default_cases, modules) + tc.loadTests(self.get_sdk_paths(d), modules) except Exception as e: import traceback bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) diff --git a/meta/recipes-core/meta/buildtools-docs-tarball.bb b/meta/recipes-core/meta/buildtools-docs-tarball.bb index 24fbc33000..98d47f7b71 100644 --- a/meta/recipes-core/meta/buildtools-docs-tarball.bb +++ b/meta/recipes-core/meta/buildtools-docs-tarball.bb @@ -18,4 +18,5 @@ TOOLCHAIN_OUTPUTNAME = "${SDK_ARCH}-buildtools-docs-nativesdk-standalone-${DISTR SDK_TITLE = "Docs Build tools tarball" -TESTSDK_CASES = "buildtools-docs-cases" +# Directory that contains testcases +TESTSDK_CASE_DIRS = "buildtools-docs" \ No newline at end of file diff --git a/meta/recipes-core/meta/buildtools-tarball.bb b/meta/recipes-core/meta/buildtools-tarball.bb index 6fa6d93a3d..02117ab84d 100644 --- a/meta/recipes-core/meta/buildtools-tarball.bb +++ b/meta/recipes-core/meta/buildtools-tarball.bb @@ -124,22 +124,7 @@ TOOLCHAIN_NEED_CONFIGSITE_CACHE = "" # The recipe doesn't need any default deps INHIBIT_DEFAULT_DEPS = "1" -# Directory in testsdk that contains testcases -TESTSDK_CASES = "buildtools-cases" +inherit testsdk -# We have our own code, avoid deferred inherit -SDK_CLASSES:remove = "testsdk" - -python do_testsdk() { - import oeqa.sdk.testsdk - testsdk = oeqa.sdk.testsdk.TestSDK() - - cases_path = os.path.join(os.path.abspath(os.path.dirname(oeqa.sdk.testsdk.__file__)), d.getVar("TESTSDK_CASES")) - testsdk.context_executor_class.default_cases = [cases_path,] - - testsdk.run(d) -} -addtask testsdk -do_testsdk[nostamp] = "1" -do_testsdk[network] = "1" -do_testsdk[depends] += "xz-native:do_populate_sysroot" +# Directory that contains testcases +TESTSDK_CASE_DIRS = "buildtools" \ No newline at end of file -- cgit v1.2.3-54-g00ecf