classes/testsdk: Split implementation into classes

Splits the SDK test implementation into configurable Python classes. The
classes used for the normal and extensible SDKs are
${TESTSDK_CLASS_NAME} and ${TESTSDKEXT_CLASS_NAME} respectively.

This allows SDK machines to override the classes used to implement the
tests. For the traditional SDK, a common "run()" function is provided by
the class (oeqa.sdk.testsdk.TestSDK), with several hook member functions
that can be overridden in child classes, making it easier to have
consistent behavior. The extensible SDK class
(oeqa.sdkext.testsdk.TestSDKEXT) also has a common "run()" function, but
no hooks have yet been added as there is not currently a known use case
for create derived classes.

These changes should be purely organizational; no functional changes
have been made to either the standard SDK or extensible SDK tests.

[YOCTO #13020]

(From OE-Core rev: a06d53928b22d5f88276023c4d57b206db2f27f9)

(From OE-Core rev: d0828522fc3a4bd3f86488edcedcce10833c18dc)

Signed-off-by: Joshua Watt <JPEWhacker@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Armin Kuster <akuster808@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 142 insertions, 0 deletions

diff --git a/meta/lib/oeqa/sdk/testsdk.py b/meta/lib/oeqa/sdk/testsdk.py
new file mode 100644
index 0000000000..632ac50d0c
--- /dev/null
+++ b/meta/lib/oeqa/sdk/testsdk.py
@@ -0,0 +1,142 @@
+# Copyright 2018 by Garmin Ltd. or its subsidiaries
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor
+
+class TestSDKBase(object):
+    @staticmethod
+    def get_sdk_configuration(d, test_type):
+        import platform
+        import oe.lsb
+        from oeqa.utils.metadata import get_layers
+        configuration = {'TEST_TYPE': test_type,
+                        'MACHINE': d.getVar("MACHINE"),
+                        'SDKMACHINE': d.getVar("SDKMACHINE"),
+                        'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+                        'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+                        'STARTTIME': d.getVar("DATETIME"),
+                        'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
+                        'LAYERS': get_layers(d.getVar("BBLAYERS"))}
+        return configuration
+
+    @staticmethod
+    def get_sdk_json_result_dir(d):
+        json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa')
+        custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+        if custom_json_result_dir:
+            json_result_dir = custom_json_result_dir
+        return json_result_dir
+
+    @staticmethod
+    def get_sdk_result_id(configuration):
+        return '%s_%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['SDKMACHINE'], configuration['MACHINE'], configuration['STARTTIME'])
+
+class TestSDK(TestSDKBase):
+    context_executor_class = OESDKTestContextExecutor
+    context_class = OESDKTestContext
+    test_type = 'sdk'
+
+    def get_tcname(self, d):
+        """
+        Get the name of the SDK file
+        """
+        return d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh")
+
+    def extract_sdk(self, tcname, sdk_dir, d):
+        """
+        Extract the SDK to the specified location
+        """
+        import subprocess
+
+        try:
+            subprocess.check_output("cd %s; %s <<EOF\n./\nY\nEOF" % (sdk_dir, tcname), shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.fatal("Couldn't install the SDK:\n%s" % e.output.decode("utf-8"))
+
+    def setup_context(self, d):
+        """
+        Return a dictionary of additional arguments that should be passed to
+        the context_class on construction
+        """
+        return dict()
+
+    def run(self, d):
+
+        import os
+        import subprocess
+        import json
+        import logging
+
+        from bb.utils import export_proxies
+        from oeqa.utils import make_logger_bitbake_compatible
+
+        pn = d.getVar("PN")
+        logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
+
+        # sdk use network for download projects for build
+        export_proxies(d)
+
+        tcname = self.get_tcname(d)
+
+        if not os.path.exists(tcname):
+            bb.fatal("The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ." % tcname)
+
+        tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json")
+        test_data = json.load(open(tdname, "r"))
+
+        target_pkg_manifest = self.context_executor_class._load_manifest(
+            d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"))
+        host_pkg_manifest = self.context_executor_class._load_manifest(
+            d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest"))
+
+        processes = d.getVar("TESTIMAGE_NUMBER_THREADS") or d.getVar("BB_NUMBER_THREADS")
+        if processes:
+            try:
+                import testtools, subunit
+            except ImportError:
+                bb.warn("Failed to import testtools or subunit, the testcases will run serially")
+                processes = None
+
+        sdk_dir = d.expand("${WORKDIR}/testimage-sdk/")
+        bb.utils.remove(sdk_dir, True)
+        bb.utils.mkdirhier(sdk_dir)
+
+        context_args = self.setup_context(d)
+
+        self.extract_sdk(tcname, sdk_dir, d)
+
+        fail = False
+        sdk_envs = self.context_executor_class._get_sdk_environs(sdk_dir)
+        for s in sdk_envs:
+            sdk_env = sdk_envs[s]
+            bb.plain("SDK testing environment: %s" % s)
+            tc = self.context_class(td=test_data, logger=logger, sdk_dir=sdk_dir,
+                sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
+                host_pkg_manifest=host_pkg_manifest, **context_args)
+
+            try:
+                tc.loadTests(self.context_executor_class.default_cases)
+            except Exception as e:
+                import traceback
+                bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
+
+            if processes:
+                result = tc.runTests(processes=int(processes))
+            else:
+                result = tc.runTests()
+
+            component = "%s %s" % (pn, self.context_executor_class.name)
+            context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
+            configuration = self.get_sdk_configuration(d, self.test_type)
+            result.logDetails(self.get_sdk_json_result_dir(d),
+                            configuration,
+                            self.get_sdk_result_id(configuration))
+            result.logSummary(component, context_msg)
+
+            if not result.wasSuccessful():
+                fail = True
+
+        if fail:
+            bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
+
+