2 files changed, 246 insertions, 0 deletions

diff --git a/meta/lib/oeqa/sdk/testsdk.py b/meta/lib/oeqa/sdk/testsdk.py
new file mode 100644
index 0000000000..632ac50d0c
--- /dev/null
+++ b/meta/lib/oeqa/sdk/testsdk.py
@@ -0,0 +1,142 @@
+# Copyright 2018 by Garmin Ltd. or its subsidiaries
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.sdk.context import OESDKTestContext, OESDKTestContextExecutor
+
+class TestSDKBase(object):
+    @staticmethod
+    def get_sdk_configuration(d, test_type):
+        import platform
+        import oe.lsb
+        from oeqa.utils.metadata import get_layers
+        configuration = {'TEST_TYPE': test_type,
+                        'MACHINE': d.getVar("MACHINE"),
+                        'SDKMACHINE': d.getVar("SDKMACHINE"),
+                        'IMAGE_BASENAME': d.getVar("IMAGE_BASENAME"),
+                        'IMAGE_PKGTYPE': d.getVar("IMAGE_PKGTYPE"),
+                        'STARTTIME': d.getVar("DATETIME"),
+                        'HOST_DISTRO': oe.lsb.distro_identifier().replace(' ', '-'),
+                        'LAYERS': get_layers(d.getVar("BBLAYERS"))}
+        return configuration
+
+    @staticmethod
+    def get_sdk_json_result_dir(d):
+        json_result_dir = os.path.join(d.getVar("LOG_DIR"), 'oeqa')
+        custom_json_result_dir = d.getVar("OEQA_JSON_RESULT_DIR")
+        if custom_json_result_dir:
+            json_result_dir = custom_json_result_dir
+        return json_result_dir
+
+    @staticmethod
+    def get_sdk_result_id(configuration):
+        return '%s_%s_%s_%s_%s' % (configuration['TEST_TYPE'], configuration['IMAGE_BASENAME'], configuration['SDKMACHINE'], configuration['MACHINE'], configuration['STARTTIME'])
+
+class TestSDK(TestSDKBase):
+    context_executor_class = OESDKTestContextExecutor
+    context_class = OESDKTestContext
+    test_type = 'sdk'
+
+    def get_tcname(self, d):
+        """
+        Get the name of the SDK file
+        """
+        return d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh")
+
+    def extract_sdk(self, tcname, sdk_dir, d):
+        """
+        Extract the SDK to the specified location
+        """
+        import subprocess
+
+        try:
+            subprocess.check_output("cd %s; %s <<EOF\n./\nY\nEOF" % (sdk_dir, tcname), shell=True)
+        except subprocess.CalledProcessError as e:
+            bb.fatal("Couldn't install the SDK:\n%s" % e.output.decode("utf-8"))
+
+    def setup_context(self, d):
+        """
+        Return a dictionary of additional arguments that should be passed to
+        the context_class on construction
+        """
+        return dict()
+
+    def run(self, d):
+
+        import os
+        import subprocess
+        import json
+        import logging
+
+        from bb.utils import export_proxies
+        from oeqa.utils import make_logger_bitbake_compatible
+
+        pn = d.getVar("PN")
+        logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
+
+        # sdk use network for download projects for build
+        export_proxies(d)
+
+        tcname = self.get_tcname(d)
+
+        if not os.path.exists(tcname):
+            bb.fatal("The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ." % tcname)
+
+        tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json")
+        test_data = json.load(open(tdname, "r"))
+
+        target_pkg_manifest = self.context_executor_class._load_manifest(
+            d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"))
+        host_pkg_manifest = self.context_executor_class._load_manifest(
+            d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest"))
+
+        processes = d.getVar("TESTIMAGE_NUMBER_THREADS") or d.getVar("BB_NUMBER_THREADS")
+        if processes:
+            try:
+                import testtools, subunit
+            except ImportError:
+                bb.warn("Failed to import testtools or subunit, the testcases will run serially")
+                processes = None
+
+        sdk_dir = d.expand("${WORKDIR}/testimage-sdk/")
+        bb.utils.remove(sdk_dir, True)
+        bb.utils.mkdirhier(sdk_dir)
+
+        context_args = self.setup_context(d)
+
+        self.extract_sdk(tcname, sdk_dir, d)
+
+        fail = False
+        sdk_envs = self.context_executor_class._get_sdk_environs(sdk_dir)
+        for s in sdk_envs:
+            sdk_env = sdk_envs[s]
+            bb.plain("SDK testing environment: %s" % s)
+            tc = self.context_class(td=test_data, logger=logger, sdk_dir=sdk_dir,
+                sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
+                host_pkg_manifest=host_pkg_manifest, **context_args)
+
+            try:
+                tc.loadTests(self.context_executor_class.default_cases)
+            except Exception as e:
+                import traceback
+                bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
+
+            if processes:
+                result = tc.runTests(processes=int(processes))
+            else:
+                result = tc.runTests()
+
+            component = "%s %s" % (pn, self.context_executor_class.name)
+            context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
+            configuration = self.get_sdk_configuration(d, self.test_type)
+            result.logDetails(self.get_sdk_json_result_dir(d),
+                            configuration,
+                            self.get_sdk_result_id(configuration))
+            result.logSummary(component, context_msg)
+
+            if not result.wasSuccessful():
+                fail = True
+
+        if fail:
+            bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
+
+
diff --git a/meta/lib/oeqa/sdkext/testsdk.py b/meta/lib/oeqa/sdkext/testsdk.py
new file mode 100644
index 0000000000..57b2e0e03f
--- /dev/null
+++ b/meta/lib/oeqa/sdkext/testsdk.py
@@ -0,0 +1,104 @@
+# Copyright 2018 by Garmin Ltd. or its subsidiaries
+# Released under the MIT license (see COPYING.MIT)
+
+from oeqa.sdk.testsdk import TestSDKBase
+
+class TestSDKExt(TestSDKBase):
+    def run(self, d):
+        import os
+        import json
+        import subprocess
+        import logging
+
+        from bb.utils import export_proxies
+        from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak
+        from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor
+
+        pn = d.getVar("PN")
+        logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
+
+        # extensible sdk use network
+        export_proxies(d)
+
+        subprocesstweak.errors_have_output()
+
+        # extensible sdk can be contaminated if native programs are
+        # in PATH, i.e. use perl-native instead of eSDK one.
+        paths_to_avoid = [d.getVar('STAGING_DIR'),
+                        d.getVar('BASE_WORKDIR')]
+        os.environ['PATH'] = avoid_paths_in_environ(paths_to_avoid)
+
+        tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh")
+        if not os.path.exists(tcname):
+            bb.fatal("The toolchain ext %s is not built. Build it before running the" \
+                    " tests: 'bitbake <image> -c populate_sdk_ext' ." % tcname)
+
+        tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.testdata.json")
+        test_data = json.load(open(tdname, "r"))
+
+        target_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
+            d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"))
+        host_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
+            d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"))
+
+        sdk_dir = d.expand("${WORKDIR}/testsdkext/")
+        bb.utils.remove(sdk_dir, True)
+        bb.utils.mkdirhier(sdk_dir)
+        try:
+            subprocess.check_output("%s -y -d %s" % (tcname, sdk_dir), shell=True)
+        except subprocess.CalledProcessError as e:
+            msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8")
+            logfn = os.path.join(sdk_dir, 'preparing_build_system.log')
+            if os.path.exists(logfn):
+                msg += '\n\nContents of preparing_build_system.log:\n'
+                with open(logfn, 'r') as f:
+                    for line in f:
+                        msg += line
+            bb.fatal(msg)
+
+        fail = False
+        sdk_envs = OESDKExtTestContextExecutor._get_sdk_environs(sdk_dir)
+        for s in sdk_envs:
+            bb.plain("Extensible SDK testing environment: %s" % s)
+
+            sdk_env = sdk_envs[s]
+
+            # Use our own SSTATE_DIR and DL_DIR so that updates to the eSDK come from our sstate cache
+            # and we don't spend hours downloading kernels for the kernel module test
+            # Abuse auto.conf since local.conf would be overwritten by the SDK
+            with open(os.path.join(sdk_dir, 'conf', 'auto.conf'), 'a+') as f:
+                f.write('SSTATE_MIRRORS += " \\n file://.* file://%s/PATH"\n' % test_data.get('SSTATE_DIR'))
+                f.write('SOURCE_MIRROR_URL = "file://%s"\n' % test_data.get('DL_DIR'))
+                f.write('INHERIT += "own-mirrors"\n')
+                f.write('PREMIRRORS_prepend = " git://git.yoctoproject.org/.* git://%s/git2/git.yoctoproject.org.BASENAME \\n "\n' % test_data.get('DL_DIR'))
+
+            # We need to do this in case we have a minimal SDK
+            subprocess.check_output(". %s > /dev/null; devtool sdk-install meta-extsdk-toolchain" % \
+                    sdk_env, cwd=sdk_dir, shell=True, stderr=subprocess.STDOUT)
+
+            tc = OESDKExtTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir,
+                sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
+                host_pkg_manifest=host_pkg_manifest)
+
+            try:
+                tc.loadTests(OESDKExtTestContextExecutor.default_cases)
+            except Exception as e:
+                import traceback
+                bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
+
+            result = tc.runTests()
+
+            component = "%s %s" % (pn, OESDKExtTestContextExecutor.name)
+            context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
+            configuration = self.get_sdk_configuration(d, 'sdkext')
+            result.logDetails(self.get_sdk_json_result_dir(d),
+                            configuration,
+                            self.get_sdk_result_id(configuration))
+            result.logSummary(component, context_msg)
+
+            if not result.wasSuccessful():
+                fail = True
+
+        if fail:
+            bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
+