14 files changed, 228 insertions, 156 deletions

diff --git a/meta/lib/oeqa/core/case.py b/meta/lib/oeqa/core/case.py
index aae451fef2..bc4446a938 100644
--- a/meta/lib/oeqa/core/case.py
+++ b/meta/lib/oeqa/core/case.py
@@ -43,8 +43,13 @@ class OETestCase(unittest.TestCase):
         clss.tearDownClassMethod()
 
     def _oeSetUp(self):
-        for d in self.decorators:
-            d.setUpDecorator()
+        try:
+            for d in self.decorators:
+                d.setUpDecorator()
+        except:
+            for d in self.decorators:
+                d.tearDownDecorator()
+            raise
         self.setUpMethod()
 
     def _oeTearDown(self):
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index 2abe353d27..9313271f58 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -81,7 +81,7 @@ class OETestContext(object):
     def runTests(self, processes=None, skips=[]):
         self.runner = self.runnerClass(self, descriptions=False, verbosity=2)
 
-        # Dinamically skip those tests specified though arguments
+        # Dynamically skip those tests specified though arguments
         self.skipTests(skips)
 
         self._run_start_time = time.time()
diff --git a/meta/lib/oeqa/core/decorator/__init__.py b/meta/lib/oeqa/core/decorator/__init__.py
index 1a82518ab6..93efd30e1d 100644
--- a/meta/lib/oeqa/core/decorator/__init__.py
+++ b/meta/lib/oeqa/core/decorator/__init__.py
@@ -5,8 +5,7 @@
 #
 
 from functools import wraps
-from abc import abstractmethod, ABCMeta
-from oeqa.core.utils.misc import strToList
+from abc import ABCMeta
 
 decoratorClasses = set()
 
@@ -65,15 +64,11 @@ class OETestDiscover(OETestDecorator):
         return registry['cases']
 
 def OETestTag(*tags):
-    expandedtags = []
-    for tag in tags:
-        expandedtags += strToList(tag)
     def decorator(item):
         if hasattr(item, "__oeqa_testtags"):
             # do not append, create a new list (to handle classes with inheritance)
-            item.__oeqa_testtags = list(item.__oeqa_testtags) + expandedtags
+            item.__oeqa_testtags = list(item.__oeqa_testtags) + list(tags)
         else:
-            item.__oeqa_testtags = expandedtags
+            item.__oeqa_testtags = tags
         return item
     return decorator
-
diff --git a/meta/lib/oeqa/core/decorator/data.py b/meta/lib/oeqa/core/decorator/data.py
index bc4939e87c..5444b2cb75 100644
--- a/meta/lib/oeqa/core/decorator/data.py
+++ b/meta/lib/oeqa/core/decorator/data.py
@@ -13,8 +13,8 @@ def has_feature(td, feature):
         Checks for feature in DISTRO_FEATURES or IMAGE_FEATURES.
     """
 
-    if (feature in td.get('DISTRO_FEATURES', '') or
-        feature in td.get('IMAGE_FEATURES', '')):
+    if (feature in td.get('DISTRO_FEATURES', '').split() or
+        feature in td.get('IMAGE_FEATURES', '').split()):
         return True
     return False
 
@@ -23,18 +23,7 @@ def has_machine(td, machine):
         Checks for MACHINE.
     """
 
-    if (machine in td.get('MACHINE', '')):
-        return True
-    return False
-
-def is_qemu(td, qemu):
-    """
-        Checks if MACHINE is qemu.
-    """
-
-    machine = td.get('MACHINE', '')
-    if (qemu in td.get('MACHINE', '') or
-    machine.startswith('qemu')):
+    if (machine == td.get('MACHINE', '')):
         return True
     return False
 
@@ -189,34 +178,53 @@ class skipIfMachine(OETestDecorator):
 @registerDecorator
 class skipIfNotQemu(OETestDecorator):
     """
-        Skip test based on MACHINE.
-
-        value must be a qemu MACHINE or it will skip the test
-        with msg as the reason.
+    Skip test if MACHINE is not qemu*
     """
+    def setUpDecorator(self):
+        self.logger.debug("Checking if not qemu MACHINE")
+        if not self.case.td.get('MACHINE', '').startswith('qemu'):
+            self.case.skipTest('Test only runs on qemu machines')
 
-    attrs = ('value', 'msg')
-
+@registerDecorator
+class skipIfNotQemuUsermode(OETestDecorator):
+    """
+    Skip test if MACHINE_FEATURES does not contain qemu-usermode
+    """
     def setUpDecorator(self):
-        msg = ('Checking if %s is not this MACHINE' % self.value)
-        self.logger.debug(msg)
-        if not is_qemu(self.case.td, self.value):
-            self.case.skipTest(self.msg)
+        self.logger.debug("Checking if MACHINE_FEATURES does not contain qemu-usermode")
+        if 'qemu-usermode' not in self.case.td.get('MACHINE_FEATURES', '').split():
+            self.case.skipTest('Test requires qemu-usermode in MACHINE_FEATURES')
 
 @registerDecorator
 class skipIfQemu(OETestDecorator):
     """
-        Skip test based on Qemu Machine.
-
-        value must not be a qemu machine or it will skip the test
-        with msg as the reason.
-   """
+    Skip test if MACHINE is qemu*
+    """
+    def setUpDecorator(self):
+        self.logger.debug("Checking if qemu MACHINE")
+        if self.case.td.get('MACHINE', '').startswith('qemu'):
+             self.case.skipTest('Test only runs on real hardware')
 
-    attrs = ('value', 'msg')
+@registerDecorator
+class skipIfArch(OETestDecorator):
+    """
+    Skip test if HOST_ARCH is present in the tuple specified.
+    """
 
+    attrs = ('archs',)
     def setUpDecorator(self):
-        msg = ('Checking if %s is this MACHINE' % self.value)
-        self.logger.debug(msg)
-        if is_qemu(self.case.td, self.value):
-             self.case.skipTest(self.msg)
+        arch = self.case.td['HOST_ARCH']
+        if arch in self.archs:
+             self.case.skipTest('Test skipped on %s' % arch)
+
+@registerDecorator
+class skipIfNotArch(OETestDecorator):
+    """
+    Skip test if HOST_ARCH is not present in the tuple specified.
+    """
 
+    attrs = ('archs',)
+    def setUpDecorator(self):
+        arch = self.case.td['HOST_ARCH']
+        if arch not in self.archs:
+             self.case.skipTest('Test skipped on %s' % arch)
diff --git a/meta/lib/oeqa/core/decorator/oetimeout.py b/meta/lib/oeqa/core/decorator/oetimeout.py
index df90d1c798..5e6873ad48 100644
--- a/meta/lib/oeqa/core/decorator/oetimeout.py
+++ b/meta/lib/oeqa/core/decorator/oetimeout.py
@@ -24,5 +24,6 @@ class OETimeout(OETestDecorator):
 
     def tearDownDecorator(self):
         signal.alarm(0)
-        signal.signal(signal.SIGALRM, self.alarmSignal)
-        self.logger.debug("Removed SIGALRM handler")
+        if hasattr(self, 'alarmSignal'):
+            signal.signal(signal.SIGALRM, self.alarmSignal)
+            self.logger.debug("Removed SIGALRM handler")
diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py
index 11978213b8..d12d5a055c 100644
--- a/meta/lib/oeqa/core/loader.py
+++ b/meta/lib/oeqa/core/loader.py
@@ -37,7 +37,7 @@ def _find_duplicated_modules(suite, directory):
         if path:
             raise ImportError("Duplicated %s module found in %s" % (module, path))
 
-def _built_modules_dict(modules):
+def _built_modules_dict(modules, logger):
     modules_dict = {}
 
     if modules == None:
@@ -48,6 +48,9 @@ def _built_modules_dict(modules):
         # characters, whereas class names do
         m = re.match(r'^([0-9a-z_.]+)(?:\.(\w[^.]*)(?:\.([^.]+))?)?$', module, flags=re.ASCII)
         if not m:
+            logger.warn("module '%s' was skipped from selected modules, "\
+                "because it doesn't match with module name assumptions: "\
+                "package and module names do not contain upper case characters, whereas class names do" % module)
             continue
 
         module_name, class_name, test_name = m.groups()
@@ -58,6 +61,8 @@ def _built_modules_dict(modules):
             modules_dict[module_name][class_name] = []
         if test_name and test_name not in modules_dict[module_name][class_name]:
             modules_dict[module_name][class_name].append(test_name)
+    if modules and not modules_dict:
+        raise OEQATestNotFound("All selected modules were skipped, this would trigger selftest with all tests and -r ignored.")
 
     return modules_dict
 
@@ -71,7 +76,7 @@ class OETestLoader(unittest.TestLoader):
             *args, **kwargs):
         self.tc = tc
 
-        self.modules = _built_modules_dict(modules)
+        self.modules = _built_modules_dict(modules, tc.logger)
 
         self.tests = tests
         self.modules_required = modules_required
@@ -311,6 +316,9 @@ class OETestLoader(unittest.TestLoader):
                                   module_name_small in self.modules) \
                                else False
 
+        if any(c.isupper() for c in module.__name__):
+            raise SystemExit("Module '%s' contains uppercase characters and this isn't supported. Please fix the module name." % module.__name__)
+
         return (load_module, load_underscore)
 
 
diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index d50690ab37..a86a706bd9 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -44,6 +44,7 @@ class OETestResult(_TestResult):
         self.endtime = {}
         self.progressinfo = {}
         self.extraresults = {}
+        self.shownmsg = []
 
         # Inject into tc so that TestDepends decorator can see results
         tc.results = self
@@ -74,6 +75,7 @@ class OETestResult(_TestResult):
             for (scase, msg) in getattr(self, t):
                 if test.id() == scase.id():
                     self.tc.logger.info(str(msg))
+                    self.shownmsg.append(test.id())
                     break
 
     def logSummary(self, component, context_msg=''):
@@ -169,7 +171,6 @@ class OETestResult(_TestResult):
 
     def logDetails(self, json_file_dir=None, configuration=None, result_id=None,
             dump_streams=False):
-        self.tc.logger.info("RESULTS:")
 
         result = self.extraresults
         logs = {}
@@ -193,6 +194,10 @@ class OETestResult(_TestResult):
             report = {'status': status}
             if log:
                 report['log'] = log
+                # Class setup failures wouldn't enter stopTest so would never display
+                if case.id() not in self.shownmsg:
+                    self.tc.logger.info("Failure (%s) for %s:\n" % (status, case.id()) + log)
+
             if duration:
                 report['duration'] = duration
 
@@ -215,6 +220,7 @@ class OETestResult(_TestResult):
                 report['stderr'] = stderr
             result[case.id()] = report
 
+        self.tc.logger.info("RESULTS:")
         for i in ['PASSED', 'SKIPPED', 'EXPECTEDFAIL', 'ERROR', 'FAILED', 'UNKNOWN']:
             if i not in logs:
                 continue
@@ -229,6 +235,10 @@ class OETestResult(_TestResult):
         # Override as we unexpected successes aren't failures for us
         return (len(self.failures) == len(self.errors) == 0)
 
+    def hasAnyFailingTest(self):
+        # Account for expected failures
+        return not self.wasSuccessful() or len(self.expectedFailures)
+
 class OEListTestsResult(object):
     def wasSuccessful(self):
         return True
diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py
index 0f29414df5..d93b3ac94a 100644
--- a/meta/lib/oeqa/core/target/qemu.py
+++ b/meta/lib/oeqa/core/target/qemu.py
@@ -8,20 +8,21 @@ import os
 import sys
 import signal
 import time
+import glob
+import subprocess
 from collections import defaultdict
 
 from .ssh import OESSHTarget
 from oeqa.utils.qemurunner import QemuRunner
-from oeqa.utils.dump import TargetDumper
 
 supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic']
 
 class OEQemuTarget(OESSHTarget):
     def __init__(self, logger, server_ip, timeout=300, user='root',
             port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False,
-            dump_dir='', dump_host_cmds='', display='', bootlog='',
+            dump_dir='', display='', bootlog='',
             tmpdir='', dir_image='', boottime=60, serial_ports=2,
-            boot_patterns = defaultdict(str), ovmf=False, **kwargs):
+            boot_patterns = defaultdict(str), ovmf=False, tmpfsdir=None, **kwargs):
 
         super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout,
                 user, port)
@@ -35,17 +36,15 @@ class OEQemuTarget(OESSHTarget):
         self.ovmf = ovmf
         self.use_slirp = slirp
         self.boot_patterns = boot_patterns
+        self.dump_dir = dump_dir
+        self.bootlog = bootlog
 
         self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir,
                                  deploy_dir_image=dir_image, display=display,
                                  logfile=bootlog, boottime=boottime,
-                                 use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir,
-                                 dump_host_cmds=dump_host_cmds, logger=logger,
+                                 use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, logger=logger,
                                  serial_ports=serial_ports, boot_patterns = boot_patterns, 
-                                 use_ovmf=ovmf)
-        dump_target_cmds = kwargs.get("testimage_dump_target")
-        self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner)
-        self.target_dumper.create_dir("qemu")
+                                 use_ovmf=ovmf, tmpfsdir=tmpfsdir)
 
     def start(self, params=None, extra_bootparams=None, runqemuparams=''):
         if self.use_slirp and not self.server_ip:
@@ -68,7 +67,28 @@ class OEQemuTarget(OESSHTarget):
                 self.server_ip = self.runner.server_ip
         else:
             self.stop()
-            raise RuntimeError("FAILED to start qemu - check the task log and the boot log")
+            # Display the first 20 lines of top and
+            # last 20 lines of the bootlog when the
+            # target is not being booted up.
+            topfile = glob.glob(self.dump_dir + "/*_qemu/host_*_top")
+            msg = "\n\n===== start: snippet =====\n\n"
+            for f in topfile:
+                msg += "file: %s\n\n" % f
+                with open(f) as tf:
+                    for x in range(20):
+                        msg += next(tf)
+            msg += "\n\n===== end: snippet =====\n\n"
+            blcmd = ["tail", "-20", self.bootlog]
+            msg += "===== start: snippet =====\n\n"
+            try:
+                out = subprocess.check_output(blcmd, stderr=subprocess.STDOUT, timeout=1).decode('utf-8')
+                msg += "file: %s\n\n" % self.bootlog
+                msg += out
+            except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as err:
+                msg += "Error running command: %s\n%s\n" % (blcmd, err)
+            msg += "\n\n===== end: snippet =====\n"
+
+            raise RuntimeError("FAILED to start qemu - check the task log and the boot log %s" % (msg))
 
     def stop(self):
         self.runner.stop()
diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py
index 461448dbc5..09cdd14c75 100644
--- a/meta/lib/oeqa/core/target/ssh.py
+++ b/meta/lib/oeqa/core/target/ssh.py
@@ -34,12 +34,17 @@ class OESSHTarget(OETarget):
         self.timeout = timeout
         self.user = user
         ssh_options = [
+                '-o', 'ServerAliveCountMax=2',
+                '-o', 'ServerAliveInterval=30',
                 '-o', 'UserKnownHostsFile=/dev/null',
                 '-o', 'StrictHostKeyChecking=no',
                 '-o', 'LogLevel=ERROR'
                 ]
+        scp_options = [
+                '-r'
+        ]
         self.ssh = ['ssh', '-l', self.user ] + ssh_options
-        self.scp = ['scp'] + ssh_options
+        self.scp = ['scp'] + ssh_options + scp_options
         if port:
             self.ssh = self.ssh + [ '-p', port ]
             self.scp = self.scp + [ '-P', port ]
@@ -67,7 +72,7 @@ class OESSHTarget(OETarget):
 
         return (status, output)
 
-    def run(self, command, timeout=None):
+    def run(self, command, timeout=None, ignore_status=True):
         """
             Runs command in target.
 
@@ -86,10 +91,9 @@ class OESSHTarget(OETarget):
         else:
             processTimeout = self.timeout
 
-        status, output = self._run(sshCmd, processTimeout, True)
-        self.logger.debug('Command: %s\nOutput:  %s\n' % (command, output))
-        if (status == 255) and (('No route to host') in output):
-            self.target_dumper.dump_target()
+        status, output = self._run(sshCmd, processTimeout, ignore_status)
+        self.logger.debug('Command: %s\nStatus: %d Output:  %s\n' % (command, status, output))
+
         return (status, output)
 
     def copyTo(self, localSrc, remoteDst):
@@ -207,27 +211,41 @@ def SSHCall(command, logger, timeout=None, **opts):
     def run():
         nonlocal output
         nonlocal process
+        output_raw = b''
         starttime = time.time()
         process = subprocess.Popen(command, **options)
+        has_timeout = False
         if timeout:
             endtime = starttime + timeout
             eof = False
-            while time.time() < endtime and not eof:
-                logger.debug('time: %s, endtime: %s' % (time.time(), endtime))
+            os.set_blocking(process.stdout.fileno(), False)
+            while not has_timeout and not eof:
                 try:
+                    logger.debug('Waiting for process output: time: %s, endtime: %s' % (time.time(), endtime))
                     if select.select([process.stdout], [], [], 5)[0] != []:
-                        reader = codecs.getreader('utf-8')(process.stdout, 'ignore')
-                        data = reader.read(1024, 4096)
+                        # wait a bit for more data, tries to avoid reading single characters
+                        time.sleep(0.2)
+                        data = process.stdout.read()
                         if not data:
-                            process.stdout.close()
                             eof = True
                         else:
-                            output += data
-                            logger.debug('Partial data from SSH call: %s' % data)
+                            output_raw += data
+                            # ignore errors to capture as much as possible
+                            logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore'))
                             endtime = time.time() + timeout
                 except InterruptedError:
+                    logger.debug('InterruptedError')
+                    continue
+                except BlockingIOError:
+                    logger.debug('BlockingIOError')
                     continue
 
+                if time.time() >= endtime:
+                    logger.debug('SSHCall has timeout! Time: %s, endtime: %s' % (time.time(), endtime))
+                    has_timeout = True
+
+            process.stdout.close()
+
             # process hasn't returned yet
             if not eof:
                 process.terminate()
@@ -235,16 +253,42 @@ def SSHCall(command, logger, timeout=None, **opts):
                 try:
                     process.kill()
                 except OSError:
+                    logger.debug('OSError when killing process')
                     pass
                 endtime = time.time() - starttime
                 lastline = ("\nProcess killed - no output for %d seconds. Total"
                             " running time: %d seconds." % (timeout, endtime))
-                logger.debug('Received data from SSH call %s ' % lastline)
+                logger.debug('Received data from SSH call:\n%s ' % lastline)
                 output += lastline
+                process.wait()
 
         else:
-            output = process.communicate()[0].decode('utf-8', errors='ignore')
-            logger.debug('Data from SSH call: %s' % output.rstrip())
+            output_raw = process.communicate()[0]
+
+        output = output_raw.decode('utf-8', errors='ignore')
+        logger.debug('Data from SSH call:\n%s' % output.rstrip())
+
+        # timout or not, make sure process exits and is not hanging
+        if process.returncode == None:
+            try:
+                process.wait(timeout=5)
+            except TimeoutExpired:
+                try:
+                    process.kill()
+                except OSError:
+                    logger.debug('OSError')
+                    pass
+                process.wait()
+
+        if has_timeout:
+            # Version of openssh before 8.6_p1 returns error code 0 when killed
+            # by a signal, when the timeout occurs we will receive a 0 error
+            # code because the process is been terminated and it's wrong because
+            # that value means success, but the process timed out.
+            # Afterwards, from version 8.6_p1 onwards, the returned code is 255.
+            # Fix this behaviour by checking the return code
+            if process.returncode == 0:
+                process.returncode = 255
 
     options = {
         "stdout": subprocess.PIPE,
@@ -271,6 +315,9 @@ def SSHCall(command, logger, timeout=None, **opts):
         # whilst running and ensure we don't leave a process behind.
         if process.poll() is None:
             process.kill()
+        if process.returncode == None:
+            process.wait()
         logger.debug('Something went wrong, killing SSH process')
         raise
-    return (process.wait(), output.rstrip())
+
+    return (process.returncode, output.rstrip())
diff --git a/meta/lib/oeqa/core/tests/cases/timeout.py b/meta/lib/oeqa/core/tests/cases/timeout.py
index 5dfecc7b7c..69cf969a67 100644
--- a/meta/lib/oeqa/core/tests/cases/timeout.py
+++ b/meta/lib/oeqa/core/tests/cases/timeout.py
@@ -8,6 +8,7 @@ from time import sleep
 
 from oeqa.core.case import OETestCase
 from oeqa.core.decorator.oetimeout import OETimeout
+from oeqa.core.decorator.depends import OETestDepends
 
 class TimeoutTest(OETestCase):
 
@@ -19,3 +20,15 @@ class TimeoutTest(OETestCase):
     def testTimeoutFail(self):
         sleep(2)
         self.assertTrue(True, msg='How is this possible?')
+
+
+    def testTimeoutSkip(self):
+        self.skipTest("This test needs to be skipped, so that testTimeoutDepends()'s OETestDepends kicks in")
+
+    @OETestDepends(["timeout.TimeoutTest.testTimeoutSkip"])
+    @OETimeout(3)
+    def testTimeoutDepends(self):
+        self.assertTrue(False, msg='How is this possible?')
+
+    def testTimeoutUnrelated(self):
+        sleep(6)
diff --git a/meta/lib/oeqa/core/tests/test_data.py b/meta/lib/oeqa/core/tests/test_data.py
index ac74098b78..acd726f3a0 100755
--- a/meta/lib/oeqa/core/tests/test_data.py
+++ b/meta/lib/oeqa/core/tests/test_data.py
@@ -33,7 +33,7 @@ class TestData(TestBase):
 
     def test_data_fail_wrong_variable(self):
         expectedError = 'AssertionError'
-        d = {'IMAGE' : 'core-image-sato', 'ARCH' : 'arm'}
+        d = {'IMAGE' : 'core-image-weston', 'ARCH' : 'arm'}
 
         tc = self._testLoader(d=d, modules=self.modules)
         results = tc.runTests()
diff --git a/meta/lib/oeqa/core/tests/test_decorators.py b/meta/lib/oeqa/core/tests/test_decorators.py
index b798bf7d33..5095f39948 100755
--- a/meta/lib/oeqa/core/tests/test_decorators.py
+++ b/meta/lib/oeqa/core/tests/test_decorators.py
@@ -133,5 +133,11 @@ class TestTimeoutDecorator(TestBase):
         msg = "OETestTimeout didn't restore SIGALRM"
         self.assertIs(alarm_signal, signal.getsignal(signal.SIGALRM), msg=msg)
 
+    def test_timeout_cancel(self):
+        tests = ['timeout.TimeoutTest.testTimeoutSkip', 'timeout.TimeoutTest.testTimeoutDepends', 'timeout.TimeoutTest.testTimeoutUnrelated']
+        msg = 'Unrelated test failed to complete'
+        tc = self._testLoader(modules=self.modules, tests=tests)
+        self.assertTrue(tc.runTests().wasSuccessful(), msg=msg)
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py
index b2eb68fb02..d10f8f7f04 100644
--- a/meta/lib/oeqa/core/utils/concurrencytest.py
+++ b/meta/lib/oeqa/core/utils/concurrencytest.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 #
+# Copyright OpenEmbedded Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-or-later
 #
 # Modified for use in OE by Richard Purdie, 2018
@@ -48,11 +50,16 @@ _all__ = [
 #
 class BBThreadsafeForwardingResult(ThreadsafeForwardingResult):
 
-    def __init__(self, target, semaphore, threadnum, totalinprocess, totaltests):
+    def __init__(self, target, semaphore, threadnum, totalinprocess, totaltests, output, finalresult):
         super(BBThreadsafeForwardingResult, self).__init__(target, semaphore)
         self.threadnum = threadnum
         self.totalinprocess = totalinprocess
         self.totaltests = totaltests
+        self.buffer = True
+        self.outputbuf = output
+        self.finalresult = finalresult
+        self.finalresult.buffer = True
+        self.target = target
 
     def _add_result_with_semaphore(self, method, test, *args, **kwargs):
         self.semaphore.acquire()
@@ -61,16 +68,19 @@ class BBThreadsafeForwardingResult(ThreadsafeForwardingResult):
                 self.result.starttime[test.id()] = self._test_start.timestamp()
                 self.result.threadprogress[self.threadnum].append(test.id())
                 totalprogress = sum(len(x) for x in self.result.threadprogress.values())
-                self.result.progressinfo[test.id()] = "%s: %s/%s %s/%s (%ss) (%s)" % (
+                self.result.progressinfo[test.id()] = "%s: %s/%s %s/%s (%ss) (%s failed) (%s)" % (
                     self.threadnum,
                     len(self.result.threadprogress[self.threadnum]),
                     self.totalinprocess,
                     totalprogress,
                     self.totaltests,
                     "{0:.2f}".format(time.time()-self._test_start.timestamp()),
+                    self.target.failed_tests,
                     test.id())
         finally:
             self.semaphore.release()
+        self.finalresult._stderr_buffer = io.StringIO(initial_value=self.outputbuf.getvalue().decode("utf-8"))
+        self.finalresult._stdout_buffer = io.StringIO()
         super(BBThreadsafeForwardingResult, self)._add_result_with_semaphore(method, test, *args, **kwargs)
 
 class ProxyTestResult:
@@ -183,35 +193,28 @@ class dummybuf(object):
 #
 class ConcurrentTestSuite(unittest.TestSuite):
 
-    def __init__(self, suite, processes, setupfunc, removefunc):
+    def __init__(self, suite, processes, setupfunc, removefunc, bb_vars):
         super(ConcurrentTestSuite, self).__init__([suite])
         self.processes = processes
         self.setupfunc = setupfunc
         self.removefunc = removefunc
+        self.bb_vars = bb_vars
 
     def run(self, result):
-        tests, totaltests = fork_for_tests(self.processes, self)
+        testservers, totaltests = fork_for_tests(self.processes, self)
         try:
             threads = {}
             queue = Queue()
             semaphore = threading.Semaphore(1)
             result.threadprogress = {}
-            for i, (test, testnum) in enumerate(tests):
+            for i, (testserver, testnum, output) in enumerate(testservers):
                 result.threadprogress[i] = []
                 process_result = BBThreadsafeForwardingResult(
                         ExtraResultsDecoderTestResult(result),
-                        semaphore, i, testnum, totaltests)
-                # Force buffering of stdout/stderr so the console doesn't get corrupted by test output
-                # as per default in parent code
-                process_result.buffer = True
-                # We have to add a buffer object to stdout to keep subunit happy
-                process_result._stderr_buffer = io.StringIO()
-                process_result._stderr_buffer.buffer = dummybuf(process_result._stderr_buffer)
-                process_result._stdout_buffer = io.StringIO()
-                process_result._stdout_buffer.buffer = dummybuf(process_result._stdout_buffer)
+                        semaphore, i, testnum, totaltests, output, result)
                 reader_thread = threading.Thread(
-                    target=self._run_test, args=(test, process_result, queue))
-                threads[test] = reader_thread, process_result
+                    target=self._run_test, args=(testserver, process_result, queue))
+                threads[testserver] = reader_thread, process_result
                 reader_thread.start()
             while threads:
                 finished_test = queue.get()
@@ -222,13 +225,13 @@ class ConcurrentTestSuite(unittest.TestSuite):
                 process_result.stop()
             raise
         finally:
-            for test in tests:
-                test[0]._stream.close()
+            for testserver in testservers:
+                testserver[0]._stream.close()
 
-    def _run_test(self, test, process_result, queue):
+    def _run_test(self, testserver, process_result, queue):
         try:
             try:
-                test.run(process_result)
+                testserver.run(process_result)
             except Exception:
                 # The run logic itself failed
                 case = testtools.ErrorHolder(
@@ -236,12 +239,12 @@ class ConcurrentTestSuite(unittest.TestSuite):
                     error=sys.exc_info())
                 case.run(process_result)
         finally:
-            queue.put(test)
+            queue.put(testserver)
 
 def fork_for_tests(concurrency_num, suite):
-    result = []
+    testservers = []
     if 'BUILDDIR' in os.environ:
-        selftestdir = get_test_layer()
+        selftestdir = get_test_layer(suite.bb_vars['BBLAYERS'])
 
     test_blocks = partition_tests(suite, concurrency_num)
     # Clear the tests from the original suite so it doesn't keep them alive
@@ -261,7 +264,7 @@ def fork_for_tests(concurrency_num, suite):
             ourpid = os.getpid()
             try:
                 newbuilddir = None
-                stream = os.fdopen(c2pwrite, 'wb', 1)
+                stream = os.fdopen(c2pwrite, 'wb')
                 os.close(c2pread)
 
                 (builddir, newbuilddir) = suite.setupfunc("-st-" + str(ourpid), selftestdir, process_suite)
@@ -273,10 +276,11 @@ def fork_for_tests(concurrency_num, suite):
                 newsi = os.open(os.devnull, os.O_RDWR)
                 os.dup2(newsi, sys.stdin.fileno())
 
+                # Send stdout/stderr over the stream
+                os.dup2(c2pwrite, sys.stdout.fileno())
+                os.dup2(c2pwrite, sys.stderr.fileno())
+
                 subunit_client = TestProtocolClient(stream)
-                # Force buffering of stdout/stderr so the console doesn't get corrupted by test output
-                # as per default in parent code
-                subunit_client.buffer = True
                 subunit_result = AutoTimingTestResultDecorator(subunit_client)
                 unittest_result = process_suite.run(ExtraResultsEncoderTestResult(subunit_result))
                 if ourpid != os.getpid():
@@ -305,10 +309,12 @@ def fork_for_tests(concurrency_num, suite):
             os._exit(0)
         else:
             os.close(c2pwrite)
-            stream = os.fdopen(c2pread, 'rb', 1)
-            test = ProtocolTestCase(stream)
-            result.append((test, numtests))
-    return result, totaltests
+            stream = os.fdopen(c2pread, 'rb')
+            # Collect stdout/stderr into an io buffer
+            output = io.BytesIO()
+            testserver = ProtocolTestCase(stream, passthrough=output)
+            testservers.append((testserver, numtests, output))
+    return testservers, totaltests
 
 def partition_tests(suite, count):
     # Keep tests from the same class together but allow tests from modules
diff --git a/meta/lib/oeqa/core/utils/misc.py b/meta/lib/oeqa/core/utils/misc.py
deleted file mode 100644
index e1a59588eb..0000000000
--- a/meta/lib/oeqa/core/utils/misc.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Copyright (C) 2016 Intel Corporation
-#
-# SPDX-License-Identifier: MIT
-#
-
-def toList(obj, obj_type, obj_name="Object"):
-    if isinstance(obj, obj_type):
-        return [obj]
-    elif isinstance(obj, list):
-        return obj
-    else:
-        raise TypeError("%s must be %s or list" % (obj_name, obj_type))
-
-def toSet(obj, obj_type, obj_name="Object"):
-    if isinstance(obj, obj_type):
-        return {obj}
-    elif isinstance(obj, list):
-        return set(obj)
-    elif isinstance(obj, set):
-        return obj
-    else:
-        raise TypeError("%s must be %s or set" % (obj_name, obj_type))
-
-def strToList(obj, obj_name="Object"):
-    return toList(obj, str, obj_name)
-
-def strToSet(obj, obj_name="Object"):
-    return toSet(obj, str, obj_name)
-
-def intToList(obj, obj_name="Object"):
-    return toList(obj, int, obj_name)
-
-def dataStoteToDict(d, variables):
-    data = {}
-
-    for v in variables:
-        data[v] = d.getVar(v)
-
-    return data
-
-def updateTestData(d, td, variables):
-    """
-    Updates variables with values of data store to test data.
-    """
-    for var in variables:
-        td[var] = d.getVar(var)