9 files changed, 388 insertions, 22 deletions

diff --git a/meta/lib/oeqa/core/case.py b/meta/lib/oeqa/core/case.py
index bc4446a938..ad5524a714 100644
--- a/meta/lib/oeqa/core/case.py
+++ b/meta/lib/oeqa/core/case.py
@@ -5,6 +5,7 @@
 #
 
 import base64
+import os
 import zlib
 import unittest
 
@@ -57,6 +58,13 @@ class OETestCase(unittest.TestCase):
             d.tearDownDecorator()
         self.tearDownMethod()
 
+    def assertFileExists(self, filename, msg=None):
+        """
+        Test that filename exists. If it does not, the test will fail.
+        """
+        if not os.path.exists(filename):
+            self.fail(msg or "%s does not exist" % filename)
+
 class OEPTestResultTestCase:
     """
     Mix-in class to provide functions to make interacting with extraresults for
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index 9313271f58..46de9135c1 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -179,9 +179,16 @@ class OETestContextExecutor(object):
         else:
             self.tc_kwargs['init']['td'] = {}
 
+        # Run image specific TEST_SUITE like testimage.bbclass by default
+        test_suites = self.tc_kwargs['init']['td'].get("TEST_SUITES")
+        if test_suites:
+            test_suites = test_suites.split()
+
         if args.run_tests:
             self.tc_kwargs['load']['modules'] = args.run_tests
             self.tc_kwargs['load']['modules_required'] = args.run_tests
+        elif test_suites:
+            self.tc_kwargs['load']['modules'] = test_suites
         else:
             self.tc_kwargs['load']['modules'] = []
 
diff --git a/meta/lib/oeqa/core/decorator/data.py b/meta/lib/oeqa/core/decorator/data.py
index 5444b2cb75..0daf46334f 100644
--- a/meta/lib/oeqa/core/decorator/data.py
+++ b/meta/lib/oeqa/core/decorator/data.py
@@ -228,3 +228,15 @@ class skipIfNotArch(OETestDecorator):
         arch = self.case.td['HOST_ARCH']
         if arch not in self.archs:
              self.case.skipTest('Test skipped on %s' % arch)
+
+@registerDecorator
+class skipIfNotBuildArch(OETestDecorator):
+    """
+    Skip test if BUILD_ARCH is not present in the tuple specified.
+    """
+
+    attrs = ('archs',)
+    def setUpDecorator(self):
+        arch = self.case.td['BUILD_ARCH']
+        if arch not in self.archs:
+             self.case.skipTest('Test skipped on %s' % arch)
diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index a86a706bd9..b683d9b80a 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -357,7 +357,7 @@ class OETestResultJSONHelper(object):
             os.makedirs(write_dir, exist_ok=True)
         test_results = self._get_existing_testresults_if_available(write_dir)
         test_results[result_id] = {'configuration': configuration, 'result': test_result}
-        json_testresults = json.dumps(test_results, sort_keys=True, indent=4)
+        json_testresults = json.dumps(test_results, sort_keys=True, indent=1)
         self._write_file(write_dir, self.testresult_filename, json_testresults)
         if has_bb:
             bb.utils.unlockfile(lf)
diff --git a/meta/lib/oeqa/core/target/__init__.py b/meta/lib/oeqa/core/target/__init__.py
index 1382aa9b52..177f648fe3 100644
--- a/meta/lib/oeqa/core/target/__init__.py
+++ b/meta/lib/oeqa/core/target/__init__.py
@@ -10,6 +10,7 @@ class OETarget(object):
 
     def __init__(self, logger, *args, **kwargs):
         self.logger = logger
+        self.runner = None
 
     @abstractmethod
     def start(self):
diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py
index d93b3ac94a..769a6fec7e 100644
--- a/meta/lib/oeqa/core/target/qemu.py
+++ b/meta/lib/oeqa/core/target/qemu.py
@@ -15,7 +15,7 @@ from collections import defaultdict
 from .ssh import OESSHTarget
 from oeqa.utils.qemurunner import QemuRunner
 
-supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic']
+supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic', 'wic.zst', 'ext3.zst', 'ext4.zst']
 
 class OEQemuTarget(OESSHTarget):
     def __init__(self, logger, server_ip, timeout=300, user='root',
diff --git a/meta/lib/oeqa/core/target/serial.py b/meta/lib/oeqa/core/target/serial.py
new file mode 100644
index 0000000000..7c2cd8b248
--- /dev/null
+++ b/meta/lib/oeqa/core/target/serial.py
@@ -0,0 +1,315 @@
+#
+# SPDX-License-Identifier: MIT
+#
+
+import base64
+import logging
+import os
+from threading import Lock
+from . import OETarget
+
+class OESerialTarget(OETarget):
+
+    def __init__(self, logger, target_ip, server_ip, server_port=0,
+                 timeout=300, serialcontrol_cmd=None, serialcontrol_extra_args=None,
+                 serialcontrol_ps1=None, serialcontrol_connect_timeout=None, 
+                 machine=None, **kwargs):
+        if not logger:
+            logger = logging.getLogger('target')
+            logger.setLevel(logging.INFO)
+            filePath = os.path.join(os.getcwd(), 'remoteTarget.log')
+            fileHandler = logging.FileHandler(filePath, 'w', 'utf-8')
+            formatter = logging.Formatter(
+                        '%(asctime)s.%(msecs)03d %(levelname)s: %(message)s',
+                        '%H:%M:%S')
+            fileHandler.setFormatter(formatter)
+            logger.addHandler(fileHandler)
+
+        super(OESerialTarget, self).__init__(logger)
+
+        if serialcontrol_ps1:
+            self.target_ps1 = serialcontrol_ps1
+        elif machine:
+            # fallback to a default value which assumes root@machine
+            self.target_ps1 = f'root@{machine}:.*# '
+        else:
+            raise ValueError("Unable to determine shell command prompt (PS1) format.")
+
+        if not serialcontrol_cmd:
+            raise ValueError("Unable to determine serial control command.")
+
+        if serialcontrol_extra_args:
+            self.connection_script = f'{serialcontrol_cmd} {serialcontrol_extra_args}'
+        else:
+            self.connection_script = serialcontrol_cmd
+
+        if serialcontrol_connect_timeout:
+            self.connect_timeout = serialcontrol_connect_timeout
+        else:
+            self.connect_timeout = 10 # default to 10s connection timeout
+
+        self.default_command_timeout = timeout
+        self.ip = target_ip
+        self.server_ip = server_ip
+        self.server_port = server_port 
+        self.conn = None
+        self.mutex = Lock()
+
+    def start(self, **kwargs):
+        pass
+
+    def stop(self, **kwargs):
+        pass
+
+    def get_connection(self):
+        if self.conn is None:
+            self.conn = SerialConnection(self.connection_script,
+                                         self.target_ps1,
+                                         self.connect_timeout,
+                                         self.default_command_timeout)
+
+        return self.conn
+
+    def run(self, cmd, timeout=None):
+        """
+            Runs command on target over the provided serial connection.
+            The first call will open the connection, and subsequent
+            calls will re-use the same connection to send new commands.
+
+            command:    Command to run on target.
+            timeout:    <value>:    Kill command after <val> seconds.
+                        None:       Kill command default value seconds.
+                        0:          No timeout, runs until return.
+        """
+        # Lock needed to avoid multiple threads running commands concurrently
+        # A serial connection can only be used by one caller at a time
+        with self.mutex:
+            conn = self.get_connection()
+
+            self.logger.debug(f"[Running]$ {cmd}")
+            # Run the command, then echo $? to get the command's return code
+            try:
+                output = conn.run_command(cmd, timeout)
+                status = conn.run_command("echo $?")
+                self.logger.debug(f"   [stdout]: {output}")
+                self.logger.debug(f"   [ret code]: {status}\n\n")
+            except SerialTimeoutException as e:
+                self.logger.debug(e)
+                output = ""
+                status = 255
+
+            # Return to $HOME after each command to simulate a stateless SSH connection
+            conn.run_command('cd "$HOME"')
+
+        return (int(status), output)
+
+    def copyTo(self, localSrc, remoteDst):
+        """
+            Copies files by converting them to base 32, then transferring
+            the ASCII text to the target, and decoding it in place on the
+            target.
+
+            On a 115k baud serial connection, this method transfers at
+            roughly 30kbps.
+        """
+        with open(localSrc, 'rb') as file:
+            data = file.read()
+
+        b32 = base64.b32encode(data).decode('utf-8')
+
+        # To avoid shell line limits, send a chunk at a time
+        SPLIT_LEN = 512 
+        lines = [b32[i:i+SPLIT_LEN] for i in range(0, len(b32), SPLIT_LEN)]
+
+        with self.mutex:
+            conn = self.get_connection()
+
+            filename = os.path.basename(localSrc)
+            TEMP = f'/tmp/{filename}.b32'
+
+            # Create or empty out the temp file
+            conn.run_command(f'echo -n "" > {TEMP}')
+
+            for line in lines:
+                conn.run_command(f'echo -n {line} >> {TEMP}')
+
+            # Check to see whether the remoteDst is a directory
+            is_directory = conn.run_command(f'[[ -d {remoteDst} ]]; echo $?')
+            if int(is_directory) == 0:
+                # append the localSrc filename to the end of remoteDst
+                remoteDst = os.path.join(remoteDst, filename)
+
+            conn.run_command(f'base32 -d {TEMP} > {remoteDst}')
+            conn.run_command(f'rm {TEMP}')
+
+        return 0, 'Success'
+
+    def copyFrom(self, remoteSrc, localDst):
+        """
+            Copies files by converting them to base 32 on the target, then
+            transferring the ASCII text to the host. That text is then
+            decoded here and written out to the destination.
+
+            On a 115k baud serial connection, this method transfers at
+            roughly 30kbps.
+        """
+        with self.mutex:
+            b32 = self.get_connection().run_command(f'base32 {remoteSrc}')
+
+            data = base64.b32decode(b32.replace('\r\n', ''))
+
+            # If the local path is a directory, get the filename from
+            # the remoteSrc path and append it to localDst
+            if os.path.isdir(localDst):
+                filename = os.path.basename(remoteSrc)
+                localDst = os.path.join(localDst, filename)
+
+            with open(localDst, 'wb') as file:
+                file.write(data)
+
+        return 0, 'Success'
+
+    def copyDirTo(self, localSrc, remoteDst):
+        """
+            Copy recursively localSrc directory to remoteDst in target.
+        """
+
+        for root, dirs, files in os.walk(localSrc):
+            # Create directories in the target as needed
+            for d in dirs:
+                tmpDir = os.path.join(root, d).replace(localSrc, "")
+                newDir = os.path.join(remoteDst, tmpDir.lstrip("/"))
+                cmd = "mkdir -p %s" % newDir
+                self.run(cmd)
+
+            # Copy files into the target
+            for f in files:
+                tmpFile = os.path.join(root, f).replace(localSrc, "")
+                dstFile = os.path.join(remoteDst, tmpFile.lstrip("/"))
+                srcFile = os.path.join(root, f)
+                self.copyTo(srcFile, dstFile)
+
+    def deleteFiles(self, remotePath, files):
+        """
+            Deletes files in target's remotePath.
+        """
+
+        cmd = "rm"
+        if not isinstance(files, list):
+            files = [files]
+
+        for f in files:
+            cmd = "%s %s" % (cmd, os.path.join(remotePath, f))
+
+        self.run(cmd)
+
+    def deleteDir(self, remotePath):
+        """
+            Deletes target's remotePath directory.
+        """
+
+        cmd = "rmdir %s" % remotePath
+        self.run(cmd)
+
+    def deleteDirStructure(self, localPath, remotePath):
+        """
+        Delete recursively localPath structure directory in target's remotePath.
+
+        This function is useful to delete a package that is installed in the
+        device under test (DUT) and the host running the test has such package
+        extracted in tmp directory.
+
+        Example:
+            pwd: /home/user/tmp
+            tree:   .
+                    └── work
+                        ├── dir1
+                        │   └── file1
+                        └── dir2
+
+            localpath = "/home/user/tmp" and remotepath = "/home/user"
+
+            With the above variables this function will try to delete the
+            directory in the DUT in this order:
+                /home/user/work/dir1/file1
+                /home/user/work/dir1        (if dir is empty)
+                /home/user/work/dir2        (if dir is empty)
+                /home/user/work             (if dir is empty)
+        """
+
+        for root, dirs, files in os.walk(localPath, topdown=False):
+            # Delete files first
+            tmpDir = os.path.join(root).replace(localPath, "")
+            remoteDir = os.path.join(remotePath, tmpDir.lstrip("/"))
+            self.deleteFiles(remoteDir, files)
+
+            # Remove dirs if empty
+            for d in dirs:
+                tmpDir = os.path.join(root, d).replace(localPath, "")
+                remoteDir = os.path.join(remotePath, tmpDir.lstrip("/"))
+                self.deleteDir(remoteDir)
+
+class SerialTimeoutException(Exception):
+    def __init__(self, msg):
+        self.msg = msg
+    def __str__(self):
+        return self.msg
+
+class SerialConnection:
+
+    def __init__(self, script, target_prompt, connect_timeout, default_command_timeout):
+        import pexpect # limiting scope to avoid build dependency
+        self.prompt = target_prompt
+        self.connect_timeout = connect_timeout
+        self.default_command_timeout = default_command_timeout
+        self.conn = pexpect.spawn('/bin/bash', ['-c', script], encoding='utf8')
+        self._seek_to_clean_shell()
+        # Disable echo to avoid the need to parse the outgoing command
+        self.run_command('stty -echo')
+
+    def _seek_to_clean_shell(self):
+        """
+            Attempts to find a clean shell, meaning it is clear and
+            ready to accept a new command. This is necessary to ensure
+            the correct output is captured from each command.
+        """
+        import pexpect # limiting scope to avoid build dependency
+        # Look for a clean shell
+        # Wait a short amount of time for the connection to finish
+        pexpect_code = self.conn.expect([self.prompt, pexpect.TIMEOUT],
+                                        timeout=self.connect_timeout)
+
+        # if a timeout occurred, send an empty line and wait for a clean shell
+        if pexpect_code == 1:
+            # send a newline to clear and present the shell
+            self.conn.sendline("")
+            pexpect_code = self.conn.expect(self.prompt)
+
+    def run_command(self, cmd, timeout=None):
+        """
+            Runs command on target over the provided serial connection.
+            Returns any output on the shell while the command was run.
+
+            command:    Command to run on target.
+            timeout:    <value>:    Kill command after <val> seconds.
+                        None:       Kill command default value seconds.
+                        0:          No timeout, runs until return.
+        """
+        import pexpect # limiting scope to avoid build dependency
+        # Convert from the OETarget defaults to pexpect timeout values
+        if timeout is None:
+            timeout = self.default_command_timeout
+        elif timeout == 0:
+            timeout = None # passing None to pexpect is infinite timeout
+
+        self.conn.sendline(cmd)
+        pexpect_code = self.conn.expect([self.prompt, pexpect.TIMEOUT], timeout=timeout)
+
+        # check for timeout
+        if pexpect_code == 1:
+            self.conn.send('\003') # send Ctrl+C
+            self._seek_to_clean_shell()
+            raise SerialTimeoutException(f'Timeout executing: {cmd} after {timeout}s')
+
+        return self.conn.before.removesuffix('\r\n')
+
diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py
index 09cdd14c75..0ac3ae4388 100644
--- a/meta/lib/oeqa/core/target/ssh.py
+++ b/meta/lib/oeqa/core/target/ssh.py
@@ -55,24 +55,28 @@ class OESSHTarget(OETarget):
     def stop(self, **kwargs):
         pass
 
-    def _run(self, command, timeout=None, ignore_status=True):
+    def _run(self, command, timeout=None, ignore_status=True, raw=False, ignore_ssh_fails=False):
         """
             Runs command in target using SSHProcess.
         """
         self.logger.debug("[Running]$ %s" % " ".join(command))
 
         starttime = time.time()
-        status, output = SSHCall(command, self.logger, timeout)
+        status, output = SSHCall(command, self.logger, timeout, raw)
         self.logger.debug("[Command returned '%d' after %.2f seconds]"
                  "" % (status, time.time() - starttime))
 
-        if status and not ignore_status:
+        if status == 255 and not ignore_ssh_fails:
+            raise AssertionError("ssh exited with status '255' for command "
+                                 "'%s': this is likely an SSH failure\n%s"
+                                 % (command, output))
+        elif status and not ignore_status:
             raise AssertionError("Command '%s' returned non-zero exit "
                                  "status %d:\n%s" % (command, status, output))
 
         return (status, output)
 
-    def run(self, command, timeout=None, ignore_status=True):
+    def run(self, command, timeout=None, ignore_status=True, raw=False, ignore_ssh_fails=False):
         """
             Runs command in target.
 
@@ -91,8 +95,11 @@ class OESSHTarget(OETarget):
         else:
             processTimeout = self.timeout
 
-        status, output = self._run(sshCmd, processTimeout, ignore_status)
-        self.logger.debug('Command: %s\nStatus: %d Output:  %s\n' % (command, status, output))
+        status, output = self._run(sshCmd, processTimeout, ignore_status, raw, ignore_ssh_fails)
+        if len(output) > (64 * 1024):
+            self.logger.debug('Command: %s\nStatus: %d Output length:  %s\n' % (command, status, len(output)))
+        else:
+            self.logger.debug('Command: %s\nStatus: %d Output:  %s\n' % (command, status, output))
 
         return (status, output)
 
@@ -206,22 +213,23 @@ class OESSHTarget(OETarget):
                 remoteDir = os.path.join(remotePath, tmpDir.lstrip("/"))
                 self.deleteDir(remoteDir)
 
-def SSHCall(command, logger, timeout=None, **opts):
+def SSHCall(command, logger, timeout=None, raw=False, **opts):
 
     def run():
         nonlocal output
         nonlocal process
-        output_raw = b''
+        output_raw = bytearray()
         starttime = time.time()
+        progress = time.time()
         process = subprocess.Popen(command, **options)
         has_timeout = False
+        appendline = None
         if timeout:
             endtime = starttime + timeout
             eof = False
             os.set_blocking(process.stdout.fileno(), False)
             while not has_timeout and not eof:
                 try:
-                    logger.debug('Waiting for process output: time: %s, endtime: %s' % (time.time(), endtime))
                     if select.select([process.stdout], [], [], 5)[0] != []:
                         # wait a bit for more data, tries to avoid reading single characters
                         time.sleep(0.2)
@@ -229,9 +237,9 @@ def SSHCall(command, logger, timeout=None, **opts):
                         if not data:
                             eof = True
                         else:
-                            output_raw += data
+                            output_raw.extend(data)
                             # ignore errors to capture as much as possible
-                            logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore'))
+                            #logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore'))
                             endtime = time.time() + timeout
                 except InterruptedError:
                     logger.debug('InterruptedError')
@@ -244,6 +252,10 @@ def SSHCall(command, logger, timeout=None, **opts):
                     logger.debug('SSHCall has timeout! Time: %s, endtime: %s' % (time.time(), endtime))
                     has_timeout = True
 
+                if time.time() >= (progress + 60):
+                    logger.debug('Waiting for process output at time: %s with datasize: %s' % (time.time(), len(output_raw)))
+                    progress = time.time()
+
             process.stdout.close()
 
             # process hasn't returned yet
@@ -256,17 +268,29 @@ def SSHCall(command, logger, timeout=None, **opts):
                     logger.debug('OSError when killing process')
                     pass
                 endtime = time.time() - starttime
-                lastline = ("\nProcess killed - no output for %d seconds. Total"
+                appendline = ("\nProcess killed - no output for %d seconds. Total"
                             " running time: %d seconds." % (timeout, endtime))
-                logger.debug('Received data from SSH call:\n%s ' % lastline)
-                output += lastline
+                logger.debug('Received data from SSH call:\n%s ' % appendline)
                 process.wait()
 
+            if raw:
+                output = bytes(output_raw)
+                if appendline:
+                    output += bytes(appendline, "utf-8")
+            else:
+                output = output_raw.decode('utf-8', errors='ignore')
+                if appendline:
+                    output += appendline
         else:
-            output_raw = process.communicate()[0]
+            output = output_raw = process.communicate()[0]
+            if not raw:
+                output = output_raw.decode('utf-8', errors='ignore')
 
-        output = output_raw.decode('utf-8', errors='ignore')
-        logger.debug('Data from SSH call:\n%s' % output.rstrip())
+        if len(output) < (64 * 1024):
+            if output.rstrip():
+                logger.debug('Data from SSH call:\n%s' % output.rstrip())
+            else:
+                logger.debug('No output from SSH call')
 
         # timout or not, make sure process exits and is not hanging
         if process.returncode == None:
@@ -292,7 +316,7 @@ def SSHCall(command, logger, timeout=None, **opts):
 
     options = {
         "stdout": subprocess.PIPE,
-        "stderr": subprocess.STDOUT,
+        "stderr": subprocess.STDOUT if not raw else None,
         "stdin": None,
         "shell": False,
         "bufsize": -1,
@@ -320,4 +344,4 @@ def SSHCall(command, logger, timeout=None, **opts):
         logger.debug('Something went wrong, killing SSH process')
         raise
 
-    return (process.returncode, output.rstrip())
+    return (process.returncode, output if raw else output.rstrip())
diff --git a/meta/lib/oeqa/core/tests/common.py b/meta/lib/oeqa/core/tests/common.py
index 88cc758ad3..bcc4fde632 100644
--- a/meta/lib/oeqa/core/tests/common.py
+++ b/meta/lib/oeqa/core/tests/common.py
@@ -9,7 +9,6 @@ import os
 
 import unittest
 import logging
-import os
 
 logger = logging.getLogger("oeqa")
 logger.setLevel(logging.INFO)