qemurunner: Add support for qmp commands

This adds support for the Qemu Machine Protocol [0] extending
the current dump process for Host and Target. The commands are
added in the testimage.bbclass.

Currently, we setup qemu to stall until qmp gets connected and
sends the initialization and continue commands, this works
correctly. If the UNIX Socket does not exist, we wait an timeout
to ensure to socket file is created.

With this version, the monitor_dumper is created in OEQemuTarget
but then set in OESSHTarget as that's where we get the SSH failure
happens. Python's @property is used to create a setter/getter type
of setup in OESSHTarget to get overridden by OEQemuTarget.

By default the data is currently dumped to files for each command in
TMPDIR/log/runtime-hostdump/<date>_qmp/unknown_<seq>_qemu_monitor as
this is the naming convenstion in the dump.py code.

We use the qmp.py from qemu, which needs to get installed in the
recipe-sysroot-native of the target image.

[0] https://github.com/qemu/qemu/blob/master/docs/interop/qmp-spec.txt

(From OE-Core rev: 42af4cd2df72fc8ed9deb3fde4312909842fcf91)

Signed-off-by: Saul Wold <saul.wold@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 97 insertions, 5 deletions

diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py
index 09a44329e0..843e19fe8a 100644
--- a/meta/lib/oeqa/utils/dump.py
+++ b/meta/lib/oeqa/utils/dump.py
@@ -4,6 +4,7 @@
 
 import os
 import sys
+import json
 import errno
 import datetime
 import itertools
@@ -51,6 +52,8 @@ class BaseDumper(object):
             prefix = "host"
         elif isinstance(self, TargetDumper):
             prefix = "target"
+        elif isinstance(self, MonitorDumper):
+            prefix = "qmp"
         else:
             prefix = "unknown"
         for i in itertools.count():
@@ -58,9 +61,12 @@ class BaseDumper(object):
             fullname = os.path.join(self.dump_dir, filename)
             if not os.path.exists(fullname):
                 break
-        with open(fullname, 'w') as dump_file:
-            dump_file.write(output)
-
+        if isinstance(self, MonitorDumper):
+            with open(fullname, 'w') as json_file:
+                json.dump(output, json_file, indent=4)
+        else:
+            with open(fullname, 'w') as dump_file:
+                dump_file.write(output)
 
 class HostDumper(BaseDumper):
     """ Class to get dumps from the host running the tests """
@@ -96,3 +102,23 @@ class TargetDumper(BaseDumper):
             except:
                 print("Tried to dump info from target but "
                         "serial console failed")
+                print("Failed CMD: %s" % (cmd))
+
+class MonitorDumper(BaseDumper):
+    """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner """
+
+    def __init__(self, cmds, parent_dir, runner):
+        super(MonitorDumper, self).__init__(cmds, parent_dir)
+        self.runner = runner
+
+    def dump_monitor(self, dump_dir=""):
+        if self.runner is None:
+            return
+        if dump_dir:
+            self.dump_dir = dump_dir
+        for cmd in self.cmds:
+            try:
+                output = self.runner.run_monitor(cmd)
+                self._write_dump(cmd, output)
+            except:
+                print("Failed to dump QMP CMD: %s" % (cmd))
diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 278904ba0b..f6e1007288 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -20,8 +20,10 @@ import string
 import threading
 import codecs
 import logging
+import tempfile
 from oeqa.utils.dump import HostDumper
 from collections import defaultdict
+import importlib
 
 # Get Unicode non printable control chars
 control_range = list(range(0,32))+list(range(127,160))
@@ -172,6 +174,21 @@ class QemuRunner:
         return self.launch(launch_cmd, qemuparams=qemuparams, get_ip=get_ip, extra_bootparams=extra_bootparams, env=env)
 
     def launch(self, launch_cmd, get_ip = True, qemuparams = None, extra_bootparams = None, env = None):
+        # use logfile to determine the recipe-sysroot-native path and
+        # then add in the site-packages path components and add that
+        # to the python sys.path so qmp.py can be found.
+        python_path = os.path.dirname(os.path.dirname(self.logfile))
+        python_path += "/recipe-sysroot-native/usr/lib/python3.9/site-packages"
+        sys.path.append(python_path)
+        importlib.invalidate_caches()
+        try:
+            qmp = importlib.import_module("qmp")
+        except:
+            self.logger.error("qemurunner: qmp.py missing, please ensure it's installed")
+            return False
+        qmp_port = self.tmpdir + "/." + next(tempfile._get_candidate_names())
+        qmp_param = ' -S -qmp unix:%s,server,wait' % (qmp_port)
+
         try:
             if self.serial_ports >= 2:
                 self.threadsock, threadport = self.create_socket()
@@ -188,7 +205,8 @@ class QemuRunner:
         # and analyze descendents in order to determine it.
         if os.path.exists(self.qemu_pidfile):
             os.remove(self.qemu_pidfile)
-        self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1}"'.format(bootparams, self.qemu_pidfile)
+        self.qemuparams = 'bootparams="{0}" qemuparams="-pidfile {1} {2}"'.format(bootparams, self.qemu_pidfile, qmp_param)
+
         if qemuparams:
             self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"'
 
@@ -242,6 +260,7 @@ class QemuRunner:
         while not self.is_alive() and time.time() < endtime:
             if self.runqemu.poll():
                 if self.runqemu_exited:
+                    self.logger.warning("runqemu during is_alive() test")
                     return False
                 if self.runqemu.returncode:
                     # No point waiting any longer
@@ -253,9 +272,51 @@ class QemuRunner:
             time.sleep(0.5)
 
         if self.runqemu_exited:
+            self.logger.warning("runqemu after timeout")
+            return False
+
+        if self.runqemu.returncode:
+            self.logger.warning('runqemu exited with code %d' % self.runqemu.returncode)
             return False
 
         if not self.is_alive():
+            self.logger.warning('is_alive() failed later')
+            return False
+
+        # Create the client socket for the QEMU Monitor Control Socket
+        # This will allow us to read status from Qemu if the the process
+        # is still alive
+        self.logger.debug("QMP Initializing to %s" % (qmp_port))
+        try:
+            self.qmp = qmp.QEMUMonitorProtocol(qmp_port)
+        except OSError as msg:
+            self.logger.warning("Failed to initialize qemu monitor socket: %s File: %s" % (msg, msg.filename))
+            return False
+
+        self.logger.debug("QMP Connecting to %s" % (qmp_port))
+        if not os.path.exists(qmp_port) and self.is_alive():
+            self.logger.debug("QMP Port does not exist waiting for it to be created")
+            endtime = time.time() + self.runqemutime
+            while not os.path.exists(qmp_port) and self.is_alive() and time.time() < endtime:
+               self.logger.warning("QMP port does not exist yet!")
+               time.sleep(0.5)
+            if not os.path.exists(qmp_port) and self.is_alive():
+                self.logger.warning("QMP Port still does not exist but QEMU is alive")
+                return False
+
+        try:
+            self.qmp.connect()
+        except OSError as msg:
+            self.logger.warning("Failed to connect qemu monitor socket: %s File: %s" % (msg, msg.filename))
+            return False
+        except qmp.QMPConnectError as msg:
+            self.logger.warning("Failed to communicate with qemu monitor: %s" % (msg))
+            return False
+
+        # Release the qemu porcess to continue running
+        self.run_monitor('cont')
+
+        if not self.is_alive():
             self.logger.error("Qemu pid didn't appear in %s seconds (%s)" %
                               (self.runqemutime, time.strftime("%D %H:%M:%S")))
 
@@ -380,7 +441,6 @@ class QemuRunner:
                         sock.close()
                         stopread = True
 
-
         if not reachedlogin:
             if time.time() >= endtime:
                 self.logger.warning("Target didn't reach login banner in %d seconds (%s)" %
@@ -441,6 +501,9 @@ class QemuRunner:
             self.runqemu.stdout.close()
             self.runqemu_exited = True
 
+        if hasattr(self, 'qmp') and self.qmp:
+            self.qmp.close()
+            self.qmp = None
         if hasattr(self, 'server_socket') and self.server_socket:
             self.server_socket.close()
             self.server_socket = None
@@ -499,6 +562,9 @@ class QemuRunner:
                         return True
         return False
 
+    def run_monitor(self, command, timeout=60):
+        return self.qmp.cmd(command)
+
     def run_serial(self, command, raw=False, timeout=60):
         # We assume target system have echo to get command status
         if not raw: