summaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa
diff options
context:
space:
mode:
authorAlejandro Hernandez Samaniego <alejandro@enedino.org>2020-02-08 02:32:02 -0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-02-13 12:19:14 +0000
commita3416a5933892df940514842a34120658096ca07 (patch)
tree270a9a6ee9903866341a9cd3209c5ede69796fcb /meta/lib/oeqa
parente5978093075fe84ac949b2f4022f92e3f83ec2c0 (diff)
downloadpoky-a3416a5933892df940514842a34120658096ca07.tar.gz
testimage: Extend runtime testing infrastructure to allow unconventional booting processes to be tested
The current runtime infrastructure contains hardcoded values which Ill refer to as patterns, these patterns are either searched through or sent via the serial terminal to communicate between HOST and TARGET. These patterns are required since they allow us to check when a device has finished booting, to log in, and to check whether a command sent from our tests has returned, this way we are able to check both the status of the commands that were sent along with its output. The testing process goes somewhat as follows: 1. Launch QEMU and start booting. 2. Check when the device has booted by looking for the pattern login:. 3. Log in as the root user (default for our images). 4. Check that we were able to log in succesfully. 5. Start running the runtime test cases defined by TEST_SUITES. 6. One of such test cases could send a command to the QEMU target. 7. Check whether that command returned. 8. Check its output and status, return whether the test case passed or failed. This patch allows this set of patterns to be defined instead of being hardcoded, but it also automatically sets the defaults that we have been using in the past if they have not been manually defined, for this reason, the patch is less invasive and should not affect in any way how tests are currently being run. Cases that can be enabled with this patch: - A customized image that does not use the root user (or maybe we want to check what happens if we dont use the root user). - An image where the PS1 env variable has been modified, and the prompt pattern wouldnt match the default. - Baremetal applications, which do not follow the conventional way of booting Linux and would probably not show a prompt for a user to log in, same applies for testing bootloaders. - poky-tiny: Using DISTRO=poky-tiny and an image such as the core-image-tiny from meta-intel, which boots directly to RAM, and does not show a log in prompt since it does not contain a conventional init process. The code itself contains comments that should be self explanatory but here is an example on how these patterns can be defined in a hypothetical case where we want to run test cases as the webserver user instead: TESTIMAGE_BOOT_PATTERNS = "send_login_user search_login_succeeded" TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n" TESTIMAGE_BOOT_PATTERNS[search_login_succeeded] = "webserver@[a-zA-Z0-9\-]+:~#" The variable TESTIMAGE_BOOT_PATTERNS defines which patterns to override when used to communicate with the target when booting, anyone familiar with the PACKAGECONFIG syntax should have no trouble setting these. Other patterns would still be set up as default, e.g. search_reached_prompt would still be login: The accepted flags for TESTIMAGE_BOOT_PATTERNS are the following: search_reached_prompt, send_login_user, search_login_succeeded, search_cmd_finished. They are prefixed with either search/send, to differentiate if the pattern is meant to be sent or searched to/from the target terminal. A working example of this code that falls under the baremetal case mentioned above along with a test case is present on the meta-freertos layer, which tests an RTOS image built with OpenEmbedded and automatically runs a test case on it after booting such image: As usual, INHERIT += "testimage" needs to be present on local.conf $ bitbake freertos-demo -c testimage RESULTS: RESULTS - freertos_echo.FreeRTOSTest.test_freertos_echo: PASSED (2.00s) SUMMARY: freertos-demo () - Ran 1 test in 2.006s freertos-demo - OK - All required tests passed (successes=1, skipped=0, failures=0, errors=0) (From OE-Core rev: 3ab2cbfeff371e8791b031a2852eeef80101a831) Signed-off-by: Alejandro Hernandez Samaniego <aehs29@gmail.com> Signed-off-by: Alejandro Hernandez Samaniego <alejandro@enedino.org> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oeqa')
-rw-r--r--meta/lib/oeqa/core/target/qemu.py7
-rw-r--r--meta/lib/oeqa/utils/qemurunner.py31
2 files changed, 31 insertions, 7 deletions
diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py
index 008a9f03ce..059106e915 100644
--- a/meta/lib/oeqa/core/target/qemu.py
+++ b/meta/lib/oeqa/core/target/qemu.py
@@ -8,6 +8,7 @@ import os
8import sys 8import sys
9import signal 9import signal
10import time 10import time
11from collections import defaultdict
11 12
12from .ssh import OESSHTarget 13from .ssh import OESSHTarget
13from oeqa.utils.qemurunner import QemuRunner 14from oeqa.utils.qemurunner import QemuRunner
@@ -18,7 +19,8 @@ class OEQemuTarget(OESSHTarget):
18 def __init__(self, logger, server_ip, timeout=300, user='root', 19 def __init__(self, logger, server_ip, timeout=300, user='root',
19 port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False, 20 port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False,
20 dump_dir='', dump_host_cmds='', display='', bootlog='', 21 dump_dir='', dump_host_cmds='', display='', bootlog='',
21 tmpdir='', dir_image='', boottime=60, serial_ports=2, **kwargs): 22 tmpdir='', dir_image='', boottime=60, serial_ports=2,
23 boot_patterns = defaultdict(str), **kwargs):
22 24
23 super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout, 25 super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout,
24 user, port) 26 user, port)
@@ -30,13 +32,14 @@ class OEQemuTarget(OESSHTarget):
30 self.kernel = kernel 32 self.kernel = kernel
31 self.kvm = kvm 33 self.kvm = kvm
32 self.use_slirp = slirp 34 self.use_slirp = slirp
35 self.boot_patterns = boot_patterns
33 36
34 self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir, 37 self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir,
35 deploy_dir_image=dir_image, display=display, 38 deploy_dir_image=dir_image, display=display,
36 logfile=bootlog, boottime=boottime, 39 logfile=bootlog, boottime=boottime,
37 use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, 40 use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir,
38 dump_host_cmds=dump_host_cmds, logger=logger, 41 dump_host_cmds=dump_host_cmds, logger=logger,
39 serial_ports=serial_ports) 42 serial_ports=serial_ports, boot_patterns = boot_patterns)
40 43
41 def start(self, params=None, extra_bootparams=None, runqemuparams=''): 44 def start(self, params=None, extra_bootparams=None, runqemuparams=''):
42 if self.use_slirp and not self.server_ip: 45 if self.use_slirp and not self.server_ip:
diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 4704422211..ed74ea8fad 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -21,6 +21,7 @@ import threading
21import codecs 21import codecs
22import logging 22import logging
23from oeqa.utils.dump import HostDumper 23from oeqa.utils.dump import HostDumper
24from collections import defaultdict
24 25
25# Get Unicode non printable control chars 26# Get Unicode non printable control chars
26control_range = list(range(0,32))+list(range(127,160)) 27control_range = list(range(0,32))+list(range(127,160))
@@ -31,7 +32,7 @@ re_control_char = re.compile('[%s]' % re.escape("".join(control_chars)))
31class QemuRunner: 32class QemuRunner:
32 33
33 def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds, 34 def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime, dump_dir, dump_host_cmds,
34 use_kvm, logger, use_slirp=False, serial_ports=2): 35 use_kvm, logger, use_slirp=False, serial_ports=2, boot_patterns = defaultdict(str)):
35 36
36 # Popen object for runqemu 37 # Popen object for runqemu
37 self.runqemu = None 38 self.runqemu = None
@@ -57,6 +58,7 @@ class QemuRunner:
57 self.use_slirp = use_slirp 58 self.use_slirp = use_slirp
58 self.serial_ports = serial_ports 59 self.serial_ports = serial_ports
59 self.msg = '' 60 self.msg = ''
61 self.boot_patterns = boot_patterns
60 62
61 self.runqemutime = 120 63 self.runqemutime = 120
62 self.qemu_pidfile = 'pidfile_'+str(os.getpid()) 64 self.qemu_pidfile = 'pidfile_'+str(os.getpid())
@@ -65,6 +67,25 @@ class QemuRunner:
65 67
66 self.logger = logger 68 self.logger = logger
67 69
70 # Enable testing other OS's
71 # Set commands for target communication, and default to Linux ALWAYS
72 # Other OS's or baremetal applications need to provide their
73 # own implementation passing it through QemuRunner's constructor
74 # or by passing them through TESTIMAGE_BOOT_PATTERNS[flag]
75 # provided variables, where <flag> is one of the mentioned below.
76 accepted_patterns = ['search_reached_prompt', 'send_login_user', 'search_login_succeeded', 'search_cmd_finished']
77 default_boot_patterns = defaultdict(str)
78 # Default to the usual paterns used to communicate with the target
79 default_boot_patterns['search_reached_prompt'] = b' login:'
80 default_boot_patterns['send_login_user'] = 'root\n'
81 default_boot_patterns['search_login_succeeded'] = r"root@[a-zA-Z0-9\-]+:~#"
82 default_boot_patterns['search_cmd_finished'] = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#"
83
84 # Only override patterns that were set e.g. login user TESTIMAGE_BOOT_PATTERNS[send_login_user] = "webserver\n"
85 for pattern in accepted_patterns:
86 if not self.boot_patterns[pattern]:
87 self.boot_patterns[pattern] = default_boot_patterns[pattern]
88
68 def create_socket(self): 89 def create_socket(self):
69 try: 90 try:
70 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 91 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
@@ -321,7 +342,7 @@ class QemuRunner:
321 self.log(data) 342 self.log(data)
322 343
323 data = b'' 344 data = b''
324 if b' login:' in bootlog: 345 if self.boot_patterns['search_reached_prompt'] in bootlog:
325 self.server_socket = qemusock 346 self.server_socket = qemusock
326 stopread = True 347 stopread = True
327 reachedlogin = True 348 reachedlogin = True
@@ -353,8 +374,8 @@ class QemuRunner:
353 374
354 # If we are not able to login the tests can continue 375 # If we are not able to login the tests can continue
355 try: 376 try:
356 (status, output) = self.run_serial("root\n", raw=True) 377 (status, output) = self.run_serial(self.boot_patterns['send_login_user'], raw=True)
357 if re.search(r"root@[a-zA-Z0-9\-]+:~#", output): 378 if re.search(self.boot_patterns['search_login_succeeded'], output):
358 self.logged = True 379 self.logged = True
359 self.logger.debug("Logged as root in serial console") 380 self.logger.debug("Logged as root in serial console")
360 if netconf: 381 if netconf:
@@ -478,7 +499,7 @@ class QemuRunner:
478 if answer: 499 if answer:
479 data += answer.decode('utf-8') 500 data += answer.decode('utf-8')
480 # Search the prompt to stop 501 # Search the prompt to stop
481 if re.search(r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+:~#", data): 502 if re.search(self.boot_patterns['search_cmd_finished'], data):
482 break 503 break
483 else: 504 else:
484 raise Exception("No data on serial console socket") 505 raise Exception("No data on serial console socket")