diff options
author | Mikko Rapeli <mikko.rapeli@linaro.org> | 2023-02-09 10:09:32 +0200 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-02-24 16:41:46 +0000 |
commit | 519f3a7196a5657466a644bdd93f7b3a7cd3d7cd (patch) | |
tree | a57b66e9800fe1b97d6623676c2a5371cba0891d | |
parent | 758c32745e2475e343a130a26ccf0f32003c661c (diff) | |
download | poky-519f3a7196a5657466a644bdd93f7b3a7cd3d7cd.tar.gz |
oeqa dump.py: add error counter and stop after 5 failures
If test target qemu machine hangs completely, dump_target() calls
over serial console are taking a long time to time out, possibly
for every failing ssh command execution and a lot of test cases,
and same with dump_monitor().
Instead of trying for ever, count errors and after 5 stop trying
to dump_target() and dump_monitor() completely.
These help to end testing earlier when a test target is completely
deadlocked and all ssh, serial and QMP communication with it are
failing.
(From OE-Core rev: 91bc1e03bc990c527d8aadbdcd7bf97217db124e)
Signed-off-by: Mikko Rapeli <mikko.rapeli@linaro.org>
Signed-off-by: Luca Ceresoli <luca.ceresoli@bootlin.com>
(cherry picked from commit d9ad0a055abba983c6cee1dca4d2f0a8a3c48782)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | meta/lib/oeqa/utils/dump.py | 23 |
1 files changed, 21 insertions, 2 deletions
diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py index 95a79a571c..6fd5832051 100644 --- a/meta/lib/oeqa/utils/dump.py +++ b/meta/lib/oeqa/utils/dump.py | |||
@@ -91,37 +91,55 @@ class HostDumper(BaseDumper): | |||
91 | self._write_dump(cmd.split()[0], result.output) | 91 | self._write_dump(cmd.split()[0], result.output) |
92 | 92 | ||
93 | class TargetDumper(BaseDumper): | 93 | class TargetDumper(BaseDumper): |
94 | """ Class to get dumps from target, it only works with QemuRunner """ | 94 | """ Class to get dumps from target, it only works with QemuRunner. |
95 | Will give up permanently after 5 errors from running commands over | ||
96 | serial console. This helps to end testing when target is really dead, hanging | ||
97 | or unresponsive. | ||
98 | """ | ||
95 | 99 | ||
96 | def __init__(self, cmds, parent_dir, runner): | 100 | def __init__(self, cmds, parent_dir, runner): |
97 | super(TargetDumper, self).__init__(cmds, parent_dir) | 101 | super(TargetDumper, self).__init__(cmds, parent_dir) |
98 | self.runner = runner | 102 | self.runner = runner |
103 | self.errors = 0 | ||
99 | 104 | ||
100 | def dump_target(self, dump_dir=""): | 105 | def dump_target(self, dump_dir=""): |
106 | if self.errors >= 5: | ||
107 | print("Too many errors when dumping data from target, assuming it is dead! Will not dump data anymore!") | ||
108 | return | ||
101 | if dump_dir: | 109 | if dump_dir: |
102 | self.dump_dir = dump_dir | 110 | self.dump_dir = dump_dir |
103 | for cmd in self.cmds: | 111 | for cmd in self.cmds: |
104 | # We can continue with the testing if serial commands fail | 112 | # We can continue with the testing if serial commands fail |
105 | try: | 113 | try: |
106 | (status, output) = self.runner.run_serial(cmd) | 114 | (status, output) = self.runner.run_serial(cmd) |
115 | if status == 0: | ||
116 | self.errors = self.errors + 1 | ||
107 | self._write_dump(cmd.split()[0], output) | 117 | self._write_dump(cmd.split()[0], output) |
108 | except: | 118 | except: |
119 | self.errors = self.errors + 1 | ||
109 | print("Tried to dump info from target but " | 120 | print("Tried to dump info from target but " |
110 | "serial console failed") | 121 | "serial console failed") |
111 | print("Failed CMD: %s" % (cmd)) | 122 | print("Failed CMD: %s" % (cmd)) |
112 | 123 | ||
113 | class MonitorDumper(BaseDumper): | 124 | class MonitorDumper(BaseDumper): |
114 | """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner """ | 125 | """ Class to get dumps via the Qemu Monitor, it only works with QemuRunner |
126 | Will stop completely if there are more than 5 errors when dumping monitor data. | ||
127 | This helps to end testing when target is really dead, hanging or unresponsive. | ||
128 | """ | ||
115 | 129 | ||
116 | def __init__(self, cmds, parent_dir, runner): | 130 | def __init__(self, cmds, parent_dir, runner): |
117 | super(MonitorDumper, self).__init__(cmds, parent_dir) | 131 | super(MonitorDumper, self).__init__(cmds, parent_dir) |
118 | self.runner = runner | 132 | self.runner = runner |
133 | self.errors = 0 | ||
119 | 134 | ||
120 | def dump_monitor(self, dump_dir=""): | 135 | def dump_monitor(self, dump_dir=""): |
121 | if self.runner is None: | 136 | if self.runner is None: |
122 | return | 137 | return |
123 | if dump_dir: | 138 | if dump_dir: |
124 | self.dump_dir = dump_dir | 139 | self.dump_dir = dump_dir |
140 | if self.errors >= 5: | ||
141 | print("Too many errors when dumping data from qemu monitor, assuming it is dead! Will not dump data anymore!") | ||
142 | return | ||
125 | for cmd in self.cmds: | 143 | for cmd in self.cmds: |
126 | cmd_name = cmd.split()[0] | 144 | cmd_name = cmd.split()[0] |
127 | try: | 145 | try: |
@@ -135,4 +153,5 @@ class MonitorDumper(BaseDumper): | |||
135 | output = self.runner.run_monitor(cmd_name) | 153 | output = self.runner.run_monitor(cmd_name) |
136 | self._write_dump(cmd_name, output) | 154 | self._write_dump(cmd_name, output) |
137 | except Exception as e: | 155 | except Exception as e: |
156 | self.errors = self.errors + 1 | ||
138 | print("Failed to dump QMP CMD: %s with\nException: %s" % (cmd_name, e)) | 157 | print("Failed to dump QMP CMD: %s with\nException: %s" % (cmd_name, e)) |