diff options
Diffstat (limited to 'meta/lib/oeqa/core/target')
-rw-r--r-- | meta/lib/oeqa/core/target/__init__.py | 1 | ||||
-rw-r--r-- | meta/lib/oeqa/core/target/qemu.py | 40 | ||||
-rw-r--r-- | meta/lib/oeqa/core/target/serial.py | 315 | ||||
-rw-r--r-- | meta/lib/oeqa/core/target/ssh.py | 115 |
4 files changed, 437 insertions, 34 deletions
diff --git a/meta/lib/oeqa/core/target/__init__.py b/meta/lib/oeqa/core/target/__init__.py index 1382aa9b52..177f648fe3 100644 --- a/meta/lib/oeqa/core/target/__init__.py +++ b/meta/lib/oeqa/core/target/__init__.py | |||
@@ -10,6 +10,7 @@ class OETarget(object): | |||
10 | 10 | ||
11 | def __init__(self, logger, *args, **kwargs): | 11 | def __init__(self, logger, *args, **kwargs): |
12 | self.logger = logger | 12 | self.logger = logger |
13 | self.runner = None | ||
13 | 14 | ||
14 | @abstractmethod | 15 | @abstractmethod |
15 | def start(self): | 16 | def start(self): |
diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py index 0f29414df5..d93b3ac94a 100644 --- a/meta/lib/oeqa/core/target/qemu.py +++ b/meta/lib/oeqa/core/target/qemu.py | |||
@@ -8,20 +8,21 @@ import os | |||
8 | import sys | 8 | import sys |
9 | import signal | 9 | import signal |
10 | import time | 10 | import time |
11 | import glob | ||
12 | import subprocess | ||
11 | from collections import defaultdict | 13 | from collections import defaultdict |
12 | 14 | ||
13 | from .ssh import OESSHTarget | 15 | from .ssh import OESSHTarget |
14 | from oeqa.utils.qemurunner import QemuRunner | 16 | from oeqa.utils.qemurunner import QemuRunner |
15 | from oeqa.utils.dump import TargetDumper | ||
16 | 17 | ||
17 | supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] | 18 | supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] |
18 | 19 | ||
19 | class OEQemuTarget(OESSHTarget): | 20 | class OEQemuTarget(OESSHTarget): |
20 | def __init__(self, logger, server_ip, timeout=300, user='root', | 21 | def __init__(self, logger, server_ip, timeout=300, user='root', |
21 | port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False, | 22 | port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False, |
22 | dump_dir='', dump_host_cmds='', display='', bootlog='', | 23 | dump_dir='', display='', bootlog='', |
23 | tmpdir='', dir_image='', boottime=60, serial_ports=2, | 24 | tmpdir='', dir_image='', boottime=60, serial_ports=2, |
24 | boot_patterns = defaultdict(str), ovmf=False, **kwargs): | 25 | boot_patterns = defaultdict(str), ovmf=False, tmpfsdir=None, **kwargs): |
25 | 26 | ||
26 | super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout, | 27 | super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout, |
27 | user, port) | 28 | user, port) |
@@ -35,17 +36,15 @@ class OEQemuTarget(OESSHTarget): | |||
35 | self.ovmf = ovmf | 36 | self.ovmf = ovmf |
36 | self.use_slirp = slirp | 37 | self.use_slirp = slirp |
37 | self.boot_patterns = boot_patterns | 38 | self.boot_patterns = boot_patterns |
39 | self.dump_dir = dump_dir | ||
40 | self.bootlog = bootlog | ||
38 | 41 | ||
39 | self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir, | 42 | self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir, |
40 | deploy_dir_image=dir_image, display=display, | 43 | deploy_dir_image=dir_image, display=display, |
41 | logfile=bootlog, boottime=boottime, | 44 | logfile=bootlog, boottime=boottime, |
42 | use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, | 45 | use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, logger=logger, |
43 | dump_host_cmds=dump_host_cmds, logger=logger, | ||
44 | serial_ports=serial_ports, boot_patterns = boot_patterns, | 46 | serial_ports=serial_ports, boot_patterns = boot_patterns, |
45 | use_ovmf=ovmf) | 47 | use_ovmf=ovmf, tmpfsdir=tmpfsdir) |
46 | dump_target_cmds = kwargs.get("testimage_dump_target") | ||
47 | self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner) | ||
48 | self.target_dumper.create_dir("qemu") | ||
49 | 48 | ||
50 | def start(self, params=None, extra_bootparams=None, runqemuparams=''): | 49 | def start(self, params=None, extra_bootparams=None, runqemuparams=''): |
51 | if self.use_slirp and not self.server_ip: | 50 | if self.use_slirp and not self.server_ip: |
@@ -68,7 +67,28 @@ class OEQemuTarget(OESSHTarget): | |||
68 | self.server_ip = self.runner.server_ip | 67 | self.server_ip = self.runner.server_ip |
69 | else: | 68 | else: |
70 | self.stop() | 69 | self.stop() |
71 | raise RuntimeError("FAILED to start qemu - check the task log and the boot log") | 70 | # Display the first 20 lines of top and |
71 | # last 20 lines of the bootlog when the | ||
72 | # target is not being booted up. | ||
73 | topfile = glob.glob(self.dump_dir + "/*_qemu/host_*_top") | ||
74 | msg = "\n\n===== start: snippet =====\n\n" | ||
75 | for f in topfile: | ||
76 | msg += "file: %s\n\n" % f | ||
77 | with open(f) as tf: | ||
78 | for x in range(20): | ||
79 | msg += next(tf) | ||
80 | msg += "\n\n===== end: snippet =====\n\n" | ||
81 | blcmd = ["tail", "-20", self.bootlog] | ||
82 | msg += "===== start: snippet =====\n\n" | ||
83 | try: | ||
84 | out = subprocess.check_output(blcmd, stderr=subprocess.STDOUT, timeout=1).decode('utf-8') | ||
85 | msg += "file: %s\n\n" % self.bootlog | ||
86 | msg += out | ||
87 | except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as err: | ||
88 | msg += "Error running command: %s\n%s\n" % (blcmd, err) | ||
89 | msg += "\n\n===== end: snippet =====\n" | ||
90 | |||
91 | raise RuntimeError("FAILED to start qemu - check the task log and the boot log %s" % (msg)) | ||
72 | 92 | ||
73 | def stop(self): | 93 | def stop(self): |
74 | self.runner.stop() | 94 | self.runner.stop() |
diff --git a/meta/lib/oeqa/core/target/serial.py b/meta/lib/oeqa/core/target/serial.py new file mode 100644 index 0000000000..7c2cd8b248 --- /dev/null +++ b/meta/lib/oeqa/core/target/serial.py | |||
@@ -0,0 +1,315 @@ | |||
1 | # | ||
2 | # SPDX-License-Identifier: MIT | ||
3 | # | ||
4 | |||
5 | import base64 | ||
6 | import logging | ||
7 | import os | ||
8 | from threading import Lock | ||
9 | from . import OETarget | ||
10 | |||
11 | class OESerialTarget(OETarget): | ||
12 | |||
13 | def __init__(self, logger, target_ip, server_ip, server_port=0, | ||
14 | timeout=300, serialcontrol_cmd=None, serialcontrol_extra_args=None, | ||
15 | serialcontrol_ps1=None, serialcontrol_connect_timeout=None, | ||
16 | machine=None, **kwargs): | ||
17 | if not logger: | ||
18 | logger = logging.getLogger('target') | ||
19 | logger.setLevel(logging.INFO) | ||
20 | filePath = os.path.join(os.getcwd(), 'remoteTarget.log') | ||
21 | fileHandler = logging.FileHandler(filePath, 'w', 'utf-8') | ||
22 | formatter = logging.Formatter( | ||
23 | '%(asctime)s.%(msecs)03d %(levelname)s: %(message)s', | ||
24 | '%H:%M:%S') | ||
25 | fileHandler.setFormatter(formatter) | ||
26 | logger.addHandler(fileHandler) | ||
27 | |||
28 | super(OESerialTarget, self).__init__(logger) | ||
29 | |||
30 | if serialcontrol_ps1: | ||
31 | self.target_ps1 = serialcontrol_ps1 | ||
32 | elif machine: | ||
33 | # fallback to a default value which assumes root@machine | ||
34 | self.target_ps1 = f'root@{machine}:.*# ' | ||
35 | else: | ||
36 | raise ValueError("Unable to determine shell command prompt (PS1) format.") | ||
37 | |||
38 | if not serialcontrol_cmd: | ||
39 | raise ValueError("Unable to determine serial control command.") | ||
40 | |||
41 | if serialcontrol_extra_args: | ||
42 | self.connection_script = f'{serialcontrol_cmd} {serialcontrol_extra_args}' | ||
43 | else: | ||
44 | self.connection_script = serialcontrol_cmd | ||
45 | |||
46 | if serialcontrol_connect_timeout: | ||
47 | self.connect_timeout = serialcontrol_connect_timeout | ||
48 | else: | ||
49 | self.connect_timeout = 10 # default to 10s connection timeout | ||
50 | |||
51 | self.default_command_timeout = timeout | ||
52 | self.ip = target_ip | ||
53 | self.server_ip = server_ip | ||
54 | self.server_port = server_port | ||
55 | self.conn = None | ||
56 | self.mutex = Lock() | ||
57 | |||
58 | def start(self, **kwargs): | ||
59 | pass | ||
60 | |||
61 | def stop(self, **kwargs): | ||
62 | pass | ||
63 | |||
64 | def get_connection(self): | ||
65 | if self.conn is None: | ||
66 | self.conn = SerialConnection(self.connection_script, | ||
67 | self.target_ps1, | ||
68 | self.connect_timeout, | ||
69 | self.default_command_timeout) | ||
70 | |||
71 | return self.conn | ||
72 | |||
73 | def run(self, cmd, timeout=None): | ||
74 | """ | ||
75 | Runs command on target over the provided serial connection. | ||
76 | The first call will open the connection, and subsequent | ||
77 | calls will re-use the same connection to send new commands. | ||
78 | |||
79 | command: Command to run on target. | ||
80 | timeout: <value>: Kill command after <val> seconds. | ||
81 | None: Kill command default value seconds. | ||
82 | 0: No timeout, runs until return. | ||
83 | """ | ||
84 | # Lock needed to avoid multiple threads running commands concurrently | ||
85 | # A serial connection can only be used by one caller at a time | ||
86 | with self.mutex: | ||
87 | conn = self.get_connection() | ||
88 | |||
89 | self.logger.debug(f"[Running]$ {cmd}") | ||
90 | # Run the command, then echo $? to get the command's return code | ||
91 | try: | ||
92 | output = conn.run_command(cmd, timeout) | ||
93 | status = conn.run_command("echo $?") | ||
94 | self.logger.debug(f" [stdout]: {output}") | ||
95 | self.logger.debug(f" [ret code]: {status}\n\n") | ||
96 | except SerialTimeoutException as e: | ||
97 | self.logger.debug(e) | ||
98 | output = "" | ||
99 | status = 255 | ||
100 | |||
101 | # Return to $HOME after each command to simulate a stateless SSH connection | ||
102 | conn.run_command('cd "$HOME"') | ||
103 | |||
104 | return (int(status), output) | ||
105 | |||
106 | def copyTo(self, localSrc, remoteDst): | ||
107 | """ | ||
108 | Copies files by converting them to base 32, then transferring | ||
109 | the ASCII text to the target, and decoding it in place on the | ||
110 | target. | ||
111 | |||
112 | On a 115k baud serial connection, this method transfers at | ||
113 | roughly 30kbps. | ||
114 | """ | ||
115 | with open(localSrc, 'rb') as file: | ||
116 | data = file.read() | ||
117 | |||
118 | b32 = base64.b32encode(data).decode('utf-8') | ||
119 | |||
120 | # To avoid shell line limits, send a chunk at a time | ||
121 | SPLIT_LEN = 512 | ||
122 | lines = [b32[i:i+SPLIT_LEN] for i in range(0, len(b32), SPLIT_LEN)] | ||
123 | |||
124 | with self.mutex: | ||
125 | conn = self.get_connection() | ||
126 | |||
127 | filename = os.path.basename(localSrc) | ||
128 | TEMP = f'/tmp/{filename}.b32' | ||
129 | |||
130 | # Create or empty out the temp file | ||
131 | conn.run_command(f'echo -n "" > {TEMP}') | ||
132 | |||
133 | for line in lines: | ||
134 | conn.run_command(f'echo -n {line} >> {TEMP}') | ||
135 | |||
136 | # Check to see whether the remoteDst is a directory | ||
137 | is_directory = conn.run_command(f'[[ -d {remoteDst} ]]; echo $?') | ||
138 | if int(is_directory) == 0: | ||
139 | # append the localSrc filename to the end of remoteDst | ||
140 | remoteDst = os.path.join(remoteDst, filename) | ||
141 | |||
142 | conn.run_command(f'base32 -d {TEMP} > {remoteDst}') | ||
143 | conn.run_command(f'rm {TEMP}') | ||
144 | |||
145 | return 0, 'Success' | ||
146 | |||
147 | def copyFrom(self, remoteSrc, localDst): | ||
148 | """ | ||
149 | Copies files by converting them to base 32 on the target, then | ||
150 | transferring the ASCII text to the host. That text is then | ||
151 | decoded here and written out to the destination. | ||
152 | |||
153 | On a 115k baud serial connection, this method transfers at | ||
154 | roughly 30kbps. | ||
155 | """ | ||
156 | with self.mutex: | ||
157 | b32 = self.get_connection().run_command(f'base32 {remoteSrc}') | ||
158 | |||
159 | data = base64.b32decode(b32.replace('\r\n', '')) | ||
160 | |||
161 | # If the local path is a directory, get the filename from | ||
162 | # the remoteSrc path and append it to localDst | ||
163 | if os.path.isdir(localDst): | ||
164 | filename = os.path.basename(remoteSrc) | ||
165 | localDst = os.path.join(localDst, filename) | ||
166 | |||
167 | with open(localDst, 'wb') as file: | ||
168 | file.write(data) | ||
169 | |||
170 | return 0, 'Success' | ||
171 | |||
172 | def copyDirTo(self, localSrc, remoteDst): | ||
173 | """ | ||
174 | Copy recursively localSrc directory to remoteDst in target. | ||
175 | """ | ||
176 | |||
177 | for root, dirs, files in os.walk(localSrc): | ||
178 | # Create directories in the target as needed | ||
179 | for d in dirs: | ||
180 | tmpDir = os.path.join(root, d).replace(localSrc, "") | ||
181 | newDir = os.path.join(remoteDst, tmpDir.lstrip("/")) | ||
182 | cmd = "mkdir -p %s" % newDir | ||
183 | self.run(cmd) | ||
184 | |||
185 | # Copy files into the target | ||
186 | for f in files: | ||
187 | tmpFile = os.path.join(root, f).replace(localSrc, "") | ||
188 | dstFile = os.path.join(remoteDst, tmpFile.lstrip("/")) | ||
189 | srcFile = os.path.join(root, f) | ||
190 | self.copyTo(srcFile, dstFile) | ||
191 | |||
192 | def deleteFiles(self, remotePath, files): | ||
193 | """ | ||
194 | Deletes files in target's remotePath. | ||
195 | """ | ||
196 | |||
197 | cmd = "rm" | ||
198 | if not isinstance(files, list): | ||
199 | files = [files] | ||
200 | |||
201 | for f in files: | ||
202 | cmd = "%s %s" % (cmd, os.path.join(remotePath, f)) | ||
203 | |||
204 | self.run(cmd) | ||
205 | |||
206 | def deleteDir(self, remotePath): | ||
207 | """ | ||
208 | Deletes target's remotePath directory. | ||
209 | """ | ||
210 | |||
211 | cmd = "rmdir %s" % remotePath | ||
212 | self.run(cmd) | ||
213 | |||
214 | def deleteDirStructure(self, localPath, remotePath): | ||
215 | """ | ||
216 | Delete recursively localPath structure directory in target's remotePath. | ||
217 | |||
218 | This function is useful to delete a package that is installed in the | ||
219 | device under test (DUT) and the host running the test has such package | ||
220 | extracted in tmp directory. | ||
221 | |||
222 | Example: | ||
223 | pwd: /home/user/tmp | ||
224 | tree: . | ||
225 | └── work | ||
226 | ├── dir1 | ||
227 | │  └── file1 | ||
228 | └── dir2 | ||
229 | |||
230 | localpath = "/home/user/tmp" and remotepath = "/home/user" | ||
231 | |||
232 | With the above variables this function will try to delete the | ||
233 | directory in the DUT in this order: | ||
234 | /home/user/work/dir1/file1 | ||
235 | /home/user/work/dir1 (if dir is empty) | ||
236 | /home/user/work/dir2 (if dir is empty) | ||
237 | /home/user/work (if dir is empty) | ||
238 | """ | ||
239 | |||
240 | for root, dirs, files in os.walk(localPath, topdown=False): | ||
241 | # Delete files first | ||
242 | tmpDir = os.path.join(root).replace(localPath, "") | ||
243 | remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) | ||
244 | self.deleteFiles(remoteDir, files) | ||
245 | |||
246 | # Remove dirs if empty | ||
247 | for d in dirs: | ||
248 | tmpDir = os.path.join(root, d).replace(localPath, "") | ||
249 | remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) | ||
250 | self.deleteDir(remoteDir) | ||
251 | |||
252 | class SerialTimeoutException(Exception): | ||
253 | def __init__(self, msg): | ||
254 | self.msg = msg | ||
255 | def __str__(self): | ||
256 | return self.msg | ||
257 | |||
258 | class SerialConnection: | ||
259 | |||
260 | def __init__(self, script, target_prompt, connect_timeout, default_command_timeout): | ||
261 | import pexpect # limiting scope to avoid build dependency | ||
262 | self.prompt = target_prompt | ||
263 | self.connect_timeout = connect_timeout | ||
264 | self.default_command_timeout = default_command_timeout | ||
265 | self.conn = pexpect.spawn('/bin/bash', ['-c', script], encoding='utf8') | ||
266 | self._seek_to_clean_shell() | ||
267 | # Disable echo to avoid the need to parse the outgoing command | ||
268 | self.run_command('stty -echo') | ||
269 | |||
270 | def _seek_to_clean_shell(self): | ||
271 | """ | ||
272 | Attempts to find a clean shell, meaning it is clear and | ||
273 | ready to accept a new command. This is necessary to ensure | ||
274 | the correct output is captured from each command. | ||
275 | """ | ||
276 | import pexpect # limiting scope to avoid build dependency | ||
277 | # Look for a clean shell | ||
278 | # Wait a short amount of time for the connection to finish | ||
279 | pexpect_code = self.conn.expect([self.prompt, pexpect.TIMEOUT], | ||
280 | timeout=self.connect_timeout) | ||
281 | |||
282 | # if a timeout occurred, send an empty line and wait for a clean shell | ||
283 | if pexpect_code == 1: | ||
284 | # send a newline to clear and present the shell | ||
285 | self.conn.sendline("") | ||
286 | pexpect_code = self.conn.expect(self.prompt) | ||
287 | |||
288 | def run_command(self, cmd, timeout=None): | ||
289 | """ | ||
290 | Runs command on target over the provided serial connection. | ||
291 | Returns any output on the shell while the command was run. | ||
292 | |||
293 | command: Command to run on target. | ||
294 | timeout: <value>: Kill command after <val> seconds. | ||
295 | None: Kill command default value seconds. | ||
296 | 0: No timeout, runs until return. | ||
297 | """ | ||
298 | import pexpect # limiting scope to avoid build dependency | ||
299 | # Convert from the OETarget defaults to pexpect timeout values | ||
300 | if timeout is None: | ||
301 | timeout = self.default_command_timeout | ||
302 | elif timeout == 0: | ||
303 | timeout = None # passing None to pexpect is infinite timeout | ||
304 | |||
305 | self.conn.sendline(cmd) | ||
306 | pexpect_code = self.conn.expect([self.prompt, pexpect.TIMEOUT], timeout=timeout) | ||
307 | |||
308 | # check for timeout | ||
309 | if pexpect_code == 1: | ||
310 | self.conn.send('\003') # send Ctrl+C | ||
311 | self._seek_to_clean_shell() | ||
312 | raise SerialTimeoutException(f'Timeout executing: {cmd} after {timeout}s') | ||
313 | |||
314 | return self.conn.before.removesuffix('\r\n') | ||
315 | |||
diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py index 461448dbc5..8b5c450a05 100644 --- a/meta/lib/oeqa/core/target/ssh.py +++ b/meta/lib/oeqa/core/target/ssh.py | |||
@@ -34,12 +34,17 @@ class OESSHTarget(OETarget): | |||
34 | self.timeout = timeout | 34 | self.timeout = timeout |
35 | self.user = user | 35 | self.user = user |
36 | ssh_options = [ | 36 | ssh_options = [ |
37 | '-o', 'ServerAliveCountMax=2', | ||
38 | '-o', 'ServerAliveInterval=30', | ||
37 | '-o', 'UserKnownHostsFile=/dev/null', | 39 | '-o', 'UserKnownHostsFile=/dev/null', |
38 | '-o', 'StrictHostKeyChecking=no', | 40 | '-o', 'StrictHostKeyChecking=no', |
39 | '-o', 'LogLevel=ERROR' | 41 | '-o', 'LogLevel=ERROR' |
40 | ] | 42 | ] |
43 | scp_options = [ | ||
44 | '-r' | ||
45 | ] | ||
41 | self.ssh = ['ssh', '-l', self.user ] + ssh_options | 46 | self.ssh = ['ssh', '-l', self.user ] + ssh_options |
42 | self.scp = ['scp'] + ssh_options | 47 | self.scp = ['scp'] + ssh_options + scp_options |
43 | if port: | 48 | if port: |
44 | self.ssh = self.ssh + [ '-p', port ] | 49 | self.ssh = self.ssh + [ '-p', port ] |
45 | self.scp = self.scp + [ '-P', port ] | 50 | self.scp = self.scp + [ '-P', port ] |
@@ -50,14 +55,14 @@ class OESSHTarget(OETarget): | |||
50 | def stop(self, **kwargs): | 55 | def stop(self, **kwargs): |
51 | pass | 56 | pass |
52 | 57 | ||
53 | def _run(self, command, timeout=None, ignore_status=True): | 58 | def _run(self, command, timeout=None, ignore_status=True, raw=False): |
54 | """ | 59 | """ |
55 | Runs command in target using SSHProcess. | 60 | Runs command in target using SSHProcess. |
56 | """ | 61 | """ |
57 | self.logger.debug("[Running]$ %s" % " ".join(command)) | 62 | self.logger.debug("[Running]$ %s" % " ".join(command)) |
58 | 63 | ||
59 | starttime = time.time() | 64 | starttime = time.time() |
60 | status, output = SSHCall(command, self.logger, timeout) | 65 | status, output = SSHCall(command, self.logger, timeout, raw) |
61 | self.logger.debug("[Command returned '%d' after %.2f seconds]" | 66 | self.logger.debug("[Command returned '%d' after %.2f seconds]" |
62 | "" % (status, time.time() - starttime)) | 67 | "" % (status, time.time() - starttime)) |
63 | 68 | ||
@@ -67,7 +72,7 @@ class OESSHTarget(OETarget): | |||
67 | 72 | ||
68 | return (status, output) | 73 | return (status, output) |
69 | 74 | ||
70 | def run(self, command, timeout=None): | 75 | def run(self, command, timeout=None, ignore_status=True, raw=False): |
71 | """ | 76 | """ |
72 | Runs command in target. | 77 | Runs command in target. |
73 | 78 | ||
@@ -86,10 +91,12 @@ class OESSHTarget(OETarget): | |||
86 | else: | 91 | else: |
87 | processTimeout = self.timeout | 92 | processTimeout = self.timeout |
88 | 93 | ||
89 | status, output = self._run(sshCmd, processTimeout, True) | 94 | status, output = self._run(sshCmd, processTimeout, ignore_status, raw) |
90 | self.logger.debug('Command: %s\nOutput: %s\n' % (command, output)) | 95 | if len(output) > (64 * 1024): |
91 | if (status == 255) and (('No route to host') in output): | 96 | self.logger.debug('Command: %s\nStatus: %d Output length: %s\n' % (command, status, len(output))) |
92 | self.target_dumper.dump_target() | 97 | else: |
98 | self.logger.debug('Command: %s\nStatus: %d Output: %s\n' % (command, status, output)) | ||
99 | |||
93 | return (status, output) | 100 | return (status, output) |
94 | 101 | ||
95 | def copyTo(self, localSrc, remoteDst): | 102 | def copyTo(self, localSrc, remoteDst): |
@@ -202,32 +209,51 @@ class OESSHTarget(OETarget): | |||
202 | remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) | 209 | remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) |
203 | self.deleteDir(remoteDir) | 210 | self.deleteDir(remoteDir) |
204 | 211 | ||
205 | def SSHCall(command, logger, timeout=None, **opts): | 212 | def SSHCall(command, logger, timeout=None, raw=False, **opts): |
206 | 213 | ||
207 | def run(): | 214 | def run(): |
208 | nonlocal output | 215 | nonlocal output |
209 | nonlocal process | 216 | nonlocal process |
217 | output_raw = bytearray() | ||
210 | starttime = time.time() | 218 | starttime = time.time() |
219 | progress = time.time() | ||
211 | process = subprocess.Popen(command, **options) | 220 | process = subprocess.Popen(command, **options) |
221 | has_timeout = False | ||
222 | appendline = None | ||
212 | if timeout: | 223 | if timeout: |
213 | endtime = starttime + timeout | 224 | endtime = starttime + timeout |
214 | eof = False | 225 | eof = False |
215 | while time.time() < endtime and not eof: | 226 | os.set_blocking(process.stdout.fileno(), False) |
216 | logger.debug('time: %s, endtime: %s' % (time.time(), endtime)) | 227 | while not has_timeout and not eof: |
217 | try: | 228 | try: |
218 | if select.select([process.stdout], [], [], 5)[0] != []: | 229 | if select.select([process.stdout], [], [], 5)[0] != []: |
219 | reader = codecs.getreader('utf-8')(process.stdout, 'ignore') | 230 | # wait a bit for more data, tries to avoid reading single characters |
220 | data = reader.read(1024, 4096) | 231 | time.sleep(0.2) |
232 | data = process.stdout.read() | ||
221 | if not data: | 233 | if not data: |
222 | process.stdout.close() | ||
223 | eof = True | 234 | eof = True |
224 | else: | 235 | else: |
225 | output += data | 236 | output_raw.extend(data) |
226 | logger.debug('Partial data from SSH call: %s' % data) | 237 | # ignore errors to capture as much as possible |
238 | #logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore')) | ||
227 | endtime = time.time() + timeout | 239 | endtime = time.time() + timeout |
228 | except InterruptedError: | 240 | except InterruptedError: |
241 | logger.debug('InterruptedError') | ||
242 | continue | ||
243 | except BlockingIOError: | ||
244 | logger.debug('BlockingIOError') | ||
229 | continue | 245 | continue |
230 | 246 | ||
247 | if time.time() >= endtime: | ||
248 | logger.debug('SSHCall has timeout! Time: %s, endtime: %s' % (time.time(), endtime)) | ||
249 | has_timeout = True | ||
250 | |||
251 | if time.time() >= (progress + 60): | ||
252 | logger.debug('Waiting for process output at time: %s with datasize: %s' % (time.time(), len(output_raw))) | ||
253 | progress = time.time() | ||
254 | |||
255 | process.stdout.close() | ||
256 | |||
231 | # process hasn't returned yet | 257 | # process hasn't returned yet |
232 | if not eof: | 258 | if not eof: |
233 | process.terminate() | 259 | process.terminate() |
@@ -235,20 +261,58 @@ def SSHCall(command, logger, timeout=None, **opts): | |||
235 | try: | 261 | try: |
236 | process.kill() | 262 | process.kill() |
237 | except OSError: | 263 | except OSError: |
264 | logger.debug('OSError when killing process') | ||
238 | pass | 265 | pass |
239 | endtime = time.time() - starttime | 266 | endtime = time.time() - starttime |
240 | lastline = ("\nProcess killed - no output for %d seconds. Total" | 267 | appendline = ("\nProcess killed - no output for %d seconds. Total" |
241 | " running time: %d seconds." % (timeout, endtime)) | 268 | " running time: %d seconds." % (timeout, endtime)) |
242 | logger.debug('Received data from SSH call %s ' % lastline) | 269 | logger.debug('Received data from SSH call:\n%s ' % appendline) |
243 | output += lastline | 270 | process.wait() |
244 | 271 | ||
272 | if raw: | ||
273 | output = bytes(output_raw) | ||
274 | if appendline: | ||
275 | output += bytes(appendline, "utf-8") | ||
276 | else: | ||
277 | output = output_raw.decode('utf-8', errors='ignore') | ||
278 | if appendline: | ||
279 | output += appendline | ||
245 | else: | 280 | else: |
246 | output = process.communicate()[0].decode('utf-8', errors='ignore') | 281 | output = output_raw = process.communicate()[0] |
247 | logger.debug('Data from SSH call: %s' % output.rstrip()) | 282 | if not raw: |
283 | output = output_raw.decode('utf-8', errors='ignore') | ||
284 | |||
285 | if len(output) < (64 * 1024): | ||
286 | if output.rstrip(): | ||
287 | logger.debug('Data from SSH call:\n%s' % output.rstrip()) | ||
288 | else: | ||
289 | logger.debug('No output from SSH call') | ||
290 | |||
291 | # timout or not, make sure process exits and is not hanging | ||
292 | if process.returncode == None: | ||
293 | try: | ||
294 | process.wait(timeout=5) | ||
295 | except TimeoutExpired: | ||
296 | try: | ||
297 | process.kill() | ||
298 | except OSError: | ||
299 | logger.debug('OSError') | ||
300 | pass | ||
301 | process.wait() | ||
302 | |||
303 | if has_timeout: | ||
304 | # Version of openssh before 8.6_p1 returns error code 0 when killed | ||
305 | # by a signal, when the timeout occurs we will receive a 0 error | ||
306 | # code because the process is been terminated and it's wrong because | ||
307 | # that value means success, but the process timed out. | ||
308 | # Afterwards, from version 8.6_p1 onwards, the returned code is 255. | ||
309 | # Fix this behaviour by checking the return code | ||
310 | if process.returncode == 0: | ||
311 | process.returncode = 255 | ||
248 | 312 | ||
249 | options = { | 313 | options = { |
250 | "stdout": subprocess.PIPE, | 314 | "stdout": subprocess.PIPE, |
251 | "stderr": subprocess.STDOUT, | 315 | "stderr": subprocess.STDOUT if not raw else None, |
252 | "stdin": None, | 316 | "stdin": None, |
253 | "shell": False, | 317 | "shell": False, |
254 | "bufsize": -1, | 318 | "bufsize": -1, |
@@ -271,6 +335,9 @@ def SSHCall(command, logger, timeout=None, **opts): | |||
271 | # whilst running and ensure we don't leave a process behind. | 335 | # whilst running and ensure we don't leave a process behind. |
272 | if process.poll() is None: | 336 | if process.poll() is None: |
273 | process.kill() | 337 | process.kill() |
338 | if process.returncode == None: | ||
339 | process.wait() | ||
274 | logger.debug('Something went wrong, killing SSH process') | 340 | logger.debug('Something went wrong, killing SSH process') |
275 | raise | 341 | raise |
276 | return (process.wait(), output.rstrip()) | 342 | |
343 | return (process.returncode, output if raw else output.rstrip()) | ||