summaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa/core/target
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/oeqa/core/target')
-rw-r--r--meta/lib/oeqa/core/target/__init__.py1
-rw-r--r--meta/lib/oeqa/core/target/qemu.py40
-rw-r--r--meta/lib/oeqa/core/target/serial.py315
-rw-r--r--meta/lib/oeqa/core/target/ssh.py115
4 files changed, 437 insertions, 34 deletions
diff --git a/meta/lib/oeqa/core/target/__init__.py b/meta/lib/oeqa/core/target/__init__.py
index 1382aa9b52..177f648fe3 100644
--- a/meta/lib/oeqa/core/target/__init__.py
+++ b/meta/lib/oeqa/core/target/__init__.py
@@ -10,6 +10,7 @@ class OETarget(object):
10 10
11 def __init__(self, logger, *args, **kwargs): 11 def __init__(self, logger, *args, **kwargs):
12 self.logger = logger 12 self.logger = logger
13 self.runner = None
13 14
14 @abstractmethod 15 @abstractmethod
15 def start(self): 16 def start(self):
diff --git a/meta/lib/oeqa/core/target/qemu.py b/meta/lib/oeqa/core/target/qemu.py
index 0f29414df5..d93b3ac94a 100644
--- a/meta/lib/oeqa/core/target/qemu.py
+++ b/meta/lib/oeqa/core/target/qemu.py
@@ -8,20 +8,21 @@ import os
8import sys 8import sys
9import signal 9import signal
10import time 10import time
11import glob
12import subprocess
11from collections import defaultdict 13from collections import defaultdict
12 14
13from .ssh import OESSHTarget 15from .ssh import OESSHTarget
14from oeqa.utils.qemurunner import QemuRunner 16from oeqa.utils.qemurunner import QemuRunner
15from oeqa.utils.dump import TargetDumper
16 17
17supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic'] 18supported_fstypes = ['ext3', 'ext4', 'cpio.gz', 'wic']
18 19
19class OEQemuTarget(OESSHTarget): 20class OEQemuTarget(OESSHTarget):
20 def __init__(self, logger, server_ip, timeout=300, user='root', 21 def __init__(self, logger, server_ip, timeout=300, user='root',
21 port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False, 22 port=None, machine='', rootfs='', kernel='', kvm=False, slirp=False,
22 dump_dir='', dump_host_cmds='', display='', bootlog='', 23 dump_dir='', display='', bootlog='',
23 tmpdir='', dir_image='', boottime=60, serial_ports=2, 24 tmpdir='', dir_image='', boottime=60, serial_ports=2,
24 boot_patterns = defaultdict(str), ovmf=False, **kwargs): 25 boot_patterns = defaultdict(str), ovmf=False, tmpfsdir=None, **kwargs):
25 26
26 super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout, 27 super(OEQemuTarget, self).__init__(logger, None, server_ip, timeout,
27 user, port) 28 user, port)
@@ -35,17 +36,15 @@ class OEQemuTarget(OESSHTarget):
35 self.ovmf = ovmf 36 self.ovmf = ovmf
36 self.use_slirp = slirp 37 self.use_slirp = slirp
37 self.boot_patterns = boot_patterns 38 self.boot_patterns = boot_patterns
39 self.dump_dir = dump_dir
40 self.bootlog = bootlog
38 41
39 self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir, 42 self.runner = QemuRunner(machine=machine, rootfs=rootfs, tmpdir=tmpdir,
40 deploy_dir_image=dir_image, display=display, 43 deploy_dir_image=dir_image, display=display,
41 logfile=bootlog, boottime=boottime, 44 logfile=bootlog, boottime=boottime,
42 use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, 45 use_kvm=kvm, use_slirp=slirp, dump_dir=dump_dir, logger=logger,
43 dump_host_cmds=dump_host_cmds, logger=logger,
44 serial_ports=serial_ports, boot_patterns = boot_patterns, 46 serial_ports=serial_ports, boot_patterns = boot_patterns,
45 use_ovmf=ovmf) 47 use_ovmf=ovmf, tmpfsdir=tmpfsdir)
46 dump_target_cmds = kwargs.get("testimage_dump_target")
47 self.target_dumper = TargetDumper(dump_target_cmds, dump_dir, self.runner)
48 self.target_dumper.create_dir("qemu")
49 48
50 def start(self, params=None, extra_bootparams=None, runqemuparams=''): 49 def start(self, params=None, extra_bootparams=None, runqemuparams=''):
51 if self.use_slirp and not self.server_ip: 50 if self.use_slirp and not self.server_ip:
@@ -68,7 +67,28 @@ class OEQemuTarget(OESSHTarget):
68 self.server_ip = self.runner.server_ip 67 self.server_ip = self.runner.server_ip
69 else: 68 else:
70 self.stop() 69 self.stop()
71 raise RuntimeError("FAILED to start qemu - check the task log and the boot log") 70 # Display the first 20 lines of top and
71 # last 20 lines of the bootlog when the
72 # target is not being booted up.
73 topfile = glob.glob(self.dump_dir + "/*_qemu/host_*_top")
74 msg = "\n\n===== start: snippet =====\n\n"
75 for f in topfile:
76 msg += "file: %s\n\n" % f
77 with open(f) as tf:
78 for x in range(20):
79 msg += next(tf)
80 msg += "\n\n===== end: snippet =====\n\n"
81 blcmd = ["tail", "-20", self.bootlog]
82 msg += "===== start: snippet =====\n\n"
83 try:
84 out = subprocess.check_output(blcmd, stderr=subprocess.STDOUT, timeout=1).decode('utf-8')
85 msg += "file: %s\n\n" % self.bootlog
86 msg += out
87 except (subprocess.CalledProcessError, subprocess.TimeoutExpired, FileNotFoundError) as err:
88 msg += "Error running command: %s\n%s\n" % (blcmd, err)
89 msg += "\n\n===== end: snippet =====\n"
90
91 raise RuntimeError("FAILED to start qemu - check the task log and the boot log %s" % (msg))
72 92
73 def stop(self): 93 def stop(self):
74 self.runner.stop() 94 self.runner.stop()
diff --git a/meta/lib/oeqa/core/target/serial.py b/meta/lib/oeqa/core/target/serial.py
new file mode 100644
index 0000000000..7c2cd8b248
--- /dev/null
+++ b/meta/lib/oeqa/core/target/serial.py
@@ -0,0 +1,315 @@
1#
2# SPDX-License-Identifier: MIT
3#
4
5import base64
6import logging
7import os
8from threading import Lock
9from . import OETarget
10
11class OESerialTarget(OETarget):
12
13 def __init__(self, logger, target_ip, server_ip, server_port=0,
14 timeout=300, serialcontrol_cmd=None, serialcontrol_extra_args=None,
15 serialcontrol_ps1=None, serialcontrol_connect_timeout=None,
16 machine=None, **kwargs):
17 if not logger:
18 logger = logging.getLogger('target')
19 logger.setLevel(logging.INFO)
20 filePath = os.path.join(os.getcwd(), 'remoteTarget.log')
21 fileHandler = logging.FileHandler(filePath, 'w', 'utf-8')
22 formatter = logging.Formatter(
23 '%(asctime)s.%(msecs)03d %(levelname)s: %(message)s',
24 '%H:%M:%S')
25 fileHandler.setFormatter(formatter)
26 logger.addHandler(fileHandler)
27
28 super(OESerialTarget, self).__init__(logger)
29
30 if serialcontrol_ps1:
31 self.target_ps1 = serialcontrol_ps1
32 elif machine:
33 # fallback to a default value which assumes root@machine
34 self.target_ps1 = f'root@{machine}:.*# '
35 else:
36 raise ValueError("Unable to determine shell command prompt (PS1) format.")
37
38 if not serialcontrol_cmd:
39 raise ValueError("Unable to determine serial control command.")
40
41 if serialcontrol_extra_args:
42 self.connection_script = f'{serialcontrol_cmd} {serialcontrol_extra_args}'
43 else:
44 self.connection_script = serialcontrol_cmd
45
46 if serialcontrol_connect_timeout:
47 self.connect_timeout = serialcontrol_connect_timeout
48 else:
49 self.connect_timeout = 10 # default to 10s connection timeout
50
51 self.default_command_timeout = timeout
52 self.ip = target_ip
53 self.server_ip = server_ip
54 self.server_port = server_port
55 self.conn = None
56 self.mutex = Lock()
57
58 def start(self, **kwargs):
59 pass
60
61 def stop(self, **kwargs):
62 pass
63
64 def get_connection(self):
65 if self.conn is None:
66 self.conn = SerialConnection(self.connection_script,
67 self.target_ps1,
68 self.connect_timeout,
69 self.default_command_timeout)
70
71 return self.conn
72
73 def run(self, cmd, timeout=None):
74 """
75 Runs command on target over the provided serial connection.
76 The first call will open the connection, and subsequent
77 calls will re-use the same connection to send new commands.
78
79 command: Command to run on target.
80 timeout: <value>: Kill command after <val> seconds.
81 None: Kill command default value seconds.
82 0: No timeout, runs until return.
83 """
84 # Lock needed to avoid multiple threads running commands concurrently
85 # A serial connection can only be used by one caller at a time
86 with self.mutex:
87 conn = self.get_connection()
88
89 self.logger.debug(f"[Running]$ {cmd}")
90 # Run the command, then echo $? to get the command's return code
91 try:
92 output = conn.run_command(cmd, timeout)
93 status = conn.run_command("echo $?")
94 self.logger.debug(f" [stdout]: {output}")
95 self.logger.debug(f" [ret code]: {status}\n\n")
96 except SerialTimeoutException as e:
97 self.logger.debug(e)
98 output = ""
99 status = 255
100
101 # Return to $HOME after each command to simulate a stateless SSH connection
102 conn.run_command('cd "$HOME"')
103
104 return (int(status), output)
105
106 def copyTo(self, localSrc, remoteDst):
107 """
108 Copies files by converting them to base 32, then transferring
109 the ASCII text to the target, and decoding it in place on the
110 target.
111
112 On a 115k baud serial connection, this method transfers at
113 roughly 30kbps.
114 """
115 with open(localSrc, 'rb') as file:
116 data = file.read()
117
118 b32 = base64.b32encode(data).decode('utf-8')
119
120 # To avoid shell line limits, send a chunk at a time
121 SPLIT_LEN = 512
122 lines = [b32[i:i+SPLIT_LEN] for i in range(0, len(b32), SPLIT_LEN)]
123
124 with self.mutex:
125 conn = self.get_connection()
126
127 filename = os.path.basename(localSrc)
128 TEMP = f'/tmp/{filename}.b32'
129
130 # Create or empty out the temp file
131 conn.run_command(f'echo -n "" > {TEMP}')
132
133 for line in lines:
134 conn.run_command(f'echo -n {line} >> {TEMP}')
135
136 # Check to see whether the remoteDst is a directory
137 is_directory = conn.run_command(f'[[ -d {remoteDst} ]]; echo $?')
138 if int(is_directory) == 0:
139 # append the localSrc filename to the end of remoteDst
140 remoteDst = os.path.join(remoteDst, filename)
141
142 conn.run_command(f'base32 -d {TEMP} > {remoteDst}')
143 conn.run_command(f'rm {TEMP}')
144
145 return 0, 'Success'
146
147 def copyFrom(self, remoteSrc, localDst):
148 """
149 Copies files by converting them to base 32 on the target, then
150 transferring the ASCII text to the host. That text is then
151 decoded here and written out to the destination.
152
153 On a 115k baud serial connection, this method transfers at
154 roughly 30kbps.
155 """
156 with self.mutex:
157 b32 = self.get_connection().run_command(f'base32 {remoteSrc}')
158
159 data = base64.b32decode(b32.replace('\r\n', ''))
160
161 # If the local path is a directory, get the filename from
162 # the remoteSrc path and append it to localDst
163 if os.path.isdir(localDst):
164 filename = os.path.basename(remoteSrc)
165 localDst = os.path.join(localDst, filename)
166
167 with open(localDst, 'wb') as file:
168 file.write(data)
169
170 return 0, 'Success'
171
172 def copyDirTo(self, localSrc, remoteDst):
173 """
174 Copy recursively localSrc directory to remoteDst in target.
175 """
176
177 for root, dirs, files in os.walk(localSrc):
178 # Create directories in the target as needed
179 for d in dirs:
180 tmpDir = os.path.join(root, d).replace(localSrc, "")
181 newDir = os.path.join(remoteDst, tmpDir.lstrip("/"))
182 cmd = "mkdir -p %s" % newDir
183 self.run(cmd)
184
185 # Copy files into the target
186 for f in files:
187 tmpFile = os.path.join(root, f).replace(localSrc, "")
188 dstFile = os.path.join(remoteDst, tmpFile.lstrip("/"))
189 srcFile = os.path.join(root, f)
190 self.copyTo(srcFile, dstFile)
191
192 def deleteFiles(self, remotePath, files):
193 """
194 Deletes files in target's remotePath.
195 """
196
197 cmd = "rm"
198 if not isinstance(files, list):
199 files = [files]
200
201 for f in files:
202 cmd = "%s %s" % (cmd, os.path.join(remotePath, f))
203
204 self.run(cmd)
205
206 def deleteDir(self, remotePath):
207 """
208 Deletes target's remotePath directory.
209 """
210
211 cmd = "rmdir %s" % remotePath
212 self.run(cmd)
213
214 def deleteDirStructure(self, localPath, remotePath):
215 """
216 Delete recursively localPath structure directory in target's remotePath.
217
218 This function is useful to delete a package that is installed in the
219 device under test (DUT) and the host running the test has such package
220 extracted in tmp directory.
221
222 Example:
223 pwd: /home/user/tmp
224 tree: .
225 └── work
226 ├── dir1
227 │   └── file1
228 └── dir2
229
230 localpath = "/home/user/tmp" and remotepath = "/home/user"
231
232 With the above variables this function will try to delete the
233 directory in the DUT in this order:
234 /home/user/work/dir1/file1
235 /home/user/work/dir1 (if dir is empty)
236 /home/user/work/dir2 (if dir is empty)
237 /home/user/work (if dir is empty)
238 """
239
240 for root, dirs, files in os.walk(localPath, topdown=False):
241 # Delete files first
242 tmpDir = os.path.join(root).replace(localPath, "")
243 remoteDir = os.path.join(remotePath, tmpDir.lstrip("/"))
244 self.deleteFiles(remoteDir, files)
245
246 # Remove dirs if empty
247 for d in dirs:
248 tmpDir = os.path.join(root, d).replace(localPath, "")
249 remoteDir = os.path.join(remotePath, tmpDir.lstrip("/"))
250 self.deleteDir(remoteDir)
251
252class SerialTimeoutException(Exception):
253 def __init__(self, msg):
254 self.msg = msg
255 def __str__(self):
256 return self.msg
257
258class SerialConnection:
259
260 def __init__(self, script, target_prompt, connect_timeout, default_command_timeout):
261 import pexpect # limiting scope to avoid build dependency
262 self.prompt = target_prompt
263 self.connect_timeout = connect_timeout
264 self.default_command_timeout = default_command_timeout
265 self.conn = pexpect.spawn('/bin/bash', ['-c', script], encoding='utf8')
266 self._seek_to_clean_shell()
267 # Disable echo to avoid the need to parse the outgoing command
268 self.run_command('stty -echo')
269
270 def _seek_to_clean_shell(self):
271 """
272 Attempts to find a clean shell, meaning it is clear and
273 ready to accept a new command. This is necessary to ensure
274 the correct output is captured from each command.
275 """
276 import pexpect # limiting scope to avoid build dependency
277 # Look for a clean shell
278 # Wait a short amount of time for the connection to finish
279 pexpect_code = self.conn.expect([self.prompt, pexpect.TIMEOUT],
280 timeout=self.connect_timeout)
281
282 # if a timeout occurred, send an empty line and wait for a clean shell
283 if pexpect_code == 1:
284 # send a newline to clear and present the shell
285 self.conn.sendline("")
286 pexpect_code = self.conn.expect(self.prompt)
287
288 def run_command(self, cmd, timeout=None):
289 """
290 Runs command on target over the provided serial connection.
291 Returns any output on the shell while the command was run.
292
293 command: Command to run on target.
294 timeout: <value>: Kill command after <val> seconds.
295 None: Kill command default value seconds.
296 0: No timeout, runs until return.
297 """
298 import pexpect # limiting scope to avoid build dependency
299 # Convert from the OETarget defaults to pexpect timeout values
300 if timeout is None:
301 timeout = self.default_command_timeout
302 elif timeout == 0:
303 timeout = None # passing None to pexpect is infinite timeout
304
305 self.conn.sendline(cmd)
306 pexpect_code = self.conn.expect([self.prompt, pexpect.TIMEOUT], timeout=timeout)
307
308 # check for timeout
309 if pexpect_code == 1:
310 self.conn.send('\003') # send Ctrl+C
311 self._seek_to_clean_shell()
312 raise SerialTimeoutException(f'Timeout executing: {cmd} after {timeout}s')
313
314 return self.conn.before.removesuffix('\r\n')
315
diff --git a/meta/lib/oeqa/core/target/ssh.py b/meta/lib/oeqa/core/target/ssh.py
index 461448dbc5..8b5c450a05 100644
--- a/meta/lib/oeqa/core/target/ssh.py
+++ b/meta/lib/oeqa/core/target/ssh.py
@@ -34,12 +34,17 @@ class OESSHTarget(OETarget):
34 self.timeout = timeout 34 self.timeout = timeout
35 self.user = user 35 self.user = user
36 ssh_options = [ 36 ssh_options = [
37 '-o', 'ServerAliveCountMax=2',
38 '-o', 'ServerAliveInterval=30',
37 '-o', 'UserKnownHostsFile=/dev/null', 39 '-o', 'UserKnownHostsFile=/dev/null',
38 '-o', 'StrictHostKeyChecking=no', 40 '-o', 'StrictHostKeyChecking=no',
39 '-o', 'LogLevel=ERROR' 41 '-o', 'LogLevel=ERROR'
40 ] 42 ]
43 scp_options = [
44 '-r'
45 ]
41 self.ssh = ['ssh', '-l', self.user ] + ssh_options 46 self.ssh = ['ssh', '-l', self.user ] + ssh_options
42 self.scp = ['scp'] + ssh_options 47 self.scp = ['scp'] + ssh_options + scp_options
43 if port: 48 if port:
44 self.ssh = self.ssh + [ '-p', port ] 49 self.ssh = self.ssh + [ '-p', port ]
45 self.scp = self.scp + [ '-P', port ] 50 self.scp = self.scp + [ '-P', port ]
@@ -50,14 +55,14 @@ class OESSHTarget(OETarget):
50 def stop(self, **kwargs): 55 def stop(self, **kwargs):
51 pass 56 pass
52 57
53 def _run(self, command, timeout=None, ignore_status=True): 58 def _run(self, command, timeout=None, ignore_status=True, raw=False):
54 """ 59 """
55 Runs command in target using SSHProcess. 60 Runs command in target using SSHProcess.
56 """ 61 """
57 self.logger.debug("[Running]$ %s" % " ".join(command)) 62 self.logger.debug("[Running]$ %s" % " ".join(command))
58 63
59 starttime = time.time() 64 starttime = time.time()
60 status, output = SSHCall(command, self.logger, timeout) 65 status, output = SSHCall(command, self.logger, timeout, raw)
61 self.logger.debug("[Command returned '%d' after %.2f seconds]" 66 self.logger.debug("[Command returned '%d' after %.2f seconds]"
62 "" % (status, time.time() - starttime)) 67 "" % (status, time.time() - starttime))
63 68
@@ -67,7 +72,7 @@ class OESSHTarget(OETarget):
67 72
68 return (status, output) 73 return (status, output)
69 74
70 def run(self, command, timeout=None): 75 def run(self, command, timeout=None, ignore_status=True, raw=False):
71 """ 76 """
72 Runs command in target. 77 Runs command in target.
73 78
@@ -86,10 +91,12 @@ class OESSHTarget(OETarget):
86 else: 91 else:
87 processTimeout = self.timeout 92 processTimeout = self.timeout
88 93
89 status, output = self._run(sshCmd, processTimeout, True) 94 status, output = self._run(sshCmd, processTimeout, ignore_status, raw)
90 self.logger.debug('Command: %s\nOutput: %s\n' % (command, output)) 95 if len(output) > (64 * 1024):
91 if (status == 255) and (('No route to host') in output): 96 self.logger.debug('Command: %s\nStatus: %d Output length: %s\n' % (command, status, len(output)))
92 self.target_dumper.dump_target() 97 else:
98 self.logger.debug('Command: %s\nStatus: %d Output: %s\n' % (command, status, output))
99
93 return (status, output) 100 return (status, output)
94 101
95 def copyTo(self, localSrc, remoteDst): 102 def copyTo(self, localSrc, remoteDst):
@@ -202,32 +209,51 @@ class OESSHTarget(OETarget):
202 remoteDir = os.path.join(remotePath, tmpDir.lstrip("/")) 209 remoteDir = os.path.join(remotePath, tmpDir.lstrip("/"))
203 self.deleteDir(remoteDir) 210 self.deleteDir(remoteDir)
204 211
205def SSHCall(command, logger, timeout=None, **opts): 212def SSHCall(command, logger, timeout=None, raw=False, **opts):
206 213
207 def run(): 214 def run():
208 nonlocal output 215 nonlocal output
209 nonlocal process 216 nonlocal process
217 output_raw = bytearray()
210 starttime = time.time() 218 starttime = time.time()
219 progress = time.time()
211 process = subprocess.Popen(command, **options) 220 process = subprocess.Popen(command, **options)
221 has_timeout = False
222 appendline = None
212 if timeout: 223 if timeout:
213 endtime = starttime + timeout 224 endtime = starttime + timeout
214 eof = False 225 eof = False
215 while time.time() < endtime and not eof: 226 os.set_blocking(process.stdout.fileno(), False)
216 logger.debug('time: %s, endtime: %s' % (time.time(), endtime)) 227 while not has_timeout and not eof:
217 try: 228 try:
218 if select.select([process.stdout], [], [], 5)[0] != []: 229 if select.select([process.stdout], [], [], 5)[0] != []:
219 reader = codecs.getreader('utf-8')(process.stdout, 'ignore') 230 # wait a bit for more data, tries to avoid reading single characters
220 data = reader.read(1024, 4096) 231 time.sleep(0.2)
232 data = process.stdout.read()
221 if not data: 233 if not data:
222 process.stdout.close()
223 eof = True 234 eof = True
224 else: 235 else:
225 output += data 236 output_raw.extend(data)
226 logger.debug('Partial data from SSH call: %s' % data) 237 # ignore errors to capture as much as possible
238 #logger.debug('Partial data from SSH call:\n%s' % data.decode('utf-8', errors='ignore'))
227 endtime = time.time() + timeout 239 endtime = time.time() + timeout
228 except InterruptedError: 240 except InterruptedError:
241 logger.debug('InterruptedError')
242 continue
243 except BlockingIOError:
244 logger.debug('BlockingIOError')
229 continue 245 continue
230 246
247 if time.time() >= endtime:
248 logger.debug('SSHCall has timeout! Time: %s, endtime: %s' % (time.time(), endtime))
249 has_timeout = True
250
251 if time.time() >= (progress + 60):
252 logger.debug('Waiting for process output at time: %s with datasize: %s' % (time.time(), len(output_raw)))
253 progress = time.time()
254
255 process.stdout.close()
256
231 # process hasn't returned yet 257 # process hasn't returned yet
232 if not eof: 258 if not eof:
233 process.terminate() 259 process.terminate()
@@ -235,20 +261,58 @@ def SSHCall(command, logger, timeout=None, **opts):
235 try: 261 try:
236 process.kill() 262 process.kill()
237 except OSError: 263 except OSError:
264 logger.debug('OSError when killing process')
238 pass 265 pass
239 endtime = time.time() - starttime 266 endtime = time.time() - starttime
240 lastline = ("\nProcess killed - no output for %d seconds. Total" 267 appendline = ("\nProcess killed - no output for %d seconds. Total"
241 " running time: %d seconds." % (timeout, endtime)) 268 " running time: %d seconds." % (timeout, endtime))
242 logger.debug('Received data from SSH call %s ' % lastline) 269 logger.debug('Received data from SSH call:\n%s ' % appendline)
243 output += lastline 270 process.wait()
244 271
272 if raw:
273 output = bytes(output_raw)
274 if appendline:
275 output += bytes(appendline, "utf-8")
276 else:
277 output = output_raw.decode('utf-8', errors='ignore')
278 if appendline:
279 output += appendline
245 else: 280 else:
246 output = process.communicate()[0].decode('utf-8', errors='ignore') 281 output = output_raw = process.communicate()[0]
247 logger.debug('Data from SSH call: %s' % output.rstrip()) 282 if not raw:
283 output = output_raw.decode('utf-8', errors='ignore')
284
285 if len(output) < (64 * 1024):
286 if output.rstrip():
287 logger.debug('Data from SSH call:\n%s' % output.rstrip())
288 else:
289 logger.debug('No output from SSH call')
290
291 # timout or not, make sure process exits and is not hanging
292 if process.returncode == None:
293 try:
294 process.wait(timeout=5)
295 except TimeoutExpired:
296 try:
297 process.kill()
298 except OSError:
299 logger.debug('OSError')
300 pass
301 process.wait()
302
303 if has_timeout:
304 # Version of openssh before 8.6_p1 returns error code 0 when killed
305 # by a signal, when the timeout occurs we will receive a 0 error
306 # code because the process is been terminated and it's wrong because
307 # that value means success, but the process timed out.
308 # Afterwards, from version 8.6_p1 onwards, the returned code is 255.
309 # Fix this behaviour by checking the return code
310 if process.returncode == 0:
311 process.returncode = 255
248 312
249 options = { 313 options = {
250 "stdout": subprocess.PIPE, 314 "stdout": subprocess.PIPE,
251 "stderr": subprocess.STDOUT, 315 "stderr": subprocess.STDOUT if not raw else None,
252 "stdin": None, 316 "stdin": None,
253 "shell": False, 317 "shell": False,
254 "bufsize": -1, 318 "bufsize": -1,
@@ -271,6 +335,9 @@ def SSHCall(command, logger, timeout=None, **opts):
271 # whilst running and ensure we don't leave a process behind. 335 # whilst running and ensure we don't leave a process behind.
272 if process.poll() is None: 336 if process.poll() is None:
273 process.kill() 337 process.kill()
338 if process.returncode == None:
339 process.wait()
274 logger.debug('Something went wrong, killing SSH process') 340 logger.debug('Something went wrong, killing SSH process')
275 raise 341 raise
276 return (process.wait(), output.rstrip()) 342
343 return (process.returncode, output if raw else output.rstrip())