classes/lib: Update to use python3 command pipeline decoding

In python3, strings are unicode by default. We need to encode/decode
from command pipelines and other places where we interface with the
real world using the correct locales. This patch updates various
call sites to use the correct encoding/decodings.

(From OE-Core rev: bb4685af1bffe17b3aa92a6d21398f38a44ea874)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

7 files changed, 25 insertions, 18 deletions

diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py
index a50f9d8187..4211ffc379 100644
--- a/meta/lib/oeqa/oetest.py
+++ b/meta/lib/oeqa/oetest.py
@@ -132,7 +132,7 @@ class oeSDKTest(oeTest):
         return False
 
     def _run(self, cmd):
-        return subprocess.check_output(". %s > /dev/null; %s;" % (self.tc.sdkenv, cmd), shell=True)
+        return subprocess.check_output(". %s > /dev/null; %s;" % (self.tc.sdkenv, cmd), shell=True).decode("utf-8")
 
 class oeSDKExtTest(oeSDKTest):
     def _run(self, cmd):
@@ -144,7 +144,7 @@ class oeSDKExtTest(oeSDKTest):
         env['PATH'] = avoid_paths_in_environ(paths_to_avoid)
 
         return subprocess.check_output(". %s > /dev/null;"\
-            " %s;" % (self.tc.sdkenv, cmd), shell=True, env=env)
+            " %s;" % (self.tc.sdkenv, cmd), shell=True, env=env).decode("utf-8")
 
 def getmodule(pos=2):
     # stack returns a list of tuples containg frame information
diff --git a/meta/lib/oeqa/runtime/parselogs.py b/meta/lib/oeqa/runtime/parselogs.py
index a93660131d..242cd8cdd5 100644
--- a/meta/lib/oeqa/runtime/parselogs.py
+++ b/meta/lib/oeqa/runtime/parselogs.py
@@ -238,7 +238,7 @@ class ParseLogsTest(oeRuntimeTest):
             result = None
             thegrep = self.build_grepcmd(errors, ignore_errors, log)
             try:
-                result = subprocess.check_output(thegrep, shell=True)
+                result = subprocess.check_output(thegrep, shell=True).decode("utf-8")
             except:
                 pass
             if (result is not None):
@@ -246,7 +246,7 @@ class ParseLogsTest(oeRuntimeTest):
                 rez = result.splitlines()
                 for xrez in rez:
                     try:
-                        grep_output = subprocess.check_output(['grep', '-F', xrez, '-B', str(lines_before), '-A', str(lines_after), log])
+                        grep_output = subprocess.check_output(['grep', '-F', xrez, '-B', str(lines_before), '-A', str(lines_after), log]).decode("utf-8")
                     except:
                         pass
                     results[log.replace('target_logs/','')][xrez]=grep_output
diff --git a/meta/lib/oeqa/runtime/ping.py b/meta/lib/oeqa/runtime/ping.py
index 80c460161b..0f27447926 100644
--- a/meta/lib/oeqa/runtime/ping.py
+++ b/meta/lib/oeqa/runtime/ping.py
@@ -14,7 +14,7 @@ class PingTest(oeRuntimeTest):
         endtime = time.time() + 60
         while count < 5 and time.time() < endtime:
             proc = subprocess.Popen("ping -c 1 %s" % self.target.ip, shell=True, stdout=subprocess.PIPE)
-            output += proc.communicate()[0]
+            output += proc.communicate()[0].decode("utf-8")
             if proc.poll() == 0:
                 count += 1
             else:
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 48f6441290..9a7c1d1375 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -78,7 +78,7 @@ class Command(object):
                 self.process.kill()
                 self.thread.join()
 
-        self.output = self.output.rstrip()
+        self.output = self.output.decode("utf-8").rstrip()
         self.status = self.process.poll()
 
         self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status))
diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py
index 695402fead..f51de99458 100644
--- a/meta/lib/oeqa/utils/qemurunner.py
+++ b/meta/lib/oeqa/utils/qemurunner.py
@@ -71,7 +71,8 @@ class QemuRunner:
         if self.logfile:
             # It is needed to sanitize the data received from qemu
             # because is possible to have control characters
-            msg = re_control_char.sub('', unicode(msg, 'utf-8'))
+            msg = msg.decode("utf-8")
+            msg = re_control_char.sub('', msg)
             with codecs.open(self.logfile, "a", encoding="utf-8") as f:
                 f.write("%s" % msg)
 
@@ -79,7 +80,7 @@ class QemuRunner:
         import fcntl
         fl = fcntl.fcntl(o, fcntl.F_GETFL)
         fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK)
-        return os.read(o.fileno(), 1000000)
+        return os.read(o.fileno(), 1000000).decode("utf-8")
 
 
     def handleSIGCHLD(self, signum, frame):
@@ -229,14 +230,19 @@ class QemuRunner:
                         socklist.remove(self.server_socket)
                         logger.info("Connection from %s:%s" % addr)
                     else:
-                        data = sock.recv(1024)
+                        data = data + sock.recv(1024)
                         if data:
-                            bootlog += data
-                            if re.search(".* login:", bootlog):
-                                self.server_socket = qemusock
-                                stopread = True
-                                reachedlogin = True
-                                logger.info("Reached login banner")
+                            try:
+                                data = data.decode("utf-8")
+                                bootlog += data
+                                data = b''
+                                if re.search(".* login:", bootlog):
+                                    self.server_socket = qemusock
+                                    stopread = True
+                                    reachedlogin = True
+                                    logger.info("Reached login banner")
+                            except UnicodeDecodeError:
+                                continue
                         else:
                             socklist.remove(sock)
                             sock.close()
@@ -325,7 +331,7 @@ class QemuRunner:
         # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd]
         #
         ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0]
-        processes = ps.split('\n')
+        processes = ps.decode("utf-8").split('\n')
         nfields = len(processes[0].split()) - 1
         pids = {}
         commands = {}
diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py
index d6ce09645c..054ab0ec5d 100644
--- a/meta/lib/oeqa/utils/qemutinyrunner.py
+++ b/meta/lib/oeqa/utils/qemutinyrunner.py
@@ -102,7 +102,7 @@ class QemuTinyRunner(QemuRunner):
             bb.note("Qemu pid didn't appeared in %s seconds" % self.runqemutime)
             output = self.runqemu.stdout
             self.stop()
-            bb.note("Output from runqemu:\n%s" % output.read())
+            bb.note("Output from runqemu:\n%s" % output.read().decode("utf-8"))
             return False
 
         return self.is_alive()
@@ -131,7 +131,7 @@ class QemuTinyRunner(QemuRunner):
         # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd]
         #
         ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0]
-        processes = ps.split('\n')
+        processes = ps.decode("utf-8").split('\n')
         nfields = len(processes[0].split()) - 1
         pids = {}
         commands = {}
diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py
index ff88d37bd9..f5d46e03cc 100644
--- a/meta/lib/oeqa/utils/sshcontrol.py
+++ b/meta/lib/oeqa/utils/sshcontrol.py
@@ -58,6 +58,7 @@ class SSHProcess(object):
                         self.process.stdout.close()
                         eof = True
                     else:
+                        data = data.decode("utf-8")
                         output += data
                         self.log(data)
                         endtime = time.time() + timeout