From 1fc840ffc0267ecf3a15c4a59ab44869ef1d6339 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Thu, 9 May 2013 16:31:22 +0000 Subject: meta: python3 megapatch This needs splutting into smaller units, WIP atm. (From OE-Core rev: 21529228a7dca96a6a1b44ed9380c523efdeeb3e) Signed-off-by: Richard Purdie --- meta/lib/oeqa/controllers/masterimage.py | 4 +-- meta/lib/oeqa/oetest.py | 10 +++++- meta/lib/oeqa/runtime/files/test.py | 2 +- meta/lib/oeqa/runtime/parselogs.py | 4 +-- meta/lib/oeqa/runtime/ping.py | 2 +- meta/lib/oeqa/runtime/systemd.py | 6 ++-- meta/lib/oeqa/selftest/_toaster.py | 4 +-- meta/lib/oeqa/selftest/devtool.py | 2 +- meta/lib/oeqa/selftest/pkgdata.py | 56 ++++++++++++++++---------------- meta/lib/oeqa/selftest/recipetool.py | 4 +-- meta/lib/oeqa/selftest/sstatetests.py | 4 +-- meta/lib/oeqa/targetcontrol.py | 4 +-- meta/lib/oeqa/utils/commands.py | 6 ++-- meta/lib/oeqa/utils/decorators.py | 19 +++++++++-- meta/lib/oeqa/utils/dump.py | 2 +- meta/lib/oeqa/utils/httpserver.py | 6 ++-- meta/lib/oeqa/utils/logparser.py | 2 +- meta/lib/oeqa/utils/qemurunner.py | 54 +++++++++++++++++------------- meta/lib/oeqa/utils/qemutinyrunner.py | 10 +++--- meta/lib/oeqa/utils/sshcontrol.py | 1 + meta/lib/oeqa/utils/targetbuild.py | 4 +-- meta/lib/oeqa/utils/testexport.py | 2 +- 22 files changed, 117 insertions(+), 91 deletions(-) (limited to 'meta/lib/oeqa') diff --git a/meta/lib/oeqa/controllers/masterimage.py b/meta/lib/oeqa/controllers/masterimage.py index 522f9ebd76..4cb75539ee 100644 --- a/meta/lib/oeqa/controllers/masterimage.py +++ b/meta/lib/oeqa/controllers/masterimage.py @@ -24,9 +24,7 @@ from oeqa.utils import CommandError from abc import ABCMeta, abstractmethod -class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget): - - __metaclass__ = ABCMeta +class MasterImageHardwareTarget(oeqa.targetcontrol.BaseTarget, metaclass=ABCMeta): supported_image_fstypes = ['tar.gz', 'tar.bz2'] diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index 3ed5bb8c2b..869132273f 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py @@ -12,6 +12,7 @@ import unittest import inspect import subprocess import signal +import functools try: import bb except ImportError: @@ -314,7 +315,14 @@ class TestContext(object): for index, suite in enumerate(suites): set_suite_depth(suite) suite.index = index - suites.sort(cmp=lambda a,b: cmp((a.depth, a.index), (b.depth, b.index))) + + def cmp(a, b): + return (a > b) - (a < b) + + def cmpfunc(a, b): + return cmp((a.depth, a.index), (b.depth, b.index)) + + suites.sort(key=functools.cmp_to_key(cmpfunc)) self.suite = testloader.suiteClass(suites) diff --git a/meta/lib/oeqa/runtime/files/test.py b/meta/lib/oeqa/runtime/files/test.py index f3a2273c52..f389225d72 100644 --- a/meta/lib/oeqa/runtime/files/test.py +++ b/meta/lib/oeqa/runtime/files/test.py @@ -3,4 +3,4 @@ import os os.system('touch /tmp/testfile.python') a = 9.01e+21 - 9.01e+21 + 0.01 -print "the value of a is %s" % a +print("the value of a is %s" % a) diff --git a/meta/lib/oeqa/runtime/parselogs.py b/meta/lib/oeqa/runtime/parselogs.py index a93660131d..242cd8cdd5 100644 --- a/meta/lib/oeqa/runtime/parselogs.py +++ b/meta/lib/oeqa/runtime/parselogs.py @@ -238,7 +238,7 @@ class ParseLogsTest(oeRuntimeTest): result = None thegrep = self.build_grepcmd(errors, ignore_errors, log) try: - result = subprocess.check_output(thegrep, shell=True) + result = subprocess.check_output(thegrep, shell=True).decode("utf-8") except: pass if (result is not None): @@ -246,7 +246,7 @@ class ParseLogsTest(oeRuntimeTest): rez = result.splitlines() for xrez in rez: try: - grep_output = subprocess.check_output(['grep', '-F', xrez, '-B', str(lines_before), '-A', str(lines_after), log]) + grep_output = subprocess.check_output(['grep', '-F', xrez, '-B', str(lines_before), '-A', str(lines_after), log]).decode("utf-8") except: pass results[log.replace('target_logs/','')][xrez]=grep_output diff --git a/meta/lib/oeqa/runtime/ping.py b/meta/lib/oeqa/runtime/ping.py index 80c460161b..0f27447926 100644 --- a/meta/lib/oeqa/runtime/ping.py +++ b/meta/lib/oeqa/runtime/ping.py @@ -14,7 +14,7 @@ class PingTest(oeRuntimeTest): endtime = time.time() + 60 while count < 5 and time.time() < endtime: proc = subprocess.Popen("ping -c 1 %s" % self.target.ip, shell=True, stdout=subprocess.PIPE) - output += proc.communicate()[0] + output += proc.communicate()[0].decode("utf-8") if proc.poll() == 0: count += 1 else: diff --git a/meta/lib/oeqa/runtime/systemd.py b/meta/lib/oeqa/runtime/systemd.py index 2b2f10d71c..a96efa28dc 100644 --- a/meta/lib/oeqa/runtime/systemd.py +++ b/meta/lib/oeqa/runtime/systemd.py @@ -153,7 +153,7 @@ class SystemdJournalTests(SystemdTest): if check_match: break # put the startup time in the test log if check_match: - print "%s" % check_match + print("%s" % check_match) else: self.skipTest("Error at obtaining the boot time from journalctl") boot_time_sec = 0 @@ -174,5 +174,5 @@ class SystemdJournalTests(SystemdTest): self.skipTest("Error when parsing time from boot string") #Assert the target boot time against systemd's unit start timeout if boot_time_sec > systemd_TimeoutStartSec: - print "Target boot time %s exceeds systemd's TimeoutStartSec %s"\ - %(boot_time_sec, systemd_TimeoutStartSec) + print("Target boot time %s exceeds systemd's TimeoutStartSec %s"\ + %(boot_time_sec, systemd_TimeoutStartSec)) diff --git a/meta/lib/oeqa/selftest/_toaster.py b/meta/lib/oeqa/selftest/_toaster.py index c424659fdc..15ea9df9ef 100644 --- a/meta/lib/oeqa/selftest/_toaster.py +++ b/meta/lib/oeqa/selftest/_toaster.py @@ -2,7 +2,7 @@ import unittest import os import sys import shlex, subprocess -import urllib, commands, time, getpass, re, json, shlex +import urllib.request, urllib.parse, urllib.error, subprocess, time, getpass, re, json, shlex import oeqa.utils.ftools as ftools from oeqa.selftest.base import oeSelfTest @@ -290,7 +290,7 @@ class Toaster_DB_Tests(ToasterSetup): layers = Layer.objects.values('id', 'layer_index_url') cnt_err = [] for layer in layers: - resp = urllib.urlopen(layer['layer_index_url']) + resp = urllib.request.urlopen(layer['layer_index_url']) if (resp.getcode() != 200): cnt_err.append(layer['id']) self.assertEqual(len(cnt_err), 0, msg = 'Errors for layer id: %s' % cnt_err) diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py index 132a73d0ec..d0421e7177 100644 --- a/meta/lib/oeqa/selftest/devtool.py +++ b/meta/lib/oeqa/selftest/devtool.py @@ -50,7 +50,7 @@ class DevtoolBase(oeSelfTest): missingvars = {} - for var, value in checkvars.iteritems(): + for var, value in checkvars.items(): if value is not None: missingvars[var] = value self.assertEqual(missingvars, {}, 'Some expected variables not found in recipe: %s' % checkvars) diff --git a/meta/lib/oeqa/selftest/pkgdata.py b/meta/lib/oeqa/selftest/pkgdata.py index 138b03aadb..5a63f89ff2 100644 --- a/meta/lib/oeqa/selftest/pkgdata.py +++ b/meta/lib/oeqa/selftest/pkgdata.py @@ -131,15 +131,15 @@ class OePkgdataUtilTests(oeSelfTest): # Test recipe-space package name result = runCmd('oe-pkgdata-util list-pkg-files zlib-dev zlib-doc') files = splitoutput(result.output) - self.assertIn('zlib-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-doc', files.keys(), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) # Test runtime package name result = runCmd('oe-pkgdata-util list-pkg-files -r libz1 libz-dev') files = splitoutput(result.output) - self.assertIn('libz1', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-dev', files.keys(), "listed pkgs. files: %s" %result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) self.assertGreater(len(files['libz1']), 1) libspec = os.path.join(base_libdir, 'libz.so.1.*') found = False @@ -152,12 +152,12 @@ class OePkgdataUtilTests(oeSelfTest): # Test recipe result = runCmd('oe-pkgdata-util list-pkg-files -p zlib') files = splitoutput(result.output) - self.assertIn('zlib-dbg', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-doc', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-staticdev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertNotIn('zlib-locale', files.keys(), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertNotIn('zlib-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) # (ignore ptest, might not be there depending on config) self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) @@ -165,36 +165,36 @@ class OePkgdataUtilTests(oeSelfTest): # Test recipe, runtime result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r') files = splitoutput(result.output) - self.assertIn('libz-dbg', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-doc', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-staticdev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz1', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertNotIn('libz-locale', files.keys(), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertNotIn('libz-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc']) self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev']) # Test recipe, unpackaged result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -u') files = splitoutput(result.output) - self.assertIn('zlib-dbg', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-doc', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-staticdev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('zlib-locale', files.keys(), "listed pkgs. files: %s" %result.output) # this is the key one + self.assertIn('zlib-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('zlib-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) # this is the key one self.assertIn(os.path.join(includedir, 'zlib.h'), files['zlib-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['zlib-doc']) self.assertIn(os.path.join(libdir, 'libz.a'), files['zlib-staticdev']) # Test recipe, runtime, unpackaged result = runCmd('oe-pkgdata-util list-pkg-files -p zlib -r -u') files = splitoutput(result.output) - self.assertIn('libz-dbg', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-doc', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-dev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-staticdev', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz1', files.keys(), "listed pkgs. files: %s" %result.output) - self.assertIn('libz-locale', files.keys(), "listed pkgs. files: %s" %result.output) # this is the key one + self.assertIn('libz-dbg', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-doc', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-dev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-staticdev', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz1', list(files.keys()), "listed pkgs. files: %s" %result.output) + self.assertIn('libz-locale', list(files.keys()), "listed pkgs. files: %s" %result.output) # this is the key one self.assertIn(os.path.join(includedir, 'zlib.h'), files['libz-dev']) self.assertIn(os.path.join(mandir, 'man3/zlib.3'), files['libz-doc']) self.assertIn(os.path.join(libdir, 'libz.a'), files['libz-staticdev']) diff --git a/meta/lib/oeqa/selftest/recipetool.py b/meta/lib/oeqa/selftest/recipetool.py index e72911b0aa..a93d18e275 100644 --- a/meta/lib/oeqa/selftest/recipetool.py +++ b/meta/lib/oeqa/selftest/recipetool.py @@ -1,7 +1,7 @@ import os import logging import tempfile -import urlparse +import urllib.parse from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer from oeqa.utils.decorators import testcase @@ -471,7 +471,7 @@ class RecipetoolAppendsrcBase(RecipetoolBase): '''Return the first file:// in SRC_URI for the specified recipe.''' src_uri = get_bb_var('SRC_URI', recipe).split() for uri in src_uri: - p = urlparse.urlparse(uri) + p = urllib.parse.urlparse(uri) if p.scheme == 'file': return p.netloc + p.path diff --git a/meta/lib/oeqa/selftest/sstatetests.py b/meta/lib/oeqa/selftest/sstatetests.py index a1e5d33580..cc64c6cb68 100644 --- a/meta/lib/oeqa/selftest/sstatetests.py +++ b/meta/lib/oeqa/selftest/sstatetests.py @@ -445,14 +445,14 @@ http_proxy = "http://example.com/" files1 = get_files(topdir + "/tmp-sstatesamehash/stamps/") files2 = get_files(topdir + "/tmp-sstatesamehash2/stamps/") # Remove items that are identical in both sets - for k,v in files1.viewitems() & files2.viewitems(): + for k,v in files1.items() & files2.items(): del files1[k] del files2[k] if not files1 and not files2: # No changes, so we're done return - for k in files1.viewkeys() | files2.viewkeys(): + for k in files1.keys() | files2.keys(): if k in files1 and k in files2: print("%s differs:" % k) print(subprocess.check_output(("bitbake-diffsigs", diff --git a/meta/lib/oeqa/targetcontrol.py b/meta/lib/oeqa/targetcontrol.py index 5422a617c4..1c57efaaef 100644 --- a/meta/lib/oeqa/targetcontrol.py +++ b/meta/lib/oeqa/targetcontrol.py @@ -43,9 +43,7 @@ def get_target_controller(d): return controller(d) -class BaseTarget(object): - - __metaclass__ = ABCMeta +class BaseTarget(object, metaclass=ABCMeta): supported_image_fstypes = [] diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py index 48f6441290..18fe39ecfe 100644 --- a/meta/lib/oeqa/utils/commands.py +++ b/meta/lib/oeqa/utils/commands.py @@ -41,7 +41,7 @@ class Command(object): self.data = data self.options = dict(self.defaultopts) - if isinstance(self.cmd, basestring): + if isinstance(self.cmd, str): self.options["shell"] = True if self.data: self.options['stdin'] = subprocess.PIPE @@ -78,7 +78,7 @@ class Command(object): self.process.kill() self.thread.join() - self.output = self.output.rstrip() + self.output = self.output.decode("utf-8").rstrip() self.status = self.process.poll() self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) @@ -123,7 +123,7 @@ def bitbake(command, ignore_status=False, timeout=None, postconfig=None, **optio else: extra_args = "" - if isinstance(command, basestring): + if isinstance(command, str): cmd = "bitbake " + extra_args + " " + command else: cmd = [ "bitbake" ] + [a for a in (command + extra_args.split(" ")) if a not in [""]] diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py index d52f326f1a..0b23565485 100644 --- a/meta/lib/oeqa/utils/decorators.py +++ b/meta/lib/oeqa/utils/decorators.py @@ -115,6 +115,8 @@ class NoParsingFilter(logging.Filter): def filter(self, record): return record.levelno == 100 +import inspect + def LogResults(original_class): orig_method = original_class.run @@ -124,6 +126,19 @@ def LogResults(original_class): logfile = os.path.join(os.getcwd(),'results-'+caller+'.'+timestamp+'.log') linkfile = os.path.join(os.getcwd(),'results-'+caller+'.log') + def get_class_that_defined_method(meth): + if inspect.ismethod(meth): + for cls in inspect.getmro(meth.__self__.__class__): + if cls.__dict__.get(meth.__name__) is meth: + return cls + meth = meth.__func__ # fallback to __qualname__ parsing + if inspect.isfunction(meth): + cls = getattr(inspect.getmodule(meth), + meth.__qualname__.split('.', 1)[0].rsplit('.', 1)[0]) + if isinstance(cls, type): + return cls + return None + #rewrite the run method of unittest.TestCase to add testcase logging def run(self, result, *args, **kws): orig_method(self, result, *args, **kws) @@ -135,7 +150,7 @@ def LogResults(original_class): except AttributeError: test_case = self._testMethodName - class_name = str(testMethod.im_class).split("'")[1] + class_name = str(get_class_that_defined_method(testMethod)).split("'")[1] #create custom logging level for filtering. custom_log_level = 100 @@ -215,7 +230,7 @@ def tag(*args, **kwargs): def wrap_ob(ob): for name in args: setattr(ob, __tag_prefix + name, True) - for name, value in kwargs.iteritems(): + for name, value in kwargs.items(): setattr(ob, __tag_prefix + name, value) return ob return wrap_ob diff --git a/meta/lib/oeqa/utils/dump.py b/meta/lib/oeqa/utils/dump.py index 63a591d366..71422a9aea 100644 --- a/meta/lib/oeqa/utils/dump.py +++ b/meta/lib/oeqa/utils/dump.py @@ -3,7 +3,7 @@ import sys import errno import datetime import itertools -from commands import runCmd +from .commands import runCmd def get_host_dumper(d): cmds = d.getVar("testimage_dump_host", True) diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py index 76518d8ef9..bd76f36468 100644 --- a/meta/lib/oeqa/utils/httpserver.py +++ b/meta/lib/oeqa/utils/httpserver.py @@ -1,8 +1,8 @@ -import SimpleHTTPServer +import http.server import multiprocessing import os -class HTTPServer(SimpleHTTPServer.BaseHTTPServer.HTTPServer): +class HTTPServer(http.server.HTTPServer): def server_start(self, root_dir): import signal @@ -10,7 +10,7 @@ class HTTPServer(SimpleHTTPServer.BaseHTTPServer.HTTPServer): os.chdir(root_dir) self.serve_forever() -class HTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): +class HTTPRequestHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format_str, *args): pass diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py index 87b50354cd..b377dcd271 100644 --- a/meta/lib/oeqa/utils/logparser.py +++ b/meta/lib/oeqa/utils/logparser.py @@ -3,7 +3,7 @@ import sys import os import re -import ftools +from . import ftools # A parser that can be used to identify weather a line is a test result or a section statement. diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py index 4bede3421c..773cf588b1 100644 --- a/meta/lib/oeqa/utils/qemurunner.py +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -22,9 +22,9 @@ import logging logger = logging.getLogger("BitBake.QemuRunner") # Get Unicode non printable control chars -control_range = range(0,32)+range(127,160) -control_chars = [unichr(x) for x in control_range - if unichr(x) not in string.printable] +control_range = list(range(0,32))+list(range(127,160)) +control_chars = [chr(x) for x in control_range + if chr(x) not in string.printable] re_control_char = re.compile('[%s]' % re.escape("".join(control_chars))) class QemuRunner: @@ -71,7 +71,8 @@ class QemuRunner: if self.logfile: # It is needed to sanitize the data received from qemu # because is possible to have control characters - msg = re_control_char.sub('', unicode(msg, 'utf-8')) + msg = msg.decode("utf-8") + msg = re_control_char.sub('', msg) with codecs.open(self.logfile, "a", encoding="utf-8") as f: f.write("%s" % msg) @@ -79,7 +80,7 @@ class QemuRunner: import fcntl fl = fcntl.fcntl(o, fcntl.F_GETFL) fcntl.fcntl(o, fcntl.F_SETFL, fl | os.O_NONBLOCK) - return os.read(o.fileno(), 1000000) + return os.read(o.fileno(), 1000000).decode("utf-8") def handleSIGCHLD(self, signum, frame): @@ -114,7 +115,7 @@ class QemuRunner: try: threadsock, threadport = self.create_socket() self.server_socket, self.serverport = self.create_socket() - except socket.error, msg: + except socket.error as msg: logger.error("Failed to create listening socket: %s" % msg[1]) return False @@ -192,7 +193,7 @@ class QemuRunner: else: self.ip = ips[0] self.server_ip = ips[1] - except IndexError, ValueError: + except (IndexError, ValueError): logger.info("Couldn't get ip from qemu process arguments! Here is the qemu command line used:\n%s\nand output from runqemu:\n%s" % (cmdline, self.getOutput(output))) self._dump_host() self.stop() @@ -219,6 +220,7 @@ class QemuRunner: stopread = False qemusock = None bootlog = '' + data = b'' while time.time() < endtime and not stopread: sread, swrite, serror = select.select(socklist, [], [], 5) for sock in sread: @@ -229,14 +231,19 @@ class QemuRunner: socklist.remove(self.server_socket) logger.info("Connection from %s:%s" % addr) else: - data = sock.recv(1024) + data = data + sock.recv(1024) if data: - bootlog += data - if re.search(".* login:", bootlog): - self.server_socket = qemusock - stopread = True - reachedlogin = True - logger.info("Reached login banner") + try: + data = data.decode("utf-8") + bootlog += data + data = b'' + if re.search(".* login:", bootlog): + self.server_socket = qemusock + stopread = True + reachedlogin = True + logger.info("Reached login banner") + except UnicodeDecodeError: + continue else: socklist.remove(sock) sock.close() @@ -277,13 +284,14 @@ class QemuRunner: if hasattr(self, "origchldhandler"): signal.signal(signal.SIGCHLD, self.origchldhandler) if self.runqemu: - os.kill(self.monitorpid, signal.SIGKILL) - logger.info("Sending SIGTERM to runqemu") - try: - os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) - except OSError as e: - if e.errno != errno.ESRCH: - raise + if hasattr(self, "monitorpid"): + os.kill(self.monitorpid, signal.SIGKILL) + logger.info("Sending SIGTERM to runqemu") + try: + os.killpg(os.getpgid(self.runqemu.pid), signal.SIGTERM) + except OSError as e: + if e.errno != errno.ESRCH: + raise endtime = time.time() + self.runqemutime while self.runqemu.poll() is None and time.time() < endtime: time.sleep(1) @@ -325,7 +333,7 @@ class QemuRunner: # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] # ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] - processes = ps.split('\n') + processes = ps.decode("utf-8").split('\n') nfields = len(processes[0].split()) - 1 pids = {} commands = {} @@ -442,7 +450,7 @@ class LoggingThread(threading.Thread): def stop(self): self.logger.info("Stopping logging thread") if self.running: - os.write(self.writepipe, "stop") + os.write(self.writepipe, bytes("stop", "utf-8")) def teardown(self): self.logger.info("Tearing down logging thread") diff --git a/meta/lib/oeqa/utils/qemutinyrunner.py b/meta/lib/oeqa/utils/qemutinyrunner.py index e3d8c669e0..f733258bce 100644 --- a/meta/lib/oeqa/utils/qemutinyrunner.py +++ b/meta/lib/oeqa/utils/qemutinyrunner.py @@ -13,7 +13,7 @@ import re import socket import select import bb -from qemurunner import QemuRunner +from .qemurunner import QemuRunner class QemuTinyRunner(QemuRunner): @@ -50,7 +50,7 @@ class QemuTinyRunner(QemuRunner): self.server_socket.connect(self.socketfile) bb.note("Created listening socket for qemu serial console.") tries = 0 - except socket.error, msg: + except socket.error as msg: self.server_socket.close() bb.fatal("Failed to create listening socket.") tries -= 1 @@ -102,7 +102,7 @@ class QemuTinyRunner(QemuRunner): bb.note("Qemu pid didn't appeared in %s seconds" % self.runqemutime) output = self.runqemu.stdout self.stop() - bb.note("Output from runqemu:\n%s" % output.read()) + bb.note("Output from runqemu:\n%s" % output.read().decode("utf-8")) return False return self.is_alive() @@ -131,7 +131,7 @@ class QemuTinyRunner(QemuRunner): # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] # ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] - processes = ps.split('\n') + processes = ps.decode("utf-8").split('\n') nfields = len(processes[0].split()) - 1 pids = {} commands = {} @@ -167,4 +167,4 @@ class QemuTinyRunner(QemuRunner): basecmd = commands[p].split()[0] basecmd = os.path.basename(basecmd) if "qemu-system" in basecmd and "-serial unix" in commands[p]: - return [int(p),commands[p]] \ No newline at end of file + return [int(p),commands[p]] diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py index ff88d37bd9..f5d46e03cc 100644 --- a/meta/lib/oeqa/utils/sshcontrol.py +++ b/meta/lib/oeqa/utils/sshcontrol.py @@ -58,6 +58,7 @@ class SSHProcess(object): self.process.stdout.close() eof = True else: + data = data.decode("utf-8") output += data self.log(data) endtime = time.time() + timeout diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py index f850d78df1..d538f6b65f 100644 --- a/meta/lib/oeqa/utils/targetbuild.py +++ b/meta/lib/oeqa/utils/targetbuild.py @@ -10,9 +10,7 @@ import bb.utils import subprocess from abc import ABCMeta, abstractmethod -class BuildProject(): - - __metaclass__ = ABCMeta +class BuildProject(metaclass=ABCMeta): def __init__(self, d, uri, foldername=None, tmpdir="/tmp/"): self.d = d diff --git a/meta/lib/oeqa/utils/testexport.py b/meta/lib/oeqa/utils/testexport.py index 4fbf4bdcb3..57be2ca449 100644 --- a/meta/lib/oeqa/utils/testexport.py +++ b/meta/lib/oeqa/utils/testexport.py @@ -6,7 +6,7 @@ import os, re, glob as g, shutil as sh,sys from time import sleep -from commands import runCmd +from .commands import runCmd from difflib import SequenceMatcher as SM try: -- cgit v1.2.3-54-g00ecf