From c527fd1f14c27855a37f2e8ac5346ce8d940ced2 Mon Sep 17 00:00:00 2001 From: Tudor Florea Date: Thu, 16 Oct 2014 03:05:19 +0200 Subject: initial commit for Enea Linux 4.0-140929 Migrated from the internal git server on the daisy-enea-point-release branch Signed-off-by: Tudor Florea --- meta/lib/oeqa/utils/__init__.py | 3 + meta/lib/oeqa/utils/commands.py | 137 +++++++++++++++++++++ meta/lib/oeqa/utils/decorators.py | 50 ++++++++ meta/lib/oeqa/utils/ftools.py | 27 +++++ meta/lib/oeqa/utils/httpserver.py | 33 ++++++ meta/lib/oeqa/utils/qemurunner.py | 237 +++++++++++++++++++++++++++++++++++++ meta/lib/oeqa/utils/sshcontrol.py | 127 ++++++++++++++++++++ meta/lib/oeqa/utils/targetbuild.py | 68 +++++++++++ 8 files changed, 682 insertions(+) create mode 100644 meta/lib/oeqa/utils/__init__.py create mode 100644 meta/lib/oeqa/utils/commands.py create mode 100644 meta/lib/oeqa/utils/decorators.py create mode 100644 meta/lib/oeqa/utils/ftools.py create mode 100644 meta/lib/oeqa/utils/httpserver.py create mode 100644 meta/lib/oeqa/utils/qemurunner.py create mode 100644 meta/lib/oeqa/utils/sshcontrol.py create mode 100644 meta/lib/oeqa/utils/targetbuild.py (limited to 'meta/lib/oeqa/utils') diff --git a/meta/lib/oeqa/utils/__init__.py b/meta/lib/oeqa/utils/__init__.py new file mode 100644 index 0000000000..8eda92763c --- /dev/null +++ b/meta/lib/oeqa/utils/__init__.py @@ -0,0 +1,3 @@ +# Enable other layers to have modules in the same named directory +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py new file mode 100644 index 0000000000..9b42620610 --- /dev/null +++ b/meta/lib/oeqa/utils/commands.py @@ -0,0 +1,137 @@ +# Copyright (c) 2013 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# DESCRIPTION +# This module is mainly used by scripts/oe-selftest and modules under meta/oeqa/selftest +# It provides a class and methods for running commands on the host in a convienent way for tests. + + + +import os +import sys +import signal +import subprocess +import threading +import logging + +class Command(object): + def __init__(self, command, bg=False, timeout=None, data=None, **options): + + self.defaultopts = { + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + "stdin": None, + "shell": False, + "bufsize": -1, + } + + self.cmd = command + self.bg = bg + self.timeout = timeout + self.data = data + + self.options = dict(self.defaultopts) + if isinstance(self.cmd, basestring): + self.options["shell"] = True + if self.data: + self.options['stdin'] = subprocess.PIPE + self.options.update(options) + + self.status = None + self.output = None + self.error = None + self.thread = None + + self.log = logging.getLogger("utils.commands") + + def run(self): + self.process = subprocess.Popen(self.cmd, **self.options) + + def commThread(): + self.output, self.error = self.process.communicate(self.data) + + self.thread = threading.Thread(target=commThread) + self.thread.start() + + self.log.debug("Running command '%s'" % self.cmd) + + if not self.bg: + self.thread.join(self.timeout) + self.stop() + + def stop(self): + if self.thread.isAlive(): + self.process.terminate() + # let's give it more time to terminate gracefully before killing it + self.thread.join(5) + if self.thread.isAlive(): + self.process.kill() + self.thread.join() + + self.output = self.output.rstrip() + self.status = self.process.poll() + + self.log.debug("Command '%s' returned %d as exit code." % (self.cmd, self.status)) + # logging the complete output is insane + # bitbake -e output is really big + # and makes the log file useless + if self.status: + lout = "\n".join(self.output.splitlines()[-20:]) + self.log.debug("Last 20 lines:\n%s" % lout) + + +class Result(object): + pass + +def runCmd(command, ignore_status=False, timeout=None, **options): + + result = Result() + + cmd = Command(command, timeout=timeout, **options) + cmd.run() + + result.command = command + result.status = cmd.status + result.output = cmd.output + result.pid = cmd.process.pid + + if result.status and not ignore_status: + raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, result.status, result.output)) + + return result + + +def bitbake(command, ignore_status=False, timeout=None, **options): + if isinstance(command, basestring): + cmd = "bitbake " + command + else: + cmd = [ "bitbake" ] + command + + return runCmd(cmd, ignore_status, timeout, **options) + + +def get_bb_env(target=None): + if target: + return runCmd("bitbake -e %s" % target).output + else: + return runCmd("bitbake -e").output + +def get_bb_var(var, target=None): + val = None + bbenv = get_bb_env(target) + for line in bbenv.splitlines(): + if line.startswith(var + "="): + val = line.split('=')[1] + val = val.replace('\"','') + break + return val + +def get_test_layer(): + layers = get_bb_var("BBLAYERS").split() + testlayer = None + for l in layers: + if "/meta-selftest" in l and os.path.isdir(l): + testlayer = l + break + return testlayer diff --git a/meta/lib/oeqa/utils/decorators.py b/meta/lib/oeqa/utils/decorators.py new file mode 100644 index 0000000000..b99da8d76d --- /dev/null +++ b/meta/lib/oeqa/utils/decorators.py @@ -0,0 +1,50 @@ +# Copyright (C) 2013 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# Some custom decorators that can be used by unittests +# Most useful is skipUnlessPassed which can be used for +# creating dependecies between two test methods. + +from oeqa.oetest import * + +class skipIfFailure(object): + + def __init__(self,testcase): + self.testcase = testcase + + def __call__(self,f): + def wrapped_f(*args): + if self.testcase in (oeTest.testFailures or oeTest.testErrors): + raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) + return f(*args) + wrapped_f.__name__ = f.__name__ + return wrapped_f + +class skipIfSkipped(object): + + def __init__(self,testcase): + self.testcase = testcase + + def __call__(self,f): + def wrapped_f(*args): + if self.testcase in oeTest.testSkipped: + raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) + return f(*args) + wrapped_f.__name__ = f.__name__ + return wrapped_f + +class skipUnlessPassed(object): + + def __init__(self,testcase): + self.testcase = testcase + + def __call__(self,f): + def wrapped_f(*args): + if self.testcase in oeTest.testSkipped or \ + self.testcase in oeTest.testFailures or \ + self.testcase in oeTest.testErrors: + raise unittest.SkipTest("Testcase dependency not met: %s" % self.testcase) + return f(*args) + wrapped_f.__name__ = f.__name__ + return wrapped_f diff --git a/meta/lib/oeqa/utils/ftools.py b/meta/lib/oeqa/utils/ftools.py new file mode 100644 index 0000000000..64ebe3d217 --- /dev/null +++ b/meta/lib/oeqa/utils/ftools.py @@ -0,0 +1,27 @@ +import os +import re + +def write_file(path, data): + wdata = data.rstrip() + "\n" + with open(path, "w") as f: + f.write(wdata) + +def append_file(path, data): + wdata = data.rstrip() + "\n" + with open(path, "a") as f: + f.write(wdata) + +def read_file(path): + data = None + with open(path) as f: + data = f.read() + return data + +def remove_from_file(path, data): + lines = read_file(path).splitlines() + rmdata = data.strip().splitlines() + for l in rmdata: + for c in range(0, lines.count(l)): + i = lines.index(l) + del(lines[i]) + write_file(path, "\n".join(lines)) diff --git a/meta/lib/oeqa/utils/httpserver.py b/meta/lib/oeqa/utils/httpserver.py new file mode 100644 index 0000000000..f161a1bddd --- /dev/null +++ b/meta/lib/oeqa/utils/httpserver.py @@ -0,0 +1,33 @@ +import SimpleHTTPServer +import multiprocessing +import os + +class HTTPServer(SimpleHTTPServer.BaseHTTPServer.HTTPServer): + + def server_start(self, root_dir): + os.chdir(root_dir) + self.serve_forever() + +class HTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + + def log_message(self, format_str, *args): + pass + +class HTTPService(object): + + def __init__(self, root_dir, host=''): + self.root_dir = root_dir + self.host = host + self.port = 0 + + def start(self): + self.server = HTTPServer((self.host, self.port), HTTPRequestHandler) + if self.port == 0: + self.port = self.server.server_port + self.process = multiprocessing.Process(target=self.server.server_start, args=[self.root_dir]) + self.process.start() + + def stop(self): + self.server.server_close() + self.process.terminate() + self.process.join() diff --git a/meta/lib/oeqa/utils/qemurunner.py b/meta/lib/oeqa/utils/qemurunner.py new file mode 100644 index 0000000000..f1a7e24ab7 --- /dev/null +++ b/meta/lib/oeqa/utils/qemurunner.py @@ -0,0 +1,237 @@ +# Copyright (C) 2013 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# This module provides a class for starting qemu images using runqemu. +# It's used by testimage.bbclass. + +import subprocess +import os +import time +import signal +import re +import socket +import select +import bb + +class QemuRunner: + + def __init__(self, machine, rootfs, display, tmpdir, deploy_dir_image, logfile, boottime): + + # Popen object for runqemu + self.runqemu = None + # pid of the qemu process that runqemu will start + self.qemupid = None + # target ip - from the command line + self.ip = None + # host ip - where qemu is running + self.server_ip = None + + self.machine = machine + self.rootfs = rootfs + self.display = display + self.tmpdir = tmpdir + self.deploy_dir_image = deploy_dir_image + self.logfile = logfile + self.boottime = boottime + + self.runqemutime = 60 + + self.create_socket() + + + def create_socket(self): + + self.bootlog = '' + self.qemusock = None + + try: + self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.server_socket.setblocking(0) + self.server_socket.bind(("127.0.0.1",0)) + self.server_socket.listen(2) + self.serverport = self.server_socket.getsockname()[1] + bb.note("Created listening socket for qemu serial console on: 127.0.0.1:%s" % self.serverport) + except socket.error, msg: + self.server_socket.close() + bb.fatal("Failed to create listening socket: %s" %msg[1]) + + + def log(self, msg): + if self.logfile: + with open(self.logfile, "a") as f: + f.write("%s" % msg) + + def start(self, qemuparams = None): + + if self.display: + os.environ["DISPLAY"] = self.display + else: + bb.error("To start qemu I need a X desktop, please set DISPLAY correctly (e.g. DISPLAY=:1)") + return False + if not os.path.exists(self.rootfs): + bb.error("Invalid rootfs %s" % self.rootfs) + return False + if not os.path.exists(self.tmpdir): + bb.error("Invalid TMPDIR path %s" % self.tmpdir) + return False + else: + os.environ["OE_TMPDIR"] = self.tmpdir + if not os.path.exists(self.deploy_dir_image): + bb.error("Invalid DEPLOY_DIR_IMAGE path %s" % self.deploy_dir_image) + return False + else: + os.environ["DEPLOY_DIR_IMAGE"] = self.deploy_dir_image + + # Set this flag so that Qemu doesn't do any grabs as SDL grabs interact + # badly with screensavers. + os.environ["QEMU_DONT_GRAB"] = "1" + self.qemuparams = 'bootparams="console=tty1 console=ttyS0,115200n8" qemuparams="-serial tcp:127.0.0.1:%s"' % self.serverport + if qemuparams: + self.qemuparams = self.qemuparams[:-1] + " " + qemuparams + " " + '\"' + + launch_cmd = 'runqemu %s %s %s' % (self.machine, self.rootfs, self.qemuparams) + self.runqemu = subprocess.Popen(launch_cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,preexec_fn=os.setpgrp) + + bb.note("runqemu started, pid is %s" % self.runqemu.pid) + bb.note("waiting at most %s seconds for qemu pid" % self.runqemutime) + endtime = time.time() + self.runqemutime + while not self.is_alive() and time.time() < endtime: + time.sleep(1) + + if self.is_alive(): + bb.note("qemu started - qemu procces pid is %s" % self.qemupid) + cmdline = '' + with open('/proc/%s/cmdline' % self.qemupid) as p: + cmdline = p.read() + ips = re.findall("((?:[0-9]{1,3}\.){3}[0-9]{1,3})", cmdline.split("ip=")[1]) + if not ips or len(ips) != 3: + bb.note("Couldn't get ip from qemu process arguments! Here is the qemu command line used: %s" % cmdline) + self.stop() + return False + else: + self.ip = ips[0] + self.server_ip = ips[1] + bb.note("Target IP: %s" % self.ip) + bb.note("Server IP: %s" % self.server_ip) + bb.note("Waiting at most %d seconds for login banner" % self.boottime ) + endtime = time.time() + self.boottime + socklist = [self.server_socket] + reachedlogin = False + stopread = False + while time.time() < endtime and not stopread: + sread, swrite, serror = select.select(socklist, [], [], 5) + for sock in sread: + if sock is self.server_socket: + self.qemusock, addr = self.server_socket.accept() + self.qemusock.setblocking(0) + socklist.append(self.qemusock) + socklist.remove(self.server_socket) + bb.note("Connection from %s:%s" % addr) + else: + data = sock.recv(1024) + if data: + self.log(data) + self.bootlog += data + if re.search("qemu.* login:", self.bootlog): + stopread = True + reachedlogin = True + bb.note("Reached login banner") + else: + socklist.remove(sock) + sock.close() + stopread = True + + if not reachedlogin: + bb.note("Target didn't reached login boot in %d seconds" % self.boottime) + lines = "\n".join(self.bootlog.splitlines()[-5:]) + bb.note("Last 5 lines of text:\n%s" % lines) + bb.note("Check full boot log: %s" % self.logfile) + self.stop() + return False + else: + bb.note("Qemu pid didn't appeared in %s seconds" % self.runqemutime) + output = self.runqemu.stdout + self.stop() + bb.note("Output from runqemu:\n%s" % output.read()) + return False + + return self.is_alive() + + def stop(self): + + if self.runqemu: + bb.note("Sending SIGTERM to runqemu") + os.killpg(self.runqemu.pid, signal.SIGTERM) + endtime = time.time() + self.runqemutime + while self.runqemu.poll() is None and time.time() < endtime: + time.sleep(1) + if self.runqemu.poll() is None: + bb.note("Sending SIGKILL to runqemu") + os.killpg(self.runqemu.pid, signal.SIGKILL) + self.runqemu = None + if self.server_socket: + self.server_socket.close() + self.server_socket = None + self.qemupid = None + self.ip = None + + def restart(self, qemuparams = None): + bb.note("Restarting qemu process") + if self.runqemu.poll() is None: + self.stop() + self.create_socket() + if self.start(qemuparams): + return True + return False + + def is_alive(self): + qemu_child = self.find_child(str(self.runqemu.pid)) + if qemu_child: + self.qemupid = qemu_child[0] + if os.path.exists("/proc/" + str(self.qemupid)): + return True + return False + + def find_child(self,parent_pid): + # + # Walk the process tree from the process specified looking for a qemu-system. Return its [pid'cmd] + # + ps = subprocess.Popen(['ps', 'axww', '-o', 'pid,ppid,command'], stdout=subprocess.PIPE).communicate()[0] + processes = ps.split('\n') + nfields = len(processes[0].split()) - 1 + pids = {} + commands = {} + for row in processes[1:]: + data = row.split(None, nfields) + if len(data) != 3: + continue + if data[1] not in pids: + pids[data[1]] = [] + + pids[data[1]].append(data[0]) + commands[data[0]] = data[2] + + if parent_pid not in pids: + return [] + + parents = [] + newparents = pids[parent_pid] + while newparents: + next = [] + for p in newparents: + if p in pids: + for n in pids[p]: + if n not in parents and n not in next: + next.append(n) + if p not in parents: + parents.append(p) + newparents = next + #print "Children matching %s:" % str(parents) + for p in parents: + # Need to be careful here since runqemu-internal runs "ldd qemu-system-xxxx" + # Also, old versions of ldd (2.11) run "LD_XXXX qemu-system-xxxx" + basecmd = commands[p].split()[0] + basecmd = os.path.basename(basecmd) + if "qemu-system" in basecmd and "-serial tcp" in commands[p]: + return [int(p),commands[p]] diff --git a/meta/lib/oeqa/utils/sshcontrol.py b/meta/lib/oeqa/utils/sshcontrol.py new file mode 100644 index 0000000000..d355d5e8e9 --- /dev/null +++ b/meta/lib/oeqa/utils/sshcontrol.py @@ -0,0 +1,127 @@ +# Copyright (C) 2013 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# Provides a class for setting up ssh connections, +# running commands and copying files to/from a target. +# It's used by testimage.bbclass and tests in lib/oeqa/runtime. + +import subprocess +import time +import os +import select + + +class SSHProcess(object): + def __init__(self, **options): + + self.defaultopts = { + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + "stdin": None, + "shell": False, + "bufsize": -1, + "preexec_fn": os.setsid, + } + self.options = dict(self.defaultopts) + self.options.update(options) + self.status = None + self.output = None + self.process = None + self.starttime = None + + def run(self, command, timeout=None): + self.starttime = time.time() + output = '' + self.process = subprocess.Popen(command, **self.options) + if timeout: + endtime = self.starttime + timeout + eof = False + while time.time() < endtime and not eof: + if select.select([self.process.stdout], [], [], 5)[0] != []: + data = os.read(self.process.stdout.fileno(), 1024) + if not data: + self.process.stdout.close() + eof = True + else: + output += data + endtime = time.time() + timeout + + # process hasn't returned yet + if not eof: + self.process.terminate() + time.sleep(5) + try: + self.process.kill() + except OSError: + pass + output += "\nProcess killed - no output for %d seconds. Total running time: %d seconds." % (timeout, time.time() - self.starttime) + else: + output = self.process.communicate()[0] + + self.status = self.process.wait() + self.output = output.rstrip() + return (self.status, self.output) + + +class SSHControl(object): + def __init__(self, ip, logfile=None, timeout=300, user='root', port=None): + self.ip = ip + self.defaulttimeout = timeout + self.ignore_status = True + self.logfile = logfile + self.user = user + self.ssh_options = [ + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'StrictHostKeyChecking=no', + '-o', 'LogLevel=ERROR' + ] + self.ssh = ['ssh', '-l', self.user ] + self.ssh_options + self.scp = ['scp'] + self.ssh_options + if port: + self.ssh = self.ssh + [ '-p', port ] + self.scp = self.scp + [ '-P', port ] + + def log(self, msg): + if self.logfile: + with open(self.logfile, "a") as f: + f.write("%s\n" % msg) + + def _internal_run(self, command, timeout=None, ignore_status = True): + self.log("[Running]$ %s" % " ".join(command)) + + proc = SSHProcess() + status, output = proc.run(command, timeout) + + self.log("%s" % output) + self.log("[Command returned '%d' after %.2f seconds]" % (status, time.time() - proc.starttime)) + + if status and not ignore_status: + raise AssertionError("Command '%s' returned non-zero exit status %d:\n%s" % (command, status, output)) + + return (status, output) + + def run(self, command, timeout=None): + """ + command - ssh command to run + timeout= - kill command if there is no output after seconds + timeout=None - kill command if there is no output after a default value seconds + timeout=0 - no timeout, let command run until it returns + """ + + # We need to source /etc/profile for a proper PATH on the target + command = self.ssh + [self.ip, ' . /etc/profile; ' + command] + + if timeout is None: + return self._internal_run(command, self.defaulttimeout, self.ignore_status) + if timeout == 0: + return self._internal_run(command, None, self.ignore_status) + return self._internal_run(command, timeout, self.ignore_status) + + def copy_to(self, localpath, remotepath): + command = self.scp + [localpath, '%s@%s:%s' % (self.user, self.ip, remotepath)] + return self._internal_run(command, ignore_status=False) + + def copy_from(self, remotepath, localpath): + command = self.scp + ['%s@%s:%s' % (self.user, self.ip, remotepath), localpath] + return self._internal_run(command, ignore_status=False) diff --git a/meta/lib/oeqa/utils/targetbuild.py b/meta/lib/oeqa/utils/targetbuild.py new file mode 100644 index 0000000000..32296762c0 --- /dev/null +++ b/meta/lib/oeqa/utils/targetbuild.py @@ -0,0 +1,68 @@ +# Copyright (C) 2013 Intel Corporation +# +# Released under the MIT license (see COPYING.MIT) + +# Provides a class for automating build tests for projects + +import os +import re +import subprocess + + +class TargetBuildProject(): + + def __init__(self, target, d, uri, foldername=None): + self.target = target + self.d = d + self.uri = uri + self.targetdir = "~/" + self.archive = os.path.basename(uri) + self.localarchive = "/tmp/" + self.archive + self.fname = re.sub(r'.tar.bz2|tar.gz$', '', self.archive) + if foldername: + self.fname = foldername + + def download_archive(self): + + exportvars = ['HTTP_PROXY', 'http_proxy', + 'HTTPS_PROXY', 'https_proxy', + 'FTP_PROXY', 'ftp_proxy', + 'FTPS_PROXY', 'ftps_proxy', + 'NO_PROXY', 'no_proxy', + 'ALL_PROXY', 'all_proxy', + 'SOCKS5_USER', 'SOCKS5_PASSWD'] + + cmd = '' + for var in exportvars: + val = self.d.getVar(var, True) + if val: + cmd = 'export ' + var + '=\"%s\"; %s' % (val, cmd) + + cmd = cmd + "wget -O %s %s" % (self.localarchive, self.uri) + subprocess.check_call(cmd, shell=True) + + (status, output) = self.target.copy_to(self.localarchive, self.targetdir) + if status != 0: + raise Exception("Failed to copy archive to target, output: %s" % output) + + (status, output) = self.target.run('tar xf %s%s -C %s' % (self.targetdir, self.archive, self.targetdir)) + if status != 0: + raise Exception("Failed to extract archive, output: %s" % output) + + #Change targetdir to project folder + self.targetdir = self.targetdir + self.fname + + # The timeout parameter of target.run is set to 0 to make the ssh command + # run with no timeout. + def run_configure(self): + return self.target.run('cd %s; ./configure' % self.targetdir, 0)[0] + + def run_make(self): + return self.target.run('cd %s; make' % self.targetdir, 0)[0] + + def run_install(self): + return self.target.run('cd %s; make install' % self.targetdir, 0)[0] + + def clean(self): + self.target.run('rm -rf %s' % self.targetdir) + subprocess.call('rm -f %s' % self.localarchive, shell=True) -- cgit v1.2.3-54-g00ecf