From 731acba417b627f6d3ef7e856cbe36491aa70d22 Mon Sep 17 00:00:00 2001 From: Stefan Stanacar Date: Mon, 3 Feb 2014 21:22:29 +0200 Subject: testimage: add ability to export tests Add the ability to export the tests so that they can run independently of the build system, as is required if you want to be able to hand the test execution off to a scheduler. Booting/deployment of the target is still handled by the build system, as before, only the execution of the tests happens outside of the build system. Tests exported are the ones defined in TEST_SUITES. No tests have been changed as interesting parts of the data store have been exported and tests can continue to query them as before. Small adjustments were made for a couple of oeqa modules though. [YOCTO #5613] (From OE-Core rev: 155dd52e0f707e06f50756584a50f744ba6b7844) Signed-off-by: Stefan Stanacar Signed-off-by: Saul Wold Signed-off-by: Richard Purdie --- meta/classes/testimage.bbclass | 107 +++++++++++++++++++++++++++++++++++------ meta/lib/oeqa/oetest.py | 17 +++---- 2 files changed, 98 insertions(+), 26 deletions(-) (limited to 'meta') diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass index 4777e140dc..75ab716270 100644 --- a/meta/classes/testimage.bbclass +++ b/meta/classes/testimage.bbclass @@ -25,15 +25,16 @@ TEST_LOG_DIR ?= "${WORKDIR}/testimage" +TEST_EXPORT_DIR ?= "${TMPDIR}/testimage/${PN}" +TEST_EXPORT_ONLY ?= "0" + DEFAULT_TEST_SUITES = "ping auto" DEFAULT_TEST_SUITES_pn-core-image-minimal = "ping" DEFAULT_TEST_SUITES_pn-core-image-sato = "ping ssh df connman syslog xorg scp vnc date rpm smart dmesg python" DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "ping ssh df connman syslog xorg scp vnc date perl ldd gcc rpm smart kernelmodule dmesg python" - TEST_SUITES ?= "${DEFAULT_TEST_SUITES}" TEST_QEMUBOOT_TIMEOUT ?= "1000" - TEST_TARGET ?= "qemu" TEST_TARGET_IP ?= "" TEST_SERVER_IP ?= "" @@ -85,6 +86,71 @@ def get_tests_list(d): return testslist + +def exportTests(d,tc): + import json + import shutil + import pkgutil + + exportpath = d.getVar("TEST_EXPORT_DIR", True) + + savedata = {} + savedata["d"] = {} + savedata["target"] = {} + for key in tc.__dict__: + # special cases + if key != "d" and key != "target": + savedata[key] = getattr(tc, key) + savedata["target"]["ip"] = tc.target.ip or d.getVar("TEST_TARGET_IP", True) + savedata["target"]["server_ip"] = tc.target.server_ip or d.getVar("TEST_SERVER_IP", True) + + keys = [ key for key in d.keys() if not key.startswith("_") and not key.startswith("BB") \ + and not key.startswith("B_pn") and not key.startswith("do_") and not d.getVarFlag(key, "func")] + for key in keys: + try: + savedata["d"][key] = d.getVar(key, True) + except bb.data_smart.ExpansionError: + # we don't care about those anyway + pass + + with open(os.path.join(exportpath, "testdata.json"), "w") as f: + json.dump(savedata, f, skipkeys=True, indent=4, sort_keys=True) + + # now start copying files + # we'll basically copy everything under meta/lib/oeqa, with these exceptions + # - oeqa/targetcontrol.py - not needed + # - oeqa/selftest - something else + # That means: + # - all tests from oeqa/runtime defined in TEST_SUITES (including from other layers) + # - the contents of oeqa/utils and oeqa/runtime/files + # - oeqa/oetest.py and oeqa/runexport.py (this will get copied to exportpath not exportpath/oeqa) + # - __init__.py files + bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/runtime/files")) + bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/utils")) + # copy test modules, this should cover tests in other layers too + for t in tc.testslist: + mod = pkgutil.get_loader(t) + shutil.copy2(mod.filename, os.path.join(exportpath, "oeqa/runtime")) + # copy __init__.py files + oeqadir = pkgutil.get_loader("oeqa").filename + shutil.copy2(os.path.join(oeqadir, "__init__.py"), os.path.join(exportpath, "oeqa")) + shutil.copy2(os.path.join(oeqadir, "runtime/__init__.py"), os.path.join(exportpath, "oeqa/runtime")) + # copy oeqa/oetest.py and oeqa/runexported.py + shutil.copy2(os.path.join(oeqadir, "oetest.py"), os.path.join(exportpath, "oeqa")) + shutil.copy2(os.path.join(oeqadir, "runexported.py"), exportpath) + # copy oeqa/utils/*.py + for root, dirs, files in os.walk(os.path.join(oeqadir, "utils")): + for f in files: + if f.endswith(".py"): + shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/utils")) + # copy oeqa/runtime/files/* + for root, dirs, files in os.walk(os.path.join(oeqadir, "runtime/files")): + for f in files: + shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/runtime/files")) + + bb.plain("Exported tests to: %s" % exportpath) + + def testimage_main(d): import unittest import os @@ -94,7 +160,11 @@ def testimage_main(d): from oeqa.targetcontrol import get_target_controller pn = d.getVar("PN", True) + export = oe.utils.conditional("TEST_EXPORT_ONLY", "1", True, False, d) bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True)) + if export: + bb.utils.remove(d.getVar("TEST_EXPORT_DIR", True), recurse=True) + bb.utils.mkdirhier(d.getVar("TEST_EXPORT_DIR", True)) # tests in TEST_SUITES become required tests # they won't be skipped even if they aren't suitable for a image (like xorg for minimal) @@ -105,13 +175,18 @@ def testimage_main(d): # the robot dance target = get_target_controller(d) - class TestContext: + class TestContext(object): def __init__(self): self.d = d self.testslist = testslist self.testsrequired = testsrequired self.filesdir = os.path.join(os.path.dirname(os.path.abspath(oeqa.runtime.__file__)),"files") self.target = target + self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split() + self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split() + manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + ".manifest") + with open(manifest) as f: + self.pkgmanifest = f.read() # test context tc = TestContext() @@ -129,19 +204,21 @@ def testimage_main(d): try: target.start() - # run tests and get the results - starttime = time.time() - result = runTests(tc) - stoptime = time.time() - if result.wasSuccessful(): - bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime)) - msg = "%s - OK - All required tests passed" % pn - skipped = len(result.skipped) - if skipped: - msg += " (skipped=%d)" % skipped - bb.plain(msg) + if export: + exportTests(d,tc) else: - raise bb.build.FuncFailed("%s - FAILED - check the task log and the ssh log" % pn ) + starttime = time.time() + result = runTests(tc) + stoptime = time.time() + if result.wasSuccessful(): + bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime)) + msg = "%s - OK - All required tests passed" % pn + skipped = len(result.skipped) + if skipped: + msg += " (skipped=%d)" % skipped + bb.plain(msg) + else: + raise bb.build.FuncFailed("%s - FAILED - check the task log and the ssh log" % pn ) finally: target.stop() diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py index 23a3e5d69f..0db6cb80a9 100644 --- a/meta/lib/oeqa/oetest.py +++ b/meta/lib/oeqa/oetest.py @@ -10,8 +10,6 @@ import os, re, mmap import unittest import inspect -import bb -from oeqa.utils.sshcontrol import SSHControl def loadTests(tc): @@ -31,15 +29,14 @@ def loadTests(tc): def runTests(tc): suite = loadTests(tc) - bb.note("Test modules %s" % tc.testslist) - bb.note("Found %s tests" % suite.countTestCases()) + print("Test modules %s" % tc.testslist) + print("Found %s tests" % suite.countTestCases()) runner = unittest.TextTestRunner(verbosity=2) result = runner.run(suite) return result - class oeTest(unittest.TestCase): longMessage = True @@ -60,18 +57,16 @@ class oeTest(unittest.TestCase): @classmethod def hasPackage(self, pkg): - manifest = os.path.join(oeTest.tc.d.getVar("DEPLOY_DIR_IMAGE", True), oeTest.tc.d.getVar("IMAGE_LINK_NAME", True) + ".manifest") - with open(manifest) as f: - data = f.read() - if re.search(pkg, data): + + if re.search(pkg, oeTest.tc.pkgmanifest): return True return False @classmethod def hasFeature(self,feature): - if feature in oeTest.tc.d.getVar("IMAGE_FEATURES", True).split() or \ - feature in oeTest.tc.d.getVar("DISTRO_FEATURES", True).split(): + if feature in oeTest.tc.imagefeatures or \ + feature in oeTest.tc.distrofeatures: return True else: return False -- cgit v1.2.3-54-g00ecf