testimage: add ability to export tests

Add the ability to export the tests so that they can run independently of
the build system, as is required if you want to be able to hand the test
execution off to a scheduler.
Booting/deployment of the target is still handled by the build system,
as before, only the execution of the tests happens outside of the build system.
Tests exported are the ones defined in TEST_SUITES.

No tests have been changed as interesting parts of the data store have been
exported and tests can continue to query them as before. Small adjustments were made
for a couple of oeqa modules though.

[YOCTO #5613]

(From OE-Core rev: 155dd52e0f707e06f50756584a50f744ba6b7844)

Signed-off-by: Stefan Stanacar <stefanx.stanacar@intel.com>
Signed-off-by: Saul Wold <sgw@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 98 insertions, 26 deletions

diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 4777e140dc..75ab716270 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -25,15 +25,16 @@
 
 TEST_LOG_DIR ?= "${WORKDIR}/testimage"
 
+TEST_EXPORT_DIR ?= "${TMPDIR}/testimage/${PN}"
+TEST_EXPORT_ONLY ?= "0"
+
 DEFAULT_TEST_SUITES = "ping auto"
 DEFAULT_TEST_SUITES_pn-core-image-minimal = "ping"
 DEFAULT_TEST_SUITES_pn-core-image-sato = "ping ssh df connman syslog xorg scp vnc date rpm smart dmesg python"
 DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "ping ssh df connman syslog xorg scp vnc date perl ldd gcc rpm smart kernelmodule dmesg python"
-
 TEST_SUITES ?= "${DEFAULT_TEST_SUITES}"
 
 TEST_QEMUBOOT_TIMEOUT ?= "1000"
-
 TEST_TARGET ?= "qemu"
 TEST_TARGET_IP ?= ""
 TEST_SERVER_IP ?= ""
@@ -85,6 +86,71 @@ def get_tests_list(d):
 
     return testslist
 
+
+def exportTests(d,tc):
+    import json
+    import shutil
+    import pkgutil
+
+    exportpath = d.getVar("TEST_EXPORT_DIR", True)
+
+    savedata = {}
+    savedata["d"] = {}
+    savedata["target"] = {}
+    for key in tc.__dict__:
+        # special cases
+        if key != "d" and key != "target":
+            savedata[key] = getattr(tc, key)
+    savedata["target"]["ip"] = tc.target.ip or d.getVar("TEST_TARGET_IP", True)
+    savedata["target"]["server_ip"] = tc.target.server_ip or d.getVar("TEST_SERVER_IP", True)
+
+    keys = [ key for key in d.keys() if not key.startswith("_") and not key.startswith("BB") \
+            and not key.startswith("B_pn") and not key.startswith("do_") and not d.getVarFlag(key, "func")]
+    for key in keys:
+        try:
+            savedata["d"][key] = d.getVar(key, True)
+        except bb.data_smart.ExpansionError:
+            # we don't care about those anyway
+            pass
+
+    with open(os.path.join(exportpath, "testdata.json"), "w") as f:
+            json.dump(savedata, f, skipkeys=True, indent=4, sort_keys=True)
+
+    # now start copying files
+    # we'll basically copy everything under meta/lib/oeqa, with these exceptions
+    #  - oeqa/targetcontrol.py - not needed
+    #  - oeqa/selftest - something else
+    # That means:
+    #   - all tests from oeqa/runtime defined in TEST_SUITES (including from other layers)
+    #   - the contents of oeqa/utils and oeqa/runtime/files
+    #   - oeqa/oetest.py and oeqa/runexport.py (this will get copied to exportpath not exportpath/oeqa)
+    #   - __init__.py files
+    bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/runtime/files"))
+    bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/utils"))
+    # copy test modules, this should cover tests in other layers too
+    for t in tc.testslist:
+        mod = pkgutil.get_loader(t)
+        shutil.copy2(mod.filename, os.path.join(exportpath, "oeqa/runtime"))
+    # copy __init__.py files
+    oeqadir = pkgutil.get_loader("oeqa").filename
+    shutil.copy2(os.path.join(oeqadir, "__init__.py"), os.path.join(exportpath, "oeqa"))
+    shutil.copy2(os.path.join(oeqadir, "runtime/__init__.py"), os.path.join(exportpath, "oeqa/runtime"))
+    # copy oeqa/oetest.py and oeqa/runexported.py
+    shutil.copy2(os.path.join(oeqadir, "oetest.py"), os.path.join(exportpath, "oeqa"))
+    shutil.copy2(os.path.join(oeqadir, "runexported.py"), exportpath)
+    # copy oeqa/utils/*.py
+    for root, dirs, files in os.walk(os.path.join(oeqadir, "utils")):
+        for f in files:
+            if f.endswith(".py"):
+                shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/utils"))
+    # copy oeqa/runtime/files/*
+    for root, dirs, files in os.walk(os.path.join(oeqadir, "runtime/files")):
+        for f in files:
+            shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/runtime/files"))
+
+    bb.plain("Exported tests to: %s" % exportpath)
+
+
 def testimage_main(d):
     import unittest
     import os
@@ -94,7 +160,11 @@ def testimage_main(d):
     from oeqa.targetcontrol import get_target_controller
 
     pn = d.getVar("PN", True)
+    export = oe.utils.conditional("TEST_EXPORT_ONLY", "1", True, False, d)
     bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
+    if export:
+        bb.utils.remove(d.getVar("TEST_EXPORT_DIR", True), recurse=True)
+        bb.utils.mkdirhier(d.getVar("TEST_EXPORT_DIR", True))
 
     # tests in TEST_SUITES become required tests
     # they won't be skipped even if they aren't suitable for a image (like xorg for minimal)
@@ -105,13 +175,18 @@ def testimage_main(d):
     # the robot dance
     target = get_target_controller(d)
 
-    class TestContext:
+    class TestContext(object):
         def __init__(self):
             self.d = d
             self.testslist = testslist
             self.testsrequired = testsrequired
             self.filesdir = os.path.join(os.path.dirname(os.path.abspath(oeqa.runtime.__file__)),"files")
             self.target = target
+            self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split()
+            self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split()
+            manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + ".manifest")
+            with open(manifest) as f:
+                self.pkgmanifest = f.read()
 
     # test context
     tc = TestContext()
@@ -129,19 +204,21 @@ def testimage_main(d):
 
     try:
         target.start()
-        # run tests and get the results
-        starttime = time.time()
-        result = runTests(tc)
-        stoptime = time.time()
-        if result.wasSuccessful():
-            bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime))
-            msg = "%s - OK - All required tests passed" % pn
-            skipped = len(result.skipped)
-            if skipped:
-                msg += " (skipped=%d)" % skipped
-            bb.plain(msg)
+        if export:
+            exportTests(d,tc)
         else:
-            raise bb.build.FuncFailed("%s - FAILED - check the task log and the ssh log" % pn )
+            starttime = time.time()
+            result = runTests(tc)
+            stoptime = time.time()
+            if result.wasSuccessful():
+                bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime))
+                msg = "%s - OK - All required tests passed" % pn
+                skipped = len(result.skipped)
+                if skipped:
+                    msg += " (skipped=%d)" % skipped
+                bb.plain(msg)
+            else:
+                raise bb.build.FuncFailed("%s - FAILED - check the task log and the ssh log" % pn )
     finally:
         target.stop()
 
diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py
index 23a3e5d69f..0db6cb80a9 100644
--- a/meta/lib/oeqa/oetest.py
+++ b/meta/lib/oeqa/oetest.py
@@ -10,8 +10,6 @@
 import os, re, mmap
 import unittest
 import inspect
-import bb
-from oeqa.utils.sshcontrol import SSHControl
 
 
 def loadTests(tc):
@@ -31,15 +29,14 @@ def loadTests(tc):
 def runTests(tc):
 
     suite = loadTests(tc)
-    bb.note("Test modules  %s" % tc.testslist)
-    bb.note("Found %s tests" % suite.countTestCases())
+    print("Test modules  %s" % tc.testslist)
+    print("Found %s tests" % suite.countTestCases())
     runner = unittest.TextTestRunner(verbosity=2)
     result = runner.run(suite)
 
     return result
 
 
-
 class oeTest(unittest.TestCase):
 
     longMessage = True
@@ -60,18 +57,16 @@ class oeTest(unittest.TestCase):
 
     @classmethod
     def hasPackage(self, pkg):
-        manifest = os.path.join(oeTest.tc.d.getVar("DEPLOY_DIR_IMAGE", True), oeTest.tc.d.getVar("IMAGE_LINK_NAME", True) + ".manifest")
-        with open(manifest) as f:
-            data = f.read()
-        if re.search(pkg, data):
+
+        if re.search(pkg, oeTest.tc.pkgmanifest):
             return True
         return False
 
     @classmethod
     def hasFeature(self,feature):
 
-        if feature in oeTest.tc.d.getVar("IMAGE_FEATURES", True).split() or \
-                feature in oeTest.tc.d.getVar("DISTRO_FEATURES", True).split():
+        if feature in oeTest.tc.imagefeatures or \
+                feature in oeTest.tc.distrofeatures:
             return True
         else:
             return False