summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorStefan Stanacar <stefanx.stanacar@intel.com>2014-02-03 21:22:29 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-02-09 09:40:00 +0000
commit731acba417b627f6d3ef7e856cbe36491aa70d22 (patch)
tree3078a3b52f1bb357e7961d86a0184dcb92d28896 /meta
parentda1331b4ceca1729fa33818b89a2203ea8301103 (diff)
downloadpoky-731acba417b627f6d3ef7e856cbe36491aa70d22.tar.gz
testimage: add ability to export tests
Add the ability to export the tests so that they can run independently of the build system, as is required if you want to be able to hand the test execution off to a scheduler. Booting/deployment of the target is still handled by the build system, as before, only the execution of the tests happens outside of the build system. Tests exported are the ones defined in TEST_SUITES. No tests have been changed as interesting parts of the data store have been exported and tests can continue to query them as before. Small adjustments were made for a couple of oeqa modules though. [YOCTO #5613] (From OE-Core rev: 155dd52e0f707e06f50756584a50f744ba6b7844) Signed-off-by: Stefan Stanacar <stefanx.stanacar@intel.com> Signed-off-by: Saul Wold <sgw@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/testimage.bbclass107
-rw-r--r--meta/lib/oeqa/oetest.py17
2 files changed, 98 insertions, 26 deletions
diff --git a/meta/classes/testimage.bbclass b/meta/classes/testimage.bbclass
index 4777e140dc..75ab716270 100644
--- a/meta/classes/testimage.bbclass
+++ b/meta/classes/testimage.bbclass
@@ -25,15 +25,16 @@
25 25
26TEST_LOG_DIR ?= "${WORKDIR}/testimage" 26TEST_LOG_DIR ?= "${WORKDIR}/testimage"
27 27
28TEST_EXPORT_DIR ?= "${TMPDIR}/testimage/${PN}"
29TEST_EXPORT_ONLY ?= "0"
30
28DEFAULT_TEST_SUITES = "ping auto" 31DEFAULT_TEST_SUITES = "ping auto"
29DEFAULT_TEST_SUITES_pn-core-image-minimal = "ping" 32DEFAULT_TEST_SUITES_pn-core-image-minimal = "ping"
30DEFAULT_TEST_SUITES_pn-core-image-sato = "ping ssh df connman syslog xorg scp vnc date rpm smart dmesg python" 33DEFAULT_TEST_SUITES_pn-core-image-sato = "ping ssh df connman syslog xorg scp vnc date rpm smart dmesg python"
31DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "ping ssh df connman syslog xorg scp vnc date perl ldd gcc rpm smart kernelmodule dmesg python" 34DEFAULT_TEST_SUITES_pn-core-image-sato-sdk = "ping ssh df connman syslog xorg scp vnc date perl ldd gcc rpm smart kernelmodule dmesg python"
32
33TEST_SUITES ?= "${DEFAULT_TEST_SUITES}" 35TEST_SUITES ?= "${DEFAULT_TEST_SUITES}"
34 36
35TEST_QEMUBOOT_TIMEOUT ?= "1000" 37TEST_QEMUBOOT_TIMEOUT ?= "1000"
36
37TEST_TARGET ?= "qemu" 38TEST_TARGET ?= "qemu"
38TEST_TARGET_IP ?= "" 39TEST_TARGET_IP ?= ""
39TEST_SERVER_IP ?= "" 40TEST_SERVER_IP ?= ""
@@ -85,6 +86,71 @@ def get_tests_list(d):
85 86
86 return testslist 87 return testslist
87 88
89
90def exportTests(d,tc):
91 import json
92 import shutil
93 import pkgutil
94
95 exportpath = d.getVar("TEST_EXPORT_DIR", True)
96
97 savedata = {}
98 savedata["d"] = {}
99 savedata["target"] = {}
100 for key in tc.__dict__:
101 # special cases
102 if key != "d" and key != "target":
103 savedata[key] = getattr(tc, key)
104 savedata["target"]["ip"] = tc.target.ip or d.getVar("TEST_TARGET_IP", True)
105 savedata["target"]["server_ip"] = tc.target.server_ip or d.getVar("TEST_SERVER_IP", True)
106
107 keys = [ key for key in d.keys() if not key.startswith("_") and not key.startswith("BB") \
108 and not key.startswith("B_pn") and not key.startswith("do_") and not d.getVarFlag(key, "func")]
109 for key in keys:
110 try:
111 savedata["d"][key] = d.getVar(key, True)
112 except bb.data_smart.ExpansionError:
113 # we don't care about those anyway
114 pass
115
116 with open(os.path.join(exportpath, "testdata.json"), "w") as f:
117 json.dump(savedata, f, skipkeys=True, indent=4, sort_keys=True)
118
119 # now start copying files
120 # we'll basically copy everything under meta/lib/oeqa, with these exceptions
121 # - oeqa/targetcontrol.py - not needed
122 # - oeqa/selftest - something else
123 # That means:
124 # - all tests from oeqa/runtime defined in TEST_SUITES (including from other layers)
125 # - the contents of oeqa/utils and oeqa/runtime/files
126 # - oeqa/oetest.py and oeqa/runexport.py (this will get copied to exportpath not exportpath/oeqa)
127 # - __init__.py files
128 bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/runtime/files"))
129 bb.utils.mkdirhier(os.path.join(exportpath, "oeqa/utils"))
130 # copy test modules, this should cover tests in other layers too
131 for t in tc.testslist:
132 mod = pkgutil.get_loader(t)
133 shutil.copy2(mod.filename, os.path.join(exportpath, "oeqa/runtime"))
134 # copy __init__.py files
135 oeqadir = pkgutil.get_loader("oeqa").filename
136 shutil.copy2(os.path.join(oeqadir, "__init__.py"), os.path.join(exportpath, "oeqa"))
137 shutil.copy2(os.path.join(oeqadir, "runtime/__init__.py"), os.path.join(exportpath, "oeqa/runtime"))
138 # copy oeqa/oetest.py and oeqa/runexported.py
139 shutil.copy2(os.path.join(oeqadir, "oetest.py"), os.path.join(exportpath, "oeqa"))
140 shutil.copy2(os.path.join(oeqadir, "runexported.py"), exportpath)
141 # copy oeqa/utils/*.py
142 for root, dirs, files in os.walk(os.path.join(oeqadir, "utils")):
143 for f in files:
144 if f.endswith(".py"):
145 shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/utils"))
146 # copy oeqa/runtime/files/*
147 for root, dirs, files in os.walk(os.path.join(oeqadir, "runtime/files")):
148 for f in files:
149 shutil.copy2(os.path.join(root, f), os.path.join(exportpath, "oeqa/runtime/files"))
150
151 bb.plain("Exported tests to: %s" % exportpath)
152
153
88def testimage_main(d): 154def testimage_main(d):
89 import unittest 155 import unittest
90 import os 156 import os
@@ -94,7 +160,11 @@ def testimage_main(d):
94 from oeqa.targetcontrol import get_target_controller 160 from oeqa.targetcontrol import get_target_controller
95 161
96 pn = d.getVar("PN", True) 162 pn = d.getVar("PN", True)
163 export = oe.utils.conditional("TEST_EXPORT_ONLY", "1", True, False, d)
97 bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True)) 164 bb.utils.mkdirhier(d.getVar("TEST_LOG_DIR", True))
165 if export:
166 bb.utils.remove(d.getVar("TEST_EXPORT_DIR", True), recurse=True)
167 bb.utils.mkdirhier(d.getVar("TEST_EXPORT_DIR", True))
98 168
99 # tests in TEST_SUITES become required tests 169 # tests in TEST_SUITES become required tests
100 # they won't be skipped even if they aren't suitable for a image (like xorg for minimal) 170 # they won't be skipped even if they aren't suitable for a image (like xorg for minimal)
@@ -105,13 +175,18 @@ def testimage_main(d):
105 # the robot dance 175 # the robot dance
106 target = get_target_controller(d) 176 target = get_target_controller(d)
107 177
108 class TestContext: 178 class TestContext(object):
109 def __init__(self): 179 def __init__(self):
110 self.d = d 180 self.d = d
111 self.testslist = testslist 181 self.testslist = testslist
112 self.testsrequired = testsrequired 182 self.testsrequired = testsrequired
113 self.filesdir = os.path.join(os.path.dirname(os.path.abspath(oeqa.runtime.__file__)),"files") 183 self.filesdir = os.path.join(os.path.dirname(os.path.abspath(oeqa.runtime.__file__)),"files")
114 self.target = target 184 self.target = target
185 self.imagefeatures = d.getVar("IMAGE_FEATURES", True).split()
186 self.distrofeatures = d.getVar("DISTRO_FEATURES", True).split()
187 manifest = os.path.join(d.getVar("DEPLOY_DIR_IMAGE", True), d.getVar("IMAGE_LINK_NAME", True) + ".manifest")
188 with open(manifest) as f:
189 self.pkgmanifest = f.read()
115 190
116 # test context 191 # test context
117 tc = TestContext() 192 tc = TestContext()
@@ -129,19 +204,21 @@ def testimage_main(d):
129 204
130 try: 205 try:
131 target.start() 206 target.start()
132 # run tests and get the results 207 if export:
133 starttime = time.time() 208 exportTests(d,tc)
134 result = runTests(tc)
135 stoptime = time.time()
136 if result.wasSuccessful():
137 bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime))
138 msg = "%s - OK - All required tests passed" % pn
139 skipped = len(result.skipped)
140 if skipped:
141 msg += " (skipped=%d)" % skipped
142 bb.plain(msg)
143 else: 209 else:
144 raise bb.build.FuncFailed("%s - FAILED - check the task log and the ssh log" % pn ) 210 starttime = time.time()
211 result = runTests(tc)
212 stoptime = time.time()
213 if result.wasSuccessful():
214 bb.plain("%s - Ran %d test%s in %.3fs" % (pn, result.testsRun, result.testsRun != 1 and "s" or "", stoptime - starttime))
215 msg = "%s - OK - All required tests passed" % pn
216 skipped = len(result.skipped)
217 if skipped:
218 msg += " (skipped=%d)" % skipped
219 bb.plain(msg)
220 else:
221 raise bb.build.FuncFailed("%s - FAILED - check the task log and the ssh log" % pn )
145 finally: 222 finally:
146 target.stop() 223 target.stop()
147 224
diff --git a/meta/lib/oeqa/oetest.py b/meta/lib/oeqa/oetest.py
index 23a3e5d69f..0db6cb80a9 100644
--- a/meta/lib/oeqa/oetest.py
+++ b/meta/lib/oeqa/oetest.py
@@ -10,8 +10,6 @@
10import os, re, mmap 10import os, re, mmap
11import unittest 11import unittest
12import inspect 12import inspect
13import bb
14from oeqa.utils.sshcontrol import SSHControl
15 13
16 14
17def loadTests(tc): 15def loadTests(tc):
@@ -31,15 +29,14 @@ def loadTests(tc):
31def runTests(tc): 29def runTests(tc):
32 30
33 suite = loadTests(tc) 31 suite = loadTests(tc)
34 bb.note("Test modules %s" % tc.testslist) 32 print("Test modules %s" % tc.testslist)
35 bb.note("Found %s tests" % suite.countTestCases()) 33 print("Found %s tests" % suite.countTestCases())
36 runner = unittest.TextTestRunner(verbosity=2) 34 runner = unittest.TextTestRunner(verbosity=2)
37 result = runner.run(suite) 35 result = runner.run(suite)
38 36
39 return result 37 return result
40 38
41 39
42
43class oeTest(unittest.TestCase): 40class oeTest(unittest.TestCase):
44 41
45 longMessage = True 42 longMessage = True
@@ -60,18 +57,16 @@ class oeTest(unittest.TestCase):
60 57
61 @classmethod 58 @classmethod
62 def hasPackage(self, pkg): 59 def hasPackage(self, pkg):
63 manifest = os.path.join(oeTest.tc.d.getVar("DEPLOY_DIR_IMAGE", True), oeTest.tc.d.getVar("IMAGE_LINK_NAME", True) + ".manifest") 60
64 with open(manifest) as f: 61 if re.search(pkg, oeTest.tc.pkgmanifest):
65 data = f.read()
66 if re.search(pkg, data):
67 return True 62 return True
68 return False 63 return False
69 64
70 @classmethod 65 @classmethod
71 def hasFeature(self,feature): 66 def hasFeature(self,feature):
72 67
73 if feature in oeTest.tc.d.getVar("IMAGE_FEATURES", True).split() or \ 68 if feature in oeTest.tc.imagefeatures or \
74 feature in oeTest.tc.d.getVar("DISTRO_FEATURES", True).split(): 69 feature in oeTest.tc.distrofeatures:
75 return True 70 return True
76 else: 71 else:
77 return False 72 return False