diff options
author | Alexandru DAMIAN <alexandru.damian@intel.com> | 2015-07-30 19:25:18 +0300 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2015-08-01 11:27:55 +0100 |
commit | 1da5e0a793adee9248ecce927952ba8c00bdcb0e (patch) | |
tree | f4cfb5f9e7db9c346b94b8eb1ef2aabae7e67864 /bitbake/lib/toaster/contrib/tts/runner.py | |
parent | e6419ccd93e87e730c0a6dba7a15e0ba2b986fff (diff) | |
download | poky-1da5e0a793adee9248ecce927952ba8c00bdcb0e.tar.gz |
bitbake: toaster: tts: execute tests in numeric order
As the tests are verifying different type of the functionality,
it is usually the case that a failing early test will completely
make the subsequent tests failing, e.g. if the system cannot
start due to a bug, there is little point in testing other
functions.
In order to prevent uneeded test runs, and to generate repeatable
test patterns, the test cases have now a numeric order in the
class name (e.g. Test01XXX). The tests are executed in this order,
and the first test failing will stop the test run.
(Bitbake rev: 639c46a08e524902018e28367fcb4e26362cd3e3)
Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster/contrib/tts/runner.py')
-rwxr-xr-x | bitbake/lib/toaster/contrib/tts/runner.py | 11 |
1 files changed, 7 insertions, 4 deletions
diff --git a/bitbake/lib/toaster/contrib/tts/runner.py b/bitbake/lib/toaster/contrib/tts/runner.py index 6fc3e77523..bed665196e 100755 --- a/bitbake/lib/toaster/contrib/tts/runner.py +++ b/bitbake/lib/toaster/contrib/tts/runner.py | |||
@@ -31,7 +31,7 @@ from __future__ import print_function | |||
31 | import sys, os | 31 | import sys, os |
32 | import unittest, importlib | 32 | import unittest, importlib |
33 | import logging, pprint, json | 33 | import logging, pprint, json |
34 | 34 | import re | |
35 | from shellutils import ShellCmdException, mkdirhier, run_shell_cmd | 35 | from shellutils import ShellCmdException, mkdirhier, run_shell_cmd |
36 | 36 | ||
37 | import config | 37 | import config |
@@ -121,13 +121,16 @@ def execute_tests(dir_under_test, testname): | |||
121 | # pylint: disable=broad-except | 121 | # pylint: disable=broad-except |
122 | # we disable the broad-except because we want to actually catch all possible exceptions | 122 | # we disable the broad-except because we want to actually catch all possible exceptions |
123 | try: | 123 | try: |
124 | # sorting the tests by the numeric order in the class name | ||
125 | tests = sorted(tests, key=lambda x: int(re.search(r"[0-9]+", x[1]).group(0))) | ||
124 | config.logger.debug("Discovered test clases: %s", pprint.pformat(tests)) | 126 | config.logger.debug("Discovered test clases: %s", pprint.pformat(tests)) |
125 | unittest.installHandler() | 127 | unittest.installHandler() |
126 | suite = unittest.TestSuite() | 128 | suite = unittest.TestSuite() |
127 | loader = unittest.TestLoader() | 129 | loader = unittest.TestLoader() |
128 | result = unittest.TestResult() | 130 | result = unittest.TestResult() |
129 | for module_file, name in tests: | 131 | result.failfast = True |
130 | suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, name))) | 132 | for module_file, test_name in tests: |
133 | suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, test_name))) | ||
131 | config.logger.info("Running %d test(s)", suite.countTestCases()) | 134 | config.logger.info("Running %d test(s)", suite.countTestCases()) |
132 | suite.run(result) | 135 | suite.run(result) |
133 | 136 | ||
@@ -202,7 +205,7 @@ def main(): | |||
202 | config.TESTDIR = testdir # we let tests know where to run | 205 | config.TESTDIR = testdir # we let tests know where to run |
203 | 206 | ||
204 | # ensure that the test dir only contains no *.pyc leftovers | 207 | # ensure that the test dir only contains no *.pyc leftovers |
205 | run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \;" % testdir) | 208 | run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \\;" % testdir) |
206 | 209 | ||
207 | no_failures = execute_tests(testdir, options.singletest) | 210 | no_failures = execute_tests(testdir, options.singletest) |
208 | 211 | ||