From 1da5e0a793adee9248ecce927952ba8c00bdcb0e Mon Sep 17 00:00:00 2001 From: Alexandru DAMIAN Date: Thu, 30 Jul 2015 19:25:18 +0300 Subject: bitbake: toaster: tts: execute tests in numeric order As the tests are verifying different type of the functionality, it is usually the case that a failing early test will completely make the subsequent tests failing, e.g. if the system cannot start due to a bug, there is little point in testing other functions. In order to prevent uneeded test runs, and to generate repeatable test patterns, the test cases have now a numeric order in the class name (e.g. Test01XXX). The tests are executed in this order, and the first test failing will stop the test run. (Bitbake rev: 639c46a08e524902018e28367fcb4e26362cd3e3) Signed-off-by: Alexandru DAMIAN Signed-off-by: Ed Bartosh Signed-off-by: Richard Purdie --- bitbake/lib/toaster/contrib/tts/runner.py | 11 +++++++---- bitbake/lib/toaster/contrib/tts/tests.py | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) (limited to 'bitbake/lib/toaster/contrib') diff --git a/bitbake/lib/toaster/contrib/tts/runner.py b/bitbake/lib/toaster/contrib/tts/runner.py index 6fc3e77523..bed665196e 100755 --- a/bitbake/lib/toaster/contrib/tts/runner.py +++ b/bitbake/lib/toaster/contrib/tts/runner.py @@ -31,7 +31,7 @@ from __future__ import print_function import sys, os import unittest, importlib import logging, pprint, json - +import re from shellutils import ShellCmdException, mkdirhier, run_shell_cmd import config @@ -121,13 +121,16 @@ def execute_tests(dir_under_test, testname): # pylint: disable=broad-except # we disable the broad-except because we want to actually catch all possible exceptions try: + # sorting the tests by the numeric order in the class name + tests = sorted(tests, key=lambda x: int(re.search(r"[0-9]+", x[1]).group(0))) config.logger.debug("Discovered test clases: %s", pprint.pformat(tests)) unittest.installHandler() suite = unittest.TestSuite() loader = unittest.TestLoader() result = unittest.TestResult() - for module_file, name in tests: - suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, name))) + result.failfast = True + for module_file, test_name in tests: + suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, test_name))) config.logger.info("Running %d test(s)", suite.countTestCases()) suite.run(result) @@ -202,7 +205,7 @@ def main(): config.TESTDIR = testdir # we let tests know where to run # ensure that the test dir only contains no *.pyc leftovers - run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \;" % testdir) + run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \\;" % testdir) no_failures = execute_tests(testdir, options.singletest) diff --git a/bitbake/lib/toaster/contrib/tts/tests.py b/bitbake/lib/toaster/contrib/tts/tests.py index 9acef46a38..3a4eed1e9f 100644 --- a/bitbake/lib/toaster/contrib/tts/tests.py +++ b/bitbake/lib/toaster/contrib/tts/tests.py @@ -30,7 +30,7 @@ import config import pexpect import sys, os, signal, time -class TestPyCompilable(unittest.TestCase): +class Test00PyCompilable(unittest.TestCase): ''' Verifies that all Python files are syntactically correct ''' def test_compile_file(self): try: @@ -38,7 +38,7 @@ class TestPyCompilable(unittest.TestCase): except ShellCmdException as exc: self.fail("Error compiling python files: %s" % (exc)) -class TestPySystemStart(unittest.TestCase): +class Test01PySystemStart(unittest.TestCase): ''' Attempts to start Toaster, verify that it is succesfull, and stop it ''' def setUp(self): run_shell_cmd("bash -c 'rm -f build/*log'") @@ -55,7 +55,7 @@ class TestPySystemStart(unittest.TestCase): except ShellCmdException as exc: self.fail("Failed starting managed mode: %s" % (exc)) -class TestHTML5Compliance(unittest.TestCase): +class Test02HTML5Compliance(unittest.TestCase): def setUp(self): self.origdir = os.getcwd() self.crtdir = os.path.dirname(config.TESTDIR) -- cgit v1.2.3-54-g00ecf