bitbake: toaster: tts: execute tests in numeric order

As the tests are verifying different type of the functionality,
it is usually the case that a failing early test will completely
make the subsequent tests failing, e.g. if the system cannot
start due to a bug, there is little point in testing other
functions.

In order to prevent uneeded test runs, and to generate repeatable
test patterns, the test cases have now a numeric order in the
class name (e.g. Test01XXX). The tests are executed in this order,
and the first test failing will stop the test run.

(Bitbake rev: 639c46a08e524902018e28367fcb4e26362cd3e3)

Signed-off-by: Alexandru DAMIAN <alexandru.damian@intel.com>
Signed-off-by: Ed Bartosh <ed.bartosh@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 7 insertions, 4 deletions

diff --git a/bitbake/lib/toaster/contrib/tts/runner.py b/bitbake/lib/toaster/contrib/tts/runner.py
index 6fc3e77523..bed665196e 100755
--- a/bitbake/lib/toaster/contrib/tts/runner.py
+++ b/bitbake/lib/toaster/contrib/tts/runner.py
@@ -31,7 +31,7 @@ from __future__ import print_function
 import sys, os
 import unittest, importlib
 import logging, pprint, json
-
+import re
 from shellutils import ShellCmdException, mkdirhier, run_shell_cmd
 
 import config
@@ -121,13 +121,16 @@ def execute_tests(dir_under_test, testname):
     # pylint: disable=broad-except
     # we disable the broad-except because we want to actually catch all possible exceptions
     try:
+        # sorting the tests by the numeric order in the class name
+        tests = sorted(tests, key=lambda x: int(re.search(r"[0-9]+", x[1]).group(0)))
         config.logger.debug("Discovered test clases: %s", pprint.pformat(tests))
         unittest.installHandler()
         suite = unittest.TestSuite()
         loader = unittest.TestLoader()
         result = unittest.TestResult()
-        for module_file, name in tests:
-            suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, name)))
+        result.failfast = True
+        for module_file, test_name in tests:
+            suite.addTest(loader.loadTestsFromName("%s.%s" % (module_file, test_name)))
         config.logger.info("Running %d test(s)", suite.countTestCases())
         suite.run(result)
 
@@ -202,7 +205,7 @@ def main():
         config.TESTDIR = testdir    # we let tests know where to run
 
         # ensure that the test dir only contains no *.pyc leftovers
-        run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \;" % testdir)
+        run_shell_cmd("find '%s' -type f -name *.pyc -exec rm {} \\;" % testdir)
 
         no_failures = execute_tests(testdir, options.singletest)