diff options
author | Daniel Istrate <daniel.alexandrux.istrate@intel.com> | 2016-01-25 16:03:21 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2016-01-25 16:29:10 +0000 |
commit | 72f98ba5775f6faa2bcad80b913948c1b953111b (patch) | |
tree | 916d26051bbc5cb0f6782b01256de76423ef534f /bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py | |
parent | c192bd60e8c7bcb9bbdd24158bd66eb691bfee5a (diff) | |
download | poky-72f98ba5775f6faa2bcad80b913948c1b953111b.tar.gz |
bitbake: toaster: Update UI test runner
Add new runner options:
--run-all-tests: finds all tests, ignores config
--run-suite <suite> (from cfg)
Without arguments, run tests from current os section (config), e.g.:
1. ./run_toastertests
2. ./run_toastertests --run-all-tests
3. ./run_toastertests --run-suite darwin
Update toaster logging to meet QA CI requirements.
(Bitbake rev: 5685feb51fbb6d54fde6027cc765b9edd8eda65a)
Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com>
Signed-off-by: Elliot Smith <elliot.smith@intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py')
-rwxr-xr-x | bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py | 164 |
1 files changed, 116 insertions, 48 deletions
diff --git a/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py b/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py index 880487cb6b..2b312cb927 100755 --- a/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py +++ b/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py | |||
@@ -28,60 +28,128 @@ | |||
28 | # put chromedriver in PATH, (e.g. /usr/bin/, bear in mind to chmod) | 28 | # put chromedriver in PATH, (e.g. /usr/bin/, bear in mind to chmod) |
29 | # For windows host, you may put chromedriver.exe in the same directory as chrome.exe | 29 | # For windows host, you may put chromedriver.exe in the same directory as chrome.exe |
30 | 30 | ||
31 | 31 | import unittest, sys, os, platform | |
32 | import unittest, time, re, sys, getopt, os, logging, platform | ||
33 | import ConfigParser | 32 | import ConfigParser |
34 | import subprocess | 33 | import argparse |
35 | 34 | from toaster_automation_test import toaster_cases | |
36 | 35 | ||
37 | class toaster_run_all(): | 36 | |
38 | def __init__(self): | 37 | def get_args_parser(): |
39 | # in case this script is called from other directory | 38 | description = "Script that runs toaster auto tests." |
40 | os.chdir(os.path.abspath(sys.path[0])) | 39 | parser = argparse.ArgumentParser(description=description) |
41 | self.starttime = time.strptime(time.ctime()) | 40 | parser.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, |
42 | self.parser = ConfigParser.SafeConfigParser() | 41 | help='Run all tests.') |
43 | found = self.parser.read('toaster_test.cfg') | 42 | parser.add_argument('--run-suite', required=False, dest='run_suite', default=False, |
44 | self.host_os = platform.system().lower() | 43 | help='run suite (defined in cfg file)') |
45 | self.run_all_cases() | 44 | |
46 | self.collect_log() | 45 | return parser |
47 | 46 | ||
48 | def get_test_cases(self): | 47 | |
49 | # we have config groups for different os type in toaster_test.cfg | 48 | def get_tests(): |
50 | cases_to_run = eval(self.parser.get('toaster_test_' + self.host_os, 'test_cases')) | 49 | testslist = [] |
51 | return cases_to_run | 50 | |
52 | 51 | prefix = 'toaster_automation_test.toaster_cases' | |
53 | 52 | ||
54 | def run_all_cases(self): | 53 | for t in dir(toaster_cases): |
55 | cases_temp = self.get_test_cases() | 54 | if t.startswith('test_'): |
56 | for case in cases_temp: | 55 | testslist.append('.'.join((prefix, t))) |
57 | single_case_cmd = "python -m unittest toaster_automation_test.toaster_cases.test_" + str(case) | 56 | |
58 | print single_case_cmd | 57 | return testslist |
59 | subprocess.call(single_case_cmd, shell=True) | 58 | |
60 | 59 | ||
61 | def collect_log(self): | 60 | def get_tests_from_cfg(suite=None): |
61 | |||
62 | testslist = [] | ||
63 | config = ConfigParser.SafeConfigParser() | ||
64 | config.read('toaster_test.cfg') | ||
65 | |||
66 | if suite is not None: | ||
67 | target_suite = suite.lower() | ||
68 | |||
69 | # TODO: if suite is valid suite | ||
70 | |||
71 | else: | ||
72 | target_suite = platform.system().lower() | ||
73 | |||
74 | try: | ||
75 | tests_from_cfg = eval(config.get('toaster_test_' + target_suite, 'test_cases')) | ||
76 | except: | ||
77 | print 'Failed to get test cases from cfg file. Make sure the format is correct.' | ||
78 | return None | ||
79 | |||
80 | prefix = 'toaster_automation_test.toaster_cases.test_' | ||
81 | for t in tests_from_cfg: | ||
82 | testslist.append(prefix + str(t)) | ||
83 | |||
84 | return testslist | ||
85 | |||
86 | def main(): | ||
87 | |||
88 | # In case this script is called from other directory | ||
89 | os.chdir(os.path.abspath(sys.path[0])) | ||
90 | |||
91 | parser = get_args_parser() | ||
92 | args = parser.parse_args() | ||
93 | |||
94 | if args.run_all_tests: | ||
95 | testslist = get_tests() | ||
96 | elif args.run_suite: | ||
97 | testslist = get_tests_from_cfg(args.run_suite) | ||
98 | os.environ['TOASTER_SUITE'] = args.run_suite | ||
99 | else: | ||
100 | testslist = get_tests_from_cfg() | ||
101 | |||
102 | if not testslist: | ||
103 | print 'Failed to get test cases.' | ||
104 | exit(1) | ||
105 | |||
106 | suite = unittest.TestSuite() | ||
107 | loader = unittest.TestLoader() | ||
108 | loader.sortTestMethodsUsing = None | ||
109 | runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args)) | ||
110 | |||
111 | for test in testslist: | ||
112 | try: | ||
113 | suite.addTests(loader.loadTestsFromName(test)) | ||
114 | except: | ||
115 | return 1 | ||
116 | |||
117 | result = runner.run(suite) | ||
118 | |||
119 | if result.wasSuccessful(): | ||
120 | return 0 | ||
121 | else: | ||
122 | return 1 | ||
123 | |||
124 | |||
125 | def buildResultClass(args): | ||
126 | """Build a Result Class to use in the testcase execution""" | ||
127 | |||
128 | class StampedResult(unittest.TextTestResult): | ||
62 | """ | 129 | """ |
63 | the log files are temporarily stored in ./log/tmp/.. | 130 | Custom TestResult that prints the time when a test starts. As toaster-auto |
64 | After all cases are done, they should be transfered to ./log/$TIMESTAMP/ | 131 | can take a long time (ie a few hours) to run, timestamps help us understand |
132 | what tests are taking a long time to execute. | ||
65 | """ | 133 | """ |
66 | def comple(number): | 134 | def startTest(self, test): |
67 | if number < 10: | 135 | import time |
68 | return str(0) + str(number) | 136 | self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ") |
69 | else: | 137 | super(StampedResult, self).startTest(test) |
70 | return str(number) | ||
71 | now = self.starttime | ||
72 | now_str = comple(now.tm_year) + comple(now.tm_mon) + comple(now.tm_mday) + \ | ||
73 | comple(now.tm_hour) + comple(now.tm_min) + comple(now.tm_sec) | ||
74 | log_dir = os.path.abspath(sys.path[0]) + os.sep + 'log' + os.sep + now_str | ||
75 | log_tmp_dir = os.path.abspath(sys.path[0]) + os.sep + 'log' + os.sep + 'tmp' | ||
76 | try: | ||
77 | os.renames(log_tmp_dir, log_dir) | ||
78 | except OSError : | ||
79 | logging.error(" Cannot create log dir(timestamp) under log, please check your privilege") | ||
80 | 138 | ||
139 | return StampedResult | ||
81 | 140 | ||
82 | if __name__ == "__main__": | ||
83 | toaster_run_all() | ||
84 | 141 | ||
142 | if __name__ == "__main__": | ||
85 | 143 | ||
144 | try: | ||
145 | ret = main() | ||
146 | except: | ||
147 | ret = 1 | ||
148 | import traceback | ||
149 | traceback.print_exc(5) | ||
150 | finally: | ||
151 | if os.getenv('TOASTER_SUITE'): | ||
152 | del os.environ['TOASTER_SUITE'] | ||
153 | sys.exit(ret) | ||
86 | 154 | ||
87 | 155 | ||