summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorDaniel Istrate <daniel.alexandrux.istrate@intel.com>2016-01-25 16:03:21 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-01-25 16:29:10 +0000
commit72f98ba5775f6faa2bcad80b913948c1b953111b (patch)
tree916d26051bbc5cb0f6782b01256de76423ef534f /bitbake
parentc192bd60e8c7bcb9bbdd24158bd66eb691bfee5a (diff)
downloadpoky-72f98ba5775f6faa2bcad80b913948c1b953111b.tar.gz
bitbake: toaster: Update UI test runner
Add new runner options: --run-all-tests: finds all tests, ignores config --run-suite <suite> (from cfg) Without arguments, run tests from current os section (config), e.g.: 1. ./run_toastertests 2. ./run_toastertests --run-all-tests 3. ./run_toastertests --run-suite darwin Update toaster logging to meet QA CI requirements. (Bitbake rev: 5685feb51fbb6d54fde6027cc765b9edd8eda65a) Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com> Signed-off-by: Elliot Smith <elliot.smith@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rwxr-xr-xbitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py164
-rwxr-xr-xbitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py92
2 files changed, 172 insertions, 84 deletions
diff --git a/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py b/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py
index 880487cb6b..2b312cb927 100755
--- a/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py
+++ b/bitbake/lib/toaster/contrib/tts/toasteruitest/run_toastertests.py
@@ -28,60 +28,128 @@
28# put chromedriver in PATH, (e.g. /usr/bin/, bear in mind to chmod) 28# put chromedriver in PATH, (e.g. /usr/bin/, bear in mind to chmod)
29# For windows host, you may put chromedriver.exe in the same directory as chrome.exe 29# For windows host, you may put chromedriver.exe in the same directory as chrome.exe
30 30
31 31import unittest, sys, os, platform
32import unittest, time, re, sys, getopt, os, logging, platform
33import ConfigParser 32import ConfigParser
34import subprocess 33import argparse
35 34from toaster_automation_test import toaster_cases
36 35
37class toaster_run_all(): 36
38 def __init__(self): 37def get_args_parser():
39 # in case this script is called from other directory 38 description = "Script that runs toaster auto tests."
40 os.chdir(os.path.abspath(sys.path[0])) 39 parser = argparse.ArgumentParser(description=description)
41 self.starttime = time.strptime(time.ctime()) 40 parser.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False,
42 self.parser = ConfigParser.SafeConfigParser() 41 help='Run all tests.')
43 found = self.parser.read('toaster_test.cfg') 42 parser.add_argument('--run-suite', required=False, dest='run_suite', default=False,
44 self.host_os = platform.system().lower() 43 help='run suite (defined in cfg file)')
45 self.run_all_cases() 44
46 self.collect_log() 45 return parser
47 46
48 def get_test_cases(self): 47
49 # we have config groups for different os type in toaster_test.cfg 48def get_tests():
50 cases_to_run = eval(self.parser.get('toaster_test_' + self.host_os, 'test_cases')) 49 testslist = []
51 return cases_to_run 50
52 51 prefix = 'toaster_automation_test.toaster_cases'
53 52
54 def run_all_cases(self): 53 for t in dir(toaster_cases):
55 cases_temp = self.get_test_cases() 54 if t.startswith('test_'):
56 for case in cases_temp: 55 testslist.append('.'.join((prefix, t)))
57 single_case_cmd = "python -m unittest toaster_automation_test.toaster_cases.test_" + str(case) 56
58 print single_case_cmd 57 return testslist
59 subprocess.call(single_case_cmd, shell=True) 58
60 59
61 def collect_log(self): 60def get_tests_from_cfg(suite=None):
61
62 testslist = []
63 config = ConfigParser.SafeConfigParser()
64 config.read('toaster_test.cfg')
65
66 if suite is not None:
67 target_suite = suite.lower()
68
69 # TODO: if suite is valid suite
70
71 else:
72 target_suite = platform.system().lower()
73
74 try:
75 tests_from_cfg = eval(config.get('toaster_test_' + target_suite, 'test_cases'))
76 except:
77 print 'Failed to get test cases from cfg file. Make sure the format is correct.'
78 return None
79
80 prefix = 'toaster_automation_test.toaster_cases.test_'
81 for t in tests_from_cfg:
82 testslist.append(prefix + str(t))
83
84 return testslist
85
86def main():
87
88 # In case this script is called from other directory
89 os.chdir(os.path.abspath(sys.path[0]))
90
91 parser = get_args_parser()
92 args = parser.parse_args()
93
94 if args.run_all_tests:
95 testslist = get_tests()
96 elif args.run_suite:
97 testslist = get_tests_from_cfg(args.run_suite)
98 os.environ['TOASTER_SUITE'] = args.run_suite
99 else:
100 testslist = get_tests_from_cfg()
101
102 if not testslist:
103 print 'Failed to get test cases.'
104 exit(1)
105
106 suite = unittest.TestSuite()
107 loader = unittest.TestLoader()
108 loader.sortTestMethodsUsing = None
109 runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args))
110
111 for test in testslist:
112 try:
113 suite.addTests(loader.loadTestsFromName(test))
114 except:
115 return 1
116
117 result = runner.run(suite)
118
119 if result.wasSuccessful():
120 return 0
121 else:
122 return 1
123
124
125def buildResultClass(args):
126 """Build a Result Class to use in the testcase execution"""
127
128 class StampedResult(unittest.TextTestResult):
62 """ 129 """
63 the log files are temporarily stored in ./log/tmp/.. 130 Custom TestResult that prints the time when a test starts. As toaster-auto
64 After all cases are done, they should be transfered to ./log/$TIMESTAMP/ 131 can take a long time (ie a few hours) to run, timestamps help us understand
132 what tests are taking a long time to execute.
65 """ 133 """
66 def comple(number): 134 def startTest(self, test):
67 if number < 10: 135 import time
68 return str(0) + str(number) 136 self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
69 else: 137 super(StampedResult, self).startTest(test)
70 return str(number)
71 now = self.starttime
72 now_str = comple(now.tm_year) + comple(now.tm_mon) + comple(now.tm_mday) + \
73 comple(now.tm_hour) + comple(now.tm_min) + comple(now.tm_sec)
74 log_dir = os.path.abspath(sys.path[0]) + os.sep + 'log' + os.sep + now_str
75 log_tmp_dir = os.path.abspath(sys.path[0]) + os.sep + 'log' + os.sep + 'tmp'
76 try:
77 os.renames(log_tmp_dir, log_dir)
78 except OSError :
79 logging.error(" Cannot create log dir(timestamp) under log, please check your privilege")
80 138
139 return StampedResult
81 140
82if __name__ == "__main__":
83 toaster_run_all()
84 141
142if __name__ == "__main__":
85 143
144 try:
145 ret = main()
146 except:
147 ret = 1
148 import traceback
149 traceback.print_exc(5)
150 finally:
151 if os.getenv('TOASTER_SUITE'):
152 del os.environ['TOASTER_SUITE']
153 sys.exit(ret)
86 154
87 155
diff --git a/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py b/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py
index d975d48acb..d8f838aeaf 100755
--- a/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py
+++ b/bitbake/lib/toaster/contrib/tts/toasteruitest/toaster_automation_test.py
@@ -230,60 +230,70 @@ class NoParsingFilter(logging.Filter):
230def LogResults(original_class): 230def LogResults(original_class):
231 orig_method = original_class.run 231 orig_method = original_class.run
232 232
233 from time import strftime, gmtime
234 caller = 'toaster'
235 timestamp = strftime('%Y%m%d%H%M%S',gmtime())
236 logfile = os.path.join(os.getcwd(),'results-'+caller+'.'+timestamp+'.log')
237 linkfile = os.path.join(os.getcwd(),'results-'+caller+'.log')
238
233 #rewrite the run method of unittest.TestCase to add testcase logging 239 #rewrite the run method of unittest.TestCase to add testcase logging
234 def run(self, result, *args, **kws): 240 def run(self, result, *args, **kws):
235 orig_method(self, result, *args, **kws) 241 orig_method(self, result, *args, **kws)
236 passed = True 242 passed = True
237 testMethod = getattr(self, self._testMethodName) 243 testMethod = getattr(self, self._testMethodName)
238
239 #if test case is decorated then use it's number, else use it's name 244 #if test case is decorated then use it's number, else use it's name
240 try: 245 try:
241 test_case = testMethod.test_case 246 test_case = testMethod.test_case
242 except AttributeError: 247 except AttributeError:
243 test_case = self._testMethodName 248 test_case = self._testMethodName
244 249
250 class_name = str(testMethod.im_class).split("'")[1]
251
245 #create custom logging level for filtering. 252 #create custom logging level for filtering.
246 custom_log_level = 100 253 custom_log_level = 100
247 logging.addLevelName(custom_log_level, 'RESULTS') 254 logging.addLevelName(custom_log_level, 'RESULTS')
248 caller = os.path.basename(sys.argv[0])
249 255
250 def results(self, message, *args, **kws): 256 def results(self, message, *args, **kws):
251 if self.isEnabledFor(custom_log_level): 257 if self.isEnabledFor(custom_log_level):
252 self.log(custom_log_level, message, *args, **kws) 258 self.log(custom_log_level, message, *args, **kws)
253 logging.Logger.results = results 259 logging.Logger.results = results
254 260
255 logging.basicConfig(filename=os.path.join(os.getcwd(),'results-'+caller+'.log'), 261 logging.basicConfig(filename=logfile,
256 filemode='w', 262 filemode='w',
257 format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', 263 format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
258 datefmt='%H:%M:%S', 264 datefmt='%H:%M:%S',
259 level=custom_log_level) 265 level=custom_log_level)
260 for handler in logging.root.handlers: 266 for handler in logging.root.handlers:
261 handler.addFilter(NoParsingFilter()) 267 handler.addFilter(NoParsingFilter())
262# local_log = logging.getLogger(caller) 268 local_log = logging.getLogger(caller)
263 local_log = logging.getLogger()
264 269
265 #check status of tests and record it 270 #check status of tests and record it
271
266 for (name, msg) in result.errors: 272 for (name, msg) in result.errors:
267 if self._testMethodName == str(name).split(' ')[0]: 273 if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
268 local_log.results("Testcase "+str(test_case)+": ERROR") 274 local_log.results("Testcase "+str(test_case)+": ERROR")
269 local_log.results("Testcase "+str(test_case)+":\n"+msg+"\n\n\n") 275 local_log.results("Testcase "+str(test_case)+":\n"+msg)
270 passed = False 276 passed = False
271 for (name, msg) in result.failures: 277 for (name, msg) in result.failures:
272 if self._testMethodName == str(name).split(' ')[0]: 278 if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
273 local_log.results("Testcase "+str(test_case)+": FAILED") 279 local_log.results("Testcase "+str(test_case)+": FAILED")
274 local_log.results("Testcase "+str(test_case)+":\n"+msg+"\n\n\n") 280 local_log.results("Testcase "+str(test_case)+":\n"+msg)
275 passed = False 281 passed = False
276 for (name, msg) in result.skipped: 282 for (name, msg) in result.skipped:
277 if self._testMethodName == str(name).split(' ')[0]: 283 if (self._testMethodName == str(name).split(' ')[0]) and (class_name in str(name).split(' ')[1]):
278 local_log.results("Testcase "+str(test_case)+": SKIPPED"+"\n\n\n") 284 local_log.results("Testcase "+str(test_case)+": SKIPPED")
279 passed = False 285 passed = False
280 if passed: 286 if passed:
281 local_log.results("Testcase "+str(test_case)+": PASSED"+"\n\n\n") 287 local_log.results("Testcase "+str(test_case)+": PASSED")
282 288
283 original_class.run = run 289 # Create symlink to the current log
284 return original_class 290 if os.path.exists(linkfile):
291 os.remove(linkfile)
292 os.symlink(logfile, linkfile)
285 293
294 original_class.run = run
286 295
296 return original_class
287 297
288 298
289########################################### 299###########################################
@@ -292,16 +302,26 @@ def LogResults(original_class):
292# # 302# #
293########################################### 303###########################################
294 304
305@LogResults
295class toaster_cases_base(unittest.TestCase): 306class toaster_cases_base(unittest.TestCase):
296 307
308 @classmethod
309 def setUpClass(cls):
310 cls.log = cls.logger_create()
311
297 def setUp(self): 312 def setUp(self):
298 self.screenshot_sequence = 1 313 self.screenshot_sequence = 1
299 self.verificationErrors = [] 314 self.verificationErrors = []
300 self.accept_next_alert = True 315 self.accept_next_alert = True
301 self.host_os = platform.system().lower() 316 self.host_os = platform.system().lower()
317 if os.getenv('TOASTER_SUITE'):
318 self.target_suite = os.getenv('TOASTER_SUITE')
319 else:
320 self.target_suite = self.host_os
321
302 self.parser = ConfigParser.SafeConfigParser() 322 self.parser = ConfigParser.SafeConfigParser()
303 configs = self.parser.read('toaster_test.cfg') 323 self.parser.read('toaster_test.cfg')
304 self.base_url = eval(self.parser.get('toaster_test_' + self.host_os, 'toaster_url')) 324 self.base_url = eval(self.parser.get('toaster_test_' + self.target_suite, 'toaster_url'))
305 325
306 # create log dir . Currently , we put log files in log/tmp. After all 326 # create log dir . Currently , we put log files in log/tmp. After all
307 # test cases are done, move them to log/$datetime dir 327 # test cases are done, move them to log/$datetime dir
@@ -310,37 +330,37 @@ class toaster_cases_base(unittest.TestCase):
310 mkdir_p(self.log_tmp_dir) 330 mkdir_p(self.log_tmp_dir)
311 except OSError : 331 except OSError :
312 logging.error("%(asctime)s Cannot create tmp dir under log, please check your privilege") 332 logging.error("%(asctime)s Cannot create tmp dir under log, please check your privilege")
313 self.log = self.logger_create() 333 # self.log = self.logger_create()
314 # driver setup 334 # driver setup
315 self.setup_browser() 335 self.setup_browser()
316 336
317 def logger_create(self): 337 @staticmethod
318 """ 338 def logger_create():
319 we use root logger for every testcase. 339 log_file = "toaster-auto-" + time.strftime("%Y%m%d%H%M%S") + ".log"
320 The reason why we don't use TOASTERXXX_logger is to avoid setting respective level for 340 if os.path.exists("toaster-auto.log"): os.remove("toaster-auto.log")
321 root logger and TOASTERXXX_logger 341 os.symlink(log_file, "toaster-auto.log")
322 To Be Discussed 342
323 """ 343 log = logging.getLogger("toaster")
324 log_level_dict = {'CRITICAL':logging.CRITICAL, 'ERROR':logging.ERROR, 'WARNING':logging.WARNING, \ 344 log.setLevel(logging.DEBUG)
325 'INFO':logging.INFO, 'DEBUG':logging.DEBUG, 'NOTSET':logging.NOTSET} 345
326 log = logging.getLogger() 346 fh = logging.FileHandler(filename=log_file, mode='w')
327# log = logging.getLogger('TOASTER_' + str(self.case_no)) 347 fh.setLevel(logging.DEBUG)
328 self.logging_level = eval(self.parser.get('toaster_test_' + self.host_os, 'logging_level')) 348
329 key = self.logging_level.upper()
330 log.setLevel(log_level_dict[key])
331 fh = logging.FileHandler(filename=self.log_tmp_dir + os.sep + 'case_all' + '.log', mode='a')
332 ch = logging.StreamHandler(sys.stdout) 349 ch = logging.StreamHandler(sys.stdout)
333 formatter = logging.Formatter('%(pathname)s - %(lineno)d - %(asctime)s \n \ 350 ch.setLevel(logging.INFO)
334 %(name)s - %(levelname)s - %(message)s') 351
352 formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
335 fh.setFormatter(formatter) 353 fh.setFormatter(formatter)
336 ch.setFormatter(formatter) 354 ch.setFormatter(formatter)
355
337 log.addHandler(fh) 356 log.addHandler(fh)
338 log.addHandler(ch) 357 log.addHandler(ch)
358
339 return log 359 return log
340 360
341 361
342 def setup_browser(self, *browser_path): 362 def setup_browser(self, *browser_path):
343 self.browser = eval(self.parser.get('toaster_test_' + self.host_os, 'test_browser')) 363 self.browser = eval(self.parser.get('toaster_test_' + self.target_suite, 'test_browser'))
344 print self.browser 364 print self.browser
345 if self.browser == "firefox": 365 if self.browser == "firefox":
346 driver = webdriver.Firefox() 366 driver = webdriver.Firefox()
@@ -660,7 +680,7 @@ class toaster_cases_base(unittest.TestCase):
660# Note: to comply with the unittest framework, we call these test_xxx functions 680# Note: to comply with the unittest framework, we call these test_xxx functions
661# from run_toastercases.py to avoid calling setUp() and tearDown() multiple times 681# from run_toastercases.py to avoid calling setUp() and tearDown() multiple times
662 682
663@LogResults 683
664class toaster_cases(toaster_cases_base): 684class toaster_cases(toaster_cases_base):
665 ############## 685 ##############
666 # CASE 901 # 686 # CASE 901 #