summaryrefslogtreecommitdiffstats
path: root/scripts/oe-selftest
diff options
context:
space:
mode:
authorDaniel Istrate <daniel.alexandrux.istrate@intel.com>2015-12-02 16:40:53 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-12-12 23:42:51 +0000
commit220a78b47e2e707e9e5364cd3d96a247c124b283 (patch)
tree10d22b4a4c55fc47d610d3223bc8b1aa3461bd54 /scripts/oe-selftest
parent98d2485fee9b041c9c97d6cdba910cbaa7e11c72 (diff)
downloadpoky-220a78b47e2e707e9e5364cd3d96a247c124b283.tar.gz
scripts: oe-selftest Added new features.
[YOCTO #8750] Allow oe-selftest to run custom test suites based on different criteria 1. Can run custom lists of tests based on different criteria: --run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags> eg: --run-tests-by module imagefeatures signing recipetool --run-tests-by id 1377 1273 935 --run-tests-by tag wic sstate bitbake 2. Can list tests based on different criteria: --list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags> eg: --list-tests-by module imagefeatures signing recipetool --list-tests-by id 1377 1273 935 --list-tests-by tag wic sstate bitbake 3. Can list all tags that have been set to test cases: --list-tags The list of tags should be kept as minimal as possible. This helps preview the tags used so far. To take advantage of the 'tag' feature: - add @tag(feature=<>) to testcases eg: @tag(feature='signing') for a single tag @tag(feature=(('signing', 'sstate')) or @tag(feature=['signing', 'sstate']) for multiple tags (From OE-Core rev: 2d3a6d22e155911e39e4b7e323317f4a7cb1cb95) Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/oe-selftest')
-rwxr-xr-xscripts/oe-selftest218
1 files changed, 216 insertions, 2 deletions
diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index 9679962ec0..bc50b2a435 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -72,6 +72,12 @@ def get_args_parser():
72 group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.') 72 group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
73 group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.') 73 group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
74 parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing") 74 parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
75 group.add_argument('--run-tests-by', required=False, dest='run_tests_by', default=False, nargs='*',
76 help='run-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
77 group.add_argument('--list-tests-by', required=False, dest='list_tests_by', default=False, nargs='*',
78 help='list-tests-by <name|class|module|id|tag> <list of tests|classes|modules|ids|tags>')
79 group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
80 help='List all tags that have been set to test cases.')
75 return parser 81 return parser
76 82
77 83
@@ -156,6 +162,187 @@ def get_tests(exclusive_modules=[], include_hidden=False):
156 162
157 return testslist 163 return testslist
158 164
165
166class Tc:
167 def __init__(self, tcname, tcclass, tcmodule, tcid=None, tctag=None):
168 self.tcname = tcname
169 self.tcclass = tcclass
170 self.tcmodule = tcmodule
171 self.tcid = tcid
172 # A test case can have multiple tags (as list or as tuples) otherwise str suffice
173 self.tctag = tctag
174 self.fullpath = '.'.join(['oeqa', 'selftest', tcmodule, tcclass, tcname])
175
176
177def get_tests_from_module(tmod):
178 tlist = []
179 prefix = 'oeqa.selftest.'
180
181 try:
182 import importlib
183 modlib = importlib.import_module(tmod)
184 for mod in vars(modlib).values():
185 if isinstance(mod, type(oeSelfTest)) and issubclass(mod, oeSelfTest) and mod is not oeSelfTest:
186 for test in dir(mod):
187 if test.startswith('test_') and hasattr(vars(mod)[test], '__call__'):
188 # Get test case id and feature tag
189 # NOTE: if testcase decorator or feature tag not set will throw error
190 try:
191 tid = vars(mod)[test].test_case
192 except:
193 print 'DEBUG: tc id missing for ' + str(test)
194 tid = None
195 try:
196 ttag = vars(mod)[test].tag__feature
197 except:
198 # print 'DEBUG: feature tag missing for ' + str(test)
199 ttag = None
200
201 # NOTE: for some reason lstrip() doesn't work for mod.__module__
202 tlist.append(Tc(test, mod.__name__, mod.__module__.replace(prefix, ''), tid, ttag))
203 except:
204 pass
205
206 return tlist
207
208
209def get_all_tests():
210 tmodules = set()
211 testlist = []
212 prefix = 'oeqa.selftest.'
213
214 # Get all the test modules (except the hidden ones)
215 for tpath in oeqa.selftest.__path__:
216 files = sorted([f for f in os.listdir(tpath) if f.endswith('.py') and not
217 f.startswith(('_', '__')) and f != 'base.py'])
218 for f in files:
219 tmodules.add(prefix + f.rstrip('.py'))
220
221 # Get all the tests from modules
222 tmodules = sorted(list(tmodules))
223
224 for tmod in tmodules:
225 testlist += get_tests_from_module(tmod)
226
227 return testlist
228
229
230def create_testsuite_by(criteria, keyword):
231 # Create a testsuite based on 'keyword'
232 # criteria: name, class, module, id, tag
233 # keyword: a list of tests, classes, modules, ids, tags
234 # NOTE: globing would be nice?
235
236 ts = set()
237 all_tests = get_all_tests()
238
239 if criteria == 'name':
240 for tc in all_tests:
241 if tc.tcname in keyword:
242 ts.add(tc.fullpath)
243
244 elif criteria == 'class':
245 for tc in all_tests:
246 if tc.tcclass in keyword:
247 ts.add(tc.fullpath)
248
249 elif criteria == 'module':
250 for tc in all_tests:
251 if tc.tcmodule in keyword:
252 ts.add(tc.fullpath)
253 elif criteria == 'id':
254 for tc in all_tests:
255 if str(tc.tcid) in keyword:
256 ts.add(tc.fullpath)
257 elif criteria == 'tag':
258 for tc in all_tests:
259 # tc can have multiple tags (as list or tuple) otherwise as str
260 if isinstance(tc.tctag, (list, tuple)):
261 for tag in tc.tctag:
262 if str(tag) in keyword:
263 ts.add(tc.fullpath)
264 elif tc.tctag in keyword:
265 ts.add(tc.fullpath)
266
267 return sorted(list(ts))
268
269
270def get_testsuite_by(criteria, keyword):
271 # Get a testsuite based on 'keyword'
272 # criteria: name, class, module, id, tag
273 # keyword: a list of tests, classes, modules, ids, tags
274 # NOTE: globing would be nice?
275 ts = set()
276 all_tests = get_all_tests()
277
278 if criteria == 'name':
279 for tc in all_tests:
280 if tc.tcname in keyword:
281 ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
282
283 elif criteria == 'class':
284 for tc in all_tests:
285 if tc.tcclass in keyword:
286 ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
287
288 elif criteria == 'module':
289 for tc in all_tests:
290 if tc.tcmodule in keyword:
291 ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
292 elif criteria == 'id':
293 for tc in all_tests:
294 if str(tc.tcid) in keyword:
295 ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
296 elif criteria == 'tag':
297 for tc in all_tests:
298 # tc can have multiple tags (as list or tuple) otherwise as str
299 if isinstance(tc.tctag, (list, tuple)):
300 for tag in tc.tctag:
301 if str(tag) in keyword:
302 ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
303 elif str(tc.tctag) in keyword:
304 ts.add((tc.tcid, tc.tctag, tc.tcname, tc.tcclass, tc.tcmodule))
305
306 return sorted(list(ts))
307
308
309def list_testsuite_by(criteria, keyword):
310 # Get a testsuite based on 'keyword'
311 # criteria: name, class, module, id, tag
312 # keyword: a list of tests, classes, modules, ids, tags
313 # NOTE: globing would be nice?
314
315 ts = get_testsuite_by(criteria, keyword)
316
317 print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % ('id', 'tag', 'name', 'class', 'module')
318 print '_' * 150
319 for t in ts:
320 if isinstance(t[1], (tuple, list)):
321 print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % (t[0], ', '.join(t[1]), t[2], t[3], t[4])
322 else:
323 print '%-4s\t%-20s\t%-60s\t%-25s\t%-20s' % t
324 print '_' * 150
325 print 'Filtering by:\t %s' % criteria
326 print 'Looking for:\t %s' % ', '.join(str(x) for x in keyword)
327 print 'Total found:\t %s' % len(ts)
328
329
330def list_tags():
331 # Get all tags set to test cases
332 # This is useful when setting tags to test cases
333 # The list of tags should be kept as minimal as possible
334 tags = set()
335 all_tests = get_all_tests()
336
337 for tc in all_tests:
338 if isinstance(tc.tctag, (tuple, list)):
339 tags.update(set(tc.tctag))
340 else:
341 tags.add(tc.tctag)
342
343 print 'Tags:\t%s' % ', '.join(str(x) for x in tags)
344
345
159def main(): 346def main():
160 parser = get_args_parser() 347 parser = get_args_parser()
161 args = parser.parse_args() 348 args = parser.parse_args()
@@ -167,6 +354,29 @@ def main():
167 sys.path.extend(layer_libdirs) 354 sys.path.extend(layer_libdirs)
168 reload(oeqa.selftest) 355 reload(oeqa.selftest)
169 356
357 if args.run_tests_by and len(args.run_tests_by) >= 2:
358 valid_options = ['name', 'class', 'module', 'id', 'tag']
359 if args.run_tests_by[0] not in valid_options:
360 print '--run-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.run_tests_by[0]
361 return 1
362 else:
363 criteria = args.run_tests_by[0]
364 keyword = args.run_tests_by[1:]
365 ts = create_testsuite_by(criteria, keyword)
366
367 if args.list_tests_by and len(args.list_tests_by) >= 2:
368 valid_options = ['name', 'class', 'module', 'id', 'tag']
369 if args.list_tests_by[0] not in valid_options:
370 print '--list-tests-by %s not a valid option. Choose one of <name|class|module|id|tag>.' % args.list_tests_by[0]
371 return 1
372 else:
373 criteria = args.list_tests_by[0]
374 keyword = args.list_tests_by[1:]
375 list_testsuite_by(criteria, keyword)
376
377 if args.list_tags:
378 list_tags()
379
170 if args.list_allclasses: 380 if args.list_allclasses:
171 args.list_modules = True 381 args.list_modules = True
172 382
@@ -195,7 +405,7 @@ def main():
195 print e 405 print e
196 pass 406 pass
197 407
198 if args.run_tests or args.run_all_tests: 408 if args.run_tests or args.run_all_tests or args.run_tests_by:
199 if not preflight_check(): 409 if not preflight_check():
200 return 1 410 return 1
201 411
@@ -235,7 +445,11 @@ def main():
235 445
236 coverage_process_start = os.environ["COVERAGE_PROCESS_START"] = coveragerc 446 coverage_process_start = os.environ["COVERAGE_PROCESS_START"] = coveragerc
237 447
238 testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False) 448 if args.run_tests_by:
449 testslist = ts
450 else:
451 testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
452
239 suite = unittest.TestSuite() 453 suite = unittest.TestSuite()
240 loader = unittest.TestLoader() 454 loader = unittest.TestLoader()
241 loader.sortTestMethodsUsing = None 455 loader.sortTestMethodsUsing = None