summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--meta/lib/oeqa/core/context.py148
-rw-r--r--meta/lib/oeqa/core/decorator/__init__.py71
-rw-r--r--meta/lib/oeqa/core/loader.py235
3 files changed, 454 insertions, 0 deletions
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
new file mode 100644
index 0000000000..cbab2f8f5f
--- /dev/null
+++ b/meta/lib/oeqa/core/context.py
@@ -0,0 +1,148 @@
1# Copyright (C) 2016 Intel Corporation
2# Released under the MIT license (see COPYING.MIT)
3
4import os
5import sys
6import json
7import time
8import logging
9import collections
10import re
11
12from oeqa.core.loader import OETestLoader
13from oeqa.core.runner import OETestRunner, OEStreamLogger, xmlEnabled
14
15class OETestContext(object):
16 loaderClass = OETestLoader
17 runnerClass = OETestRunner
18 streamLoggerClass = OEStreamLogger
19
20 files_dir = os.path.abspath(os.path.join(os.path.dirname(
21 os.path.abspath(__file__)), "../files"))
22
23 def __init__(self, td=None, logger=None):
24 if not type(td) is dict:
25 raise TypeError("td isn't dictionary type")
26
27 self.td = td
28 self.logger = logger
29 self._registry = {}
30 self._registry['cases'] = collections.OrderedDict()
31 self._results = {}
32
33 def _read_modules_from_manifest(self, manifest):
34 if not os.path.exists(manifest):
35 raise
36
37 modules = []
38 for line in open(manifest).readlines():
39 line = line.strip()
40 if line and not line.startswith("#"):
41 modules.append(line)
42
43 return modules
44
45 def loadTests(self, module_paths, modules=[], tests=[],
46 modules_manifest="", modules_required=[], filters={}):
47 if modules_manifest:
48 modules = self._read_modules_from_manifest(modules_manifest)
49
50 self.loader = self.loaderClass(self, module_paths, modules, tests,
51 modules_required, filters)
52 self.suites = self.loader.discover()
53
54 def runTests(self):
55 streamLogger = self.streamLoggerClass(self.logger)
56 self.runner = self.runnerClass(self, stream=streamLogger, verbosity=2)
57
58 self._run_start_time = time.time()
59 result = self.runner.run(self.suites)
60 self._run_end_time = time.time()
61
62 return result
63
64 def logSummary(self, result, component, context_msg=''):
65 self.logger.info("SUMMARY:")
66 self.logger.info("%s (%s) - Ran %d test%s in %.3fs" % (component,
67 context_msg, result.testsRun, result.testsRun != 1 and "s" or "",
68 (self._run_end_time - self._run_start_time)))
69
70 if result.wasSuccessful():
71 msg = "%s - OK - All required tests passed" % component
72 else:
73 msg = "%s - FAIL - Required tests failed" % component
74 skipped = len(self._results['skipped'])
75 if skipped:
76 msg += " (skipped=%d)" % skipped
77 self.logger.info(msg)
78
79 def _getDetailsNotPassed(self, case, type, desc):
80 found = False
81
82 for (scase, msg) in self._results[type]:
83 # XXX: When XML reporting is enabled scase is
84 # xmlrunner.result._TestInfo instance instead of
85 # string.
86 if xmlEnabled:
87 if case.id() == scase.test_id:
88 found = True
89 break
90 scase_str = scase.test_id
91 else:
92 if case == scase:
93 found = True
94 break
95 scase_str = str(scase)
96
97 # When fails at module or class level the class name is passed as string
98 # so figure out to see if match
99 m = re.search("^setUpModule \((?P<module_name>.*)\)$", scase_str)
100 if m:
101 if case.__class__.__module__ == m.group('module_name'):
102 found = True
103 break
104
105 m = re.search("^setUpClass \((?P<class_name>.*)\)$", scase_str)
106 if m:
107 class_name = "%s.%s" % (case.__class__.__module__,
108 case.__class__.__name__)
109
110 if class_name == m.group('class_name'):
111 found = True
112 break
113
114 if found:
115 return (found, msg)
116
117 return (found, None)
118
119 def logDetails(self):
120 self.logger.info("RESULTS:")
121 for case_name in self._registry['cases']:
122 case = self._registry['cases'][case_name]
123
124 result_types = ['failures', 'errors', 'skipped', 'expectedFailures']
125 result_desc = ['FAILED', 'ERROR', 'SKIPPED', 'EXPECTEDFAIL']
126
127 fail = False
128 desc = None
129 for idx, name in enumerate(result_types):
130 (fail, msg) = self._getDetailsNotPassed(case, result_types[idx],
131 result_desc[idx])
132 if fail:
133 desc = result_desc[idx]
134 break
135
136 oeid = -1
137 for d in case.decorators:
138 if hasattr(d, 'oeid'):
139 oeid = d.oeid
140
141 if fail:
142 self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
143 oeid, desc))
144 if msg:
145 self.logger.info(msg)
146 else:
147 self.logger.info("RESULTS - %s - Testcase %s: %s" % (case.id(),
148 oeid, 'PASSED'))
diff --git a/meta/lib/oeqa/core/decorator/__init__.py b/meta/lib/oeqa/core/decorator/__init__.py
new file mode 100644
index 0000000000..855b6b9d28
--- /dev/null
+++ b/meta/lib/oeqa/core/decorator/__init__.py
@@ -0,0 +1,71 @@
1# Copyright (C) 2016 Intel Corporation
2# Released under the MIT license (see COPYING.MIT)
3
4from functools import wraps
5from abc import abstractmethod
6
7decoratorClasses = set()
8
9def registerDecorator(obj):
10 decoratorClasses.add(obj)
11 return obj
12
13class OETestDecorator(object):
14 case = None # Reference of OETestCase decorated
15 attrs = None # Attributes to be loaded by decorator implementation
16
17 def __init__(self, *args, **kwargs):
18 if not self.attrs:
19 return
20
21 for idx, attr in enumerate(self.attrs):
22 if attr in kwargs:
23 value = kwargs[attr]
24 else:
25 value = args[idx]
26 setattr(self, attr, value)
27
28 def __call__(self, func):
29 @wraps(func)
30 def wrapped_f(*args, **kwargs):
31 self.attrs = self.attrs # XXX: Enables OETestLoader discover
32 return func(*args, **kwargs)
33 return wrapped_f
34
35 # OETestLoader call it when is loading test cases.
36 # XXX: Most methods would change the registry for later
37 # processing; be aware that filtrate method needs to
38 # run later than bind, so there could be data (in the
39 # registry) of a cases that were filtered.
40 def bind(self, registry, case):
41 self.case = case
42 self.logger = case.tc.logger
43 self.case.decorators.append(self)
44
45 # OETestRunner call this method when tries to run
46 # the test case.
47 def setUpDecorator(self):
48 pass
49
50 # OETestRunner call it after a test method has been
51 # called even if the method raised an exception.
52 def tearDownDecorator(self):
53 pass
54
55class OETestDiscover(OETestDecorator):
56
57 # OETestLoader call it after discover test cases
58 # needs to return the cases to be run.
59 @staticmethod
60 def discover(registry):
61 return registry['cases']
62
63class OETestFilter(OETestDecorator):
64
65 # OETestLoader call it while loading the tests
66 # in loadTestsFromTestCase method, it needs to
67 # return a bool, True if needs to be filtered.
68 # This method must consume the filter used.
69 @abstractmethod
70 def filtrate(self, filters):
71 return False
diff --git a/meta/lib/oeqa/core/loader.py b/meta/lib/oeqa/core/loader.py
new file mode 100644
index 0000000000..c73ef9a0fc
--- /dev/null
+++ b/meta/lib/oeqa/core/loader.py
@@ -0,0 +1,235 @@
1# Copyright (C) 2016 Intel Corporation
2# Released under the MIT license (see COPYING.MIT)
3
4import os
5import sys
6import unittest
7
8from oeqa.core.utils.path import findFile
9from oeqa.core.utils.test import getSuiteModules, getCaseID
10
11from oeqa.core.case import OETestCase
12from oeqa.core.decorator import decoratorClasses, OETestDecorator, \
13 OETestFilter, OETestDiscover
14
15def _make_failed_test(classname, methodname, exception, suiteClass):
16 """
17 When loading tests unittest framework stores the exception in a new
18 class created for be displayed into run().
19
20 For our purposes will be better to raise the exception in loading
21 step instead of wait to run the test suite.
22 """
23 raise exception
24unittest.loader._make_failed_test = _make_failed_test
25
26def _find_duplicated_modules(suite, directory):
27 for module in getSuiteModules(suite):
28 path = findFile('%s.py' % module, directory)
29 if path:
30 raise ImportError("Duplicated %s module found in %s" % (module, path))
31
32class OETestLoader(unittest.TestLoader):
33 caseClass = OETestCase
34
35 kwargs_names = ['testMethodPrefix', 'sortTestMethodUsing', 'suiteClass',
36 '_top_level_dir']
37
38 def __init__(self, tc, module_paths, modules, tests, modules_required,
39 filters, *args, **kwargs):
40 self.tc = tc
41
42 self.modules = modules
43 self.tests = tests
44 self.modules_required = modules_required
45
46 self.filters = filters
47 self.decorator_filters = [d for d in decoratorClasses if \
48 issubclass(d, OETestFilter)]
49 self._validateFilters(self.filters, self.decorator_filters)
50 self.used_filters = [d for d in self.decorator_filters
51 for f in self.filters
52 if f in d.attrs]
53
54 if isinstance(module_paths, str):
55 module_paths = [module_paths]
56 elif not isinstance(module_paths, list):
57 raise TypeError('module_paths must be a str or a list of str')
58 self.module_paths = module_paths
59
60 for kwname in self.kwargs_names:
61 if kwname in kwargs:
62 setattr(self, kwname, kwargs[kwname])
63
64 self._patchCaseClass(self.caseClass)
65
66 def _patchCaseClass(self, testCaseClass):
67 # Adds custom attributes to the OETestCase class
68 setattr(testCaseClass, 'tc', self.tc)
69 setattr(testCaseClass, 'td', self.tc.td)
70 setattr(testCaseClass, 'logger', self.tc.logger)
71
72 def _validateFilters(self, filters, decorator_filters):
73 # Validate if filter isn't empty
74 for key,value in filters.items():
75 if not value:
76 raise TypeError("Filter %s specified is empty" % key)
77
78 # Validate unique attributes
79 attr_filters = [attr for clss in decorator_filters \
80 for attr in clss.attrs]
81 dup_attr = [attr for attr in attr_filters
82 if attr_filters.count(attr) > 1]
83 if dup_attr:
84 raise TypeError('Detected duplicated attribute(s) %s in filter'
85 ' decorators' % ' ,'.join(dup_attr))
86
87 # Validate if filter is supported
88 for f in filters:
89 if f not in attr_filters:
90 classes = ', '.join([d.__name__ for d in decorator_filters])
91 raise TypeError('Found "%s" filter but not declared in any of '
92 '%s decorators' % (f, classes))
93
94 def _registerTestCase(self, case):
95 case_id = case.id()
96 self.tc._registry['cases'][case_id] = case
97
98 def _handleTestCaseDecorators(self, case):
99 def _handle(obj):
100 if isinstance(obj, OETestDecorator):
101 if not obj.__class__ in decoratorClasses:
102 raise Exception("Decorator %s isn't registered" \
103 " in decoratorClasses." % obj.__name__)
104 obj.bind(self.tc._registry, case)
105
106 def _walk_closure(obj):
107 if hasattr(obj, '__closure__') and obj.__closure__:
108 for f in obj.__closure__:
109 obj = f.cell_contents
110 _handle(obj)
111 _walk_closure(obj)
112 method = getattr(case, case._testMethodName, None)
113 _walk_closure(method)
114
115 def _filterTest(self, case):
116 """
117 Returns True if test case must be filtered, False otherwise.
118 """
119 if self.filters:
120 filters = self.filters.copy()
121 case_decorators = [cd for cd in case.decorators
122 if cd.__class__ in self.used_filters]
123
124 # Iterate over case decorators to check if needs to be filtered.
125 for cd in case_decorators:
126 if cd.filtrate(filters):
127 return True
128
129 # Case is missing one or more decorators for all the filters
130 # being used, so filter test case.
131 if filters:
132 return True
133
134 return False
135
136 def _getTestCase(self, testCaseClass, tcName):
137 if not hasattr(testCaseClass, '__oeqa_loader'):
138 # In order to support data_vars validation
139 # monkey patch the default setUp/tearDown{Class} to use
140 # the ones provided by OETestCase
141 setattr(testCaseClass, 'setUpClassMethod',
142 getattr(testCaseClass, 'setUpClass'))
143 setattr(testCaseClass, 'tearDownClassMethod',
144 getattr(testCaseClass, 'tearDownClass'))
145 setattr(testCaseClass, 'setUpClass',
146 testCaseClass._oeSetUpClass)
147 setattr(testCaseClass, 'tearDownClass',
148 testCaseClass._oeTearDownClass)
149
150 # In order to support decorators initialization
151 # monkey patch the default setUp/tearDown to use
152 # a setUpDecorators/tearDownDecorators that methods
153 # will call setUp/tearDown original methods.
154 setattr(testCaseClass, 'setUpMethod',
155 getattr(testCaseClass, 'setUp'))
156 setattr(testCaseClass, 'tearDownMethod',
157 getattr(testCaseClass, 'tearDown'))
158 setattr(testCaseClass, 'setUp', testCaseClass._oeSetUp)
159 setattr(testCaseClass, 'tearDown', testCaseClass._oeTearDown)
160
161 setattr(testCaseClass, '__oeqa_loader', True)
162
163 case = testCaseClass(tcName)
164 setattr(case, 'decorators', [])
165
166 return case
167
168 def loadTestsFromTestCase(self, testCaseClass):
169 """
170 Returns a suite of all tests cases contained in testCaseClass.
171 """
172 if issubclass(testCaseClass, unittest.suite.TestSuite):
173 raise TypeError("Test cases should not be derived from TestSuite." \
174 " Maybe you meant to derive from TestCase?")
175 if not issubclass(testCaseClass, self.caseClass):
176 raise TypeError("Test cases need to be derived from %s" % \
177 caseClass.__name__)
178
179
180 testCaseNames = self.getTestCaseNames(testCaseClass)
181 if not testCaseNames and hasattr(testCaseClass, 'runTest'):
182 testCaseNames = ['runTest']
183
184 suite = []
185 for tcName in testCaseNames:
186 case = self._getTestCase(testCaseClass, tcName)
187 # Filer by case id
188 if not (self.tests and not 'all' in self.tests
189 and not getCaseID(case) in self.tests):
190 self._handleTestCaseDecorators(case)
191
192 # Filter by decorators
193 if not self._filterTest(case):
194 self._registerTestCase(case)
195 suite.append(case)
196
197 return self.suiteClass(suite)
198
199 def discover(self):
200 big_suite = self.suiteClass()
201 for path in self.module_paths:
202 _find_duplicated_modules(big_suite, path)
203 suite = super(OETestLoader, self).discover(path,
204 pattern='*.py', top_level_dir=path)
205 big_suite.addTests(suite)
206
207 cases = None
208 discover_classes = [clss for clss in decoratorClasses
209 if issubclass(clss, OETestDiscover)]
210 for clss in discover_classes:
211 cases = clss.discover(self.tc._registry)
212
213 return self.suiteClass(cases) if cases else big_suite
214
215 # XXX After Python 3.5, remove backward compatibility hacks for
216 # use_load_tests deprecation via *args and **kws. See issue 16662.
217 if sys.version_info >= (3,5):
218 def loadTestsFromModule(self, module, *args, pattern=None, **kws):
219 if not self.modules or "all" in self.modules or \
220 module.__name__ in self.modules:
221 return super(OETestLoader, self).loadTestsFromModule(
222 module, *args, pattern=pattern, **kws)
223 else:
224 return self.suiteClass()
225 else:
226 def loadTestsFromModule(self, module, use_load_tests=True):
227 """
228 Returns a suite of all tests cases contained in module.
229 """
230 if not self.modules or "all" in self.modules or \
231 module.__name__ in self.modules:
232 return super(OETestLoader, self).loadTestsFromModule(
233 module, use_load_tests)
234 else:
235 return self.suiteClass()