summaryrefslogtreecommitdiffstats
path: root/meta/lib
diff options
context:
space:
mode:
authorLeonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>2017-08-03 07:01:38 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-08-13 09:27:37 +0100
commit5ec91f9a325aef2898286fdca41b80a8e26320f2 (patch)
treec724a2e83c12358c37b63d600175b320676e6cc0 /meta/lib
parent582097c103a495807e10b3c5719aad31803decca (diff)
downloadpoky-5ec91f9a325aef2898286fdca41b80a8e26320f2.tar.gz
context: Include a command line argument to run all except certain tests
A new command line argument (-R, which is the oposite of current -r) that allows to run all test cases except the ones indicated through the command line. Some command line examples: * Run all except the distro test case: $ oe-selftest -R distrodata * Run all except the archiver test case and a single bblayers unit test $ oe-selftest -R archiver bblayers.BitbakeLayers.test_bitbakelayers_add_remove [YOCTO #11847] (From OE-Core rev: e40eeaa790b95d9c25832405c0b0d5b3a0d0292b) Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib')
-rw-r--r--meta/lib/oeqa/core/context.py20
-rw-r--r--meta/lib/oeqa/selftest/context.py11
2 files changed, 25 insertions, 6 deletions
diff --git a/meta/lib/oeqa/core/context.py b/meta/lib/oeqa/core/context.py
index 422e289992..acd547416f 100644
--- a/meta/lib/oeqa/core/context.py
+++ b/meta/lib/oeqa/core/context.py
@@ -41,6 +41,14 @@ class OETestContext(object):
41 41
42 return modules 42 return modules
43 43
44 def skipTests(self, skips):
45 if not skips:
46 return
47 for test in self.suites:
48 for skip in skips:
49 if test.id().startswith(skip):
50 setattr(test, 'setUp', lambda: test.skipTest('Skip by the command line argument "%s"' % skip))
51
44 def loadTests(self, module_paths, modules=[], tests=[], 52 def loadTests(self, module_paths, modules=[], tests=[],
45 modules_manifest="", modules_required=[], filters={}): 53 modules_manifest="", modules_required=[], filters={}):
46 if modules_manifest: 54 if modules_manifest:
@@ -50,9 +58,12 @@ class OETestContext(object):
50 modules_required, filters) 58 modules_required, filters)
51 self.suites = self.loader.discover() 59 self.suites = self.loader.discover()
52 60
53 def runTests(self): 61 def runTests(self, skips=[]):
54 self.runner = self.runnerClass(self, descriptions=False, verbosity=2) 62 self.runner = self.runnerClass(self, descriptions=False, verbosity=2)
55 63
64 # Dinamically skip those tests specified though arguments
65 self.skipTests(skips)
66
56 self._run_start_time = time.time() 67 self._run_start_time = time.time()
57 result = self.runner.run(self.suites) 68 result = self.runner.run(self.suites)
58 self._run_end_time = time.time() 69 self._run_end_time = time.time()
@@ -128,7 +139,8 @@ class OETestContextExecutor(object):
128 self.tc_kwargs = {} 139 self.tc_kwargs = {}
129 self.tc_kwargs['init'] = {} 140 self.tc_kwargs['init'] = {}
130 self.tc_kwargs['load'] = {} 141 self.tc_kwargs['load'] = {}
131 self.tc_kwargs['run'] = {} 142 self.tc_kwargs['list'] = {}
143 self.tc_kwargs['run'] = {}
132 144
133 self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args) 145 self.tc_kwargs['init']['logger'] = self._setup_logger(logger, args)
134 if args.test_data_file: 146 if args.test_data_file:
@@ -143,6 +155,8 @@ class OETestContextExecutor(object):
143 else: 155 else:
144 self.tc_kwargs['load']['modules'] = [] 156 self.tc_kwargs['load']['modules'] = []
145 157
158 self.tc_kwargs['run']['skips'] = []
159
146 self.module_paths = args.CASES_PATHS 160 self.module_paths = args.CASES_PATHS
147 161
148 def _pre_run(self): 162 def _pre_run(self):
@@ -159,7 +173,7 @@ class OETestContextExecutor(object):
159 sys.exit(1) 173 sys.exit(1)
160 174
161 if args.list_tests: 175 if args.list_tests:
162 rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run']) 176 rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
163 else: 177 else:
164 self._pre_run() 178 self._pre_run()
165 rc = self.tc.runTests(**self.tc_kwargs['run']) 179 rc = self.tc.runTests(**self.tc_kwargs['run'])
diff --git a/meta/lib/oeqa/selftest/context.py b/meta/lib/oeqa/selftest/context.py
index 990c761f29..9e90d3c256 100644
--- a/meta/lib/oeqa/selftest/context.py
+++ b/meta/lib/oeqa/selftest/context.py
@@ -25,14 +25,14 @@ class OESelftestTestContext(OETestContext):
25 self.custommachine = None 25 self.custommachine = None
26 self.config_paths = config_paths 26 self.config_paths = config_paths
27 27
28 def runTests(self, machine=None): 28 def runTests(self, machine=None, skips=[]):
29 if machine: 29 if machine:
30 self.custommachine = machine 30 self.custommachine = machine
31 if machine == 'random': 31 if machine == 'random':
32 self.custommachine = choice(self.machines) 32 self.custommachine = choice(self.machines)
33 self.logger.info('Run tests with custom MACHINE set to: %s' % \ 33 self.logger.info('Run tests with custom MACHINE set to: %s' % \
34 self.custommachine) 34 self.custommachine)
35 return super(OESelftestTestContext, self).runTests() 35 return super(OESelftestTestContext, self).runTests(skips)
36 36
37 def listTests(self, display_type, machine=None): 37 def listTests(self, display_type, machine=None):
38 return super(OESelftestTestContext, self).listTests(display_type) 38 return super(OESelftestTestContext, self).listTests(display_type)
@@ -51,6 +51,9 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
51 group.add_argument('-a', '--run-all-tests', default=False, 51 group.add_argument('-a', '--run-all-tests', default=False,
52 action="store_true", dest="run_all_tests", 52 action="store_true", dest="run_all_tests",
53 help='Run all (unhidden) tests') 53 help='Run all (unhidden) tests')
54 group.add_argument('-R', '--skip-tests', required=False, action='store',
55 nargs='+', dest="skips", default=None,
56 help='Run all (unhidden) tests except the ones specified. Format should be <module>[.<class>[.<test_method>]]')
54 group.add_argument('-r', '--run-tests', required=False, action='store', 57 group.add_argument('-r', '--run-tests', required=False, action='store',
55 nargs='+', dest="run_tests", default=None, 58 nargs='+', dest="run_tests", default=None,
56 help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>') 59 help='Select what tests to run (modules, classes or test methods). Format should be: <module>.<class>.<test_method>')
@@ -133,6 +136,8 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
133 copyfile(self.tc_kwargs['init']['config_paths']['bblayers'], 136 copyfile(self.tc_kwargs['init']['config_paths']['bblayers'],
134 self.tc_kwargs['init']['config_paths']['bblayers_backup']) 137 self.tc_kwargs['init']['config_paths']['bblayers_backup'])
135 138
139 self.tc_kwargs['run']['skips'] = args.skips
140
136 def _pre_run(self): 141 def _pre_run(self):
137 def _check_required_env_variables(vars): 142 def _check_required_env_variables(vars):
138 for var in vars: 143 for var in vars:
@@ -203,7 +208,7 @@ class OESelftestTestContextExecutor(OETestContextExecutor):
203 sys.exit(1) 208 sys.exit(1)
204 209
205 if args.list_tests: 210 if args.list_tests:
206 rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['run']) 211 rc = self.tc.listTests(args.list_tests, **self.tc_kwargs['list'])
207 else: 212 else:
208 self._pre_run() 213 self._pre_run()
209 rc = self.tc.runTests(**self.tc_kwargs['run']) 214 rc = self.tc.runTests(**self.tc_kwargs['run'])