summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDaniel Istrate <daniel.alexandrux.istrate@intel.com>2016-01-05 16:53:34 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-01-07 13:40:17 +0000
commitc2bda6c000794b881274de953818e0de6635cc4a (patch)
tree409cabdb95cdd93acce5a995587e75eb165f56b4
parent8e1435eb717b22f15e898bee15f9c75128deae03 (diff)
downloadpoky-c2bda6c000794b881274de953818e0de6635cc4a.tar.gz
scripts/oe-selftest: Allow to run tests on random/all MACHINEs
Add an option for random MACHINE into oe-selftest: --machine [random/all] 1. random: will set a random MACHINE for each test 2. all: will run tests for all machines Custom machine sets only weak default values (??=) for MACHINE in machine.inc. This let test cases that require a specific MACHINE to be able to override it, using (?= or =). e.g.: oe-selftest --run-tests signing --machine random --> will run all tests switching MACHINE randomly for each test oe-selftest --run-tests signing --machine all --> for each machine will run all tests oe-selftest --run-all-tests --machine random Also update oeqa/selftest/base.py to accomodate this feature. Fix for [YOCTO #5880]. (From OE-Core rev: 4a9c3653eecd9a3c1b45bab5e16c8d679c55f8bd) Signed-off-by: Daniel Istrate <daniel.alexandrux.istrate@intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oeqa/selftest/base.py64
-rwxr-xr-xscripts/oe-selftest37
2 files changed, 86 insertions, 15 deletions
diff --git a/meta/lib/oeqa/selftest/base.py b/meta/lib/oeqa/selftest/base.py
index 9bddc23f87..dc937310dc 100644
--- a/meta/lib/oeqa/selftest/base.py
+++ b/meta/lib/oeqa/selftest/base.py
@@ -16,6 +16,8 @@ import errno
16import oeqa.utils.ftools as ftools 16import oeqa.utils.ftools as ftools
17from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer 17from oeqa.utils.commands import runCmd, bitbake, get_bb_var, get_test_layer
18from oeqa.utils.decorators import LogResults 18from oeqa.utils.decorators import LogResults
19from random import choice
20import glob
19 21
20@LogResults 22@LogResults
21class oeSelfTest(unittest.TestCase): 23class oeSelfTest(unittest.TestCase):
@@ -29,9 +31,10 @@ class oeSelfTest(unittest.TestCase):
29 self.testinc_path = os.path.join(self.builddir, "conf/selftest.inc") 31 self.testinc_path = os.path.join(self.builddir, "conf/selftest.inc")
30 self.local_bblayers_path = os.path.join(self.builddir, "conf/bblayers.conf") 32 self.local_bblayers_path = os.path.join(self.builddir, "conf/bblayers.conf")
31 self.testinc_bblayers_path = os.path.join(self.builddir, "conf/bblayers.inc") 33 self.testinc_bblayers_path = os.path.join(self.builddir, "conf/bblayers.inc")
34 self.machineinc_path = os.path.join(self.builddir, "conf/machine.inc")
32 self.testlayer_path = oeSelfTest.testlayer_path 35 self.testlayer_path = oeSelfTest.testlayer_path
33 self._extra_tear_down_commands = [] 36 self._extra_tear_down_commands = []
34 self._track_for_cleanup = [self.testinc_path] 37 self._track_for_cleanup = [self.testinc_path, self.testinc_bblayers_path, self.machineinc_path]
35 super(oeSelfTest, self).__init__(methodName) 38 super(oeSelfTest, self).__init__(methodName)
36 39
37 def setUp(self): 40 def setUp(self):
@@ -47,11 +50,25 @@ class oeSelfTest(unittest.TestCase):
47 for f in files: 50 for f in files:
48 if f == 'test_recipe.inc': 51 if f == 'test_recipe.inc':
49 os.remove(os.path.join(root, f)) 52 os.remove(os.path.join(root, f))
50 try: 53
51 os.remove(self.testinc_bblayers_path) 54 for incl_file in [self.testinc_bblayers_path, self.machineinc_path]:
52 except OSError as e: 55 try:
53 if e.errno != errno.ENOENT: 56 os.remove(incl_file)
54 raise 57 except OSError as e:
58 if e.errno != errno.ENOENT:
59 raise
60
61 # Get CUSTOMMACHINE from env (set by --machine argument to oe-selftest)
62 custommachine = os.getenv('CUSTOMMACHINE')
63 if custommachine:
64 if custommachine == 'random':
65 machine = get_random_machine()
66 else:
67 machine = custommachine
68 machine_conf = 'MACHINE ??= "%s"\n' % machine
69 self.set_machine_config(machine_conf)
70 print 'MACHINE: %s' % machine
71
55 # tests might need their own setup 72 # tests might need their own setup
56 # but if they overwrite this one they have to call 73 # but if they overwrite this one they have to call
57 # super each time, so let's give them an alternative 74 # super each time, so let's give them an alternative
@@ -99,11 +116,21 @@ class oeSelfTest(unittest.TestCase):
99 self.log.debug("Writing to: %s\n%s\n" % (self.testinc_path, data)) 116 self.log.debug("Writing to: %s\n%s\n" % (self.testinc_path, data))
100 ftools.write_file(self.testinc_path, data) 117 ftools.write_file(self.testinc_path, data)
101 118
119 custommachine = os.getenv('CUSTOMMACHINE')
120 if custommachine and 'MACHINE' in data:
121 machine = get_bb_var('MACHINE')
122 self.log.warning('MACHINE overridden: %s' % machine)
123
102 # append to <builddir>/conf/selftest.inc 124 # append to <builddir>/conf/selftest.inc
103 def append_config(self, data): 125 def append_config(self, data):
104 self.log.debug("Appending to: %s\n%s\n" % (self.testinc_path, data)) 126 self.log.debug("Appending to: %s\n%s\n" % (self.testinc_path, data))
105 ftools.append_file(self.testinc_path, data) 127 ftools.append_file(self.testinc_path, data)
106 128
129 custommachine = os.getenv('CUSTOMMACHINE')
130 if custommachine and 'MACHINE' in data:
131 machine = get_bb_var('MACHINE')
132 self.log.warning('MACHINE overridden: %s' % machine)
133
107 # remove data from <builddir>/conf/selftest.inc 134 # remove data from <builddir>/conf/selftest.inc
108 def remove_config(self, data): 135 def remove_config(self, data):
109 self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_path, data)) 136 self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_path, data))
@@ -151,3 +178,28 @@ class oeSelfTest(unittest.TestCase):
151 def remove_bblayers_config(self, data): 178 def remove_bblayers_config(self, data):
152 self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_bblayers_path, data)) 179 self.log.debug("Removing from: %s\n\%s\n" % (self.testinc_bblayers_path, data))
153 ftools.remove_from_file(self.testinc_bblayers_path, data) 180 ftools.remove_from_file(self.testinc_bblayers_path, data)
181
182 # write to <builddir>/conf/machine.inc
183 def set_machine_config(self, data):
184 self.log.debug("Writing to: %s\n%s\n" % (self.machineinc_path, data))
185 ftools.write_file(self.machineinc_path, data)
186
187
188def get_available_machines():
189 # Get a list of all available machines
190 bbpath = get_bb_var('BBPATH').split(':')
191 machines = []
192
193 for path in bbpath:
194 found_machines = glob.glob(os.path.join(path, 'conf', 'machine', '*.conf'))
195 if found_machines:
196 for i in found_machines:
197 # eg: '/home/<user>/poky/meta-intel/conf/machine/intel-core2-32.conf'
198 machines.append(os.path.splitext(os.path.basename(i))[0])
199
200 return machines
201
202
203def get_random_machine():
204 # Get a random machine
205 return choice(get_available_machines())
diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index e77768b292..4eb404b087 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -42,7 +42,7 @@ import argparse_oe
42import oeqa.selftest 42import oeqa.selftest
43import oeqa.utils.ftools as ftools 43import oeqa.utils.ftools as ftools
44from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer 44from oeqa.utils.commands import runCmd, get_bb_var, get_test_layer
45from oeqa.selftest.base import oeSelfTest 45from oeqa.selftest.base import oeSelfTest, get_available_machines
46 46
47def logger_create(): 47def logger_create():
48 log_file = "oe-selftest-" + t.strftime("%Y-%m-%d_%H:%M:%S") + ".log" 48 log_file = "oe-selftest-" + t.strftime("%Y-%m-%d_%H:%M:%S") + ".log"
@@ -86,6 +86,8 @@ def get_args_parser():
86 help='List all available tests.') 86 help='List all available tests.')
87 group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true", 87 group.add_argument('--list-tags', required=False, dest='list_tags', default=False, action="store_true",
88 help='List all tags that have been set to test cases.') 88 help='List all tags that have been set to test cases.')
89 parser.add_argument('--machine', required=False, dest='machine', choices=['random', 'all'], default=None,
90 help='Run tests on different machines (random/all).')
89 return parser 91 return parser
90 92
91 93
@@ -117,7 +119,7 @@ def add_include():
117 not in ftools.read_file(os.path.join(builddir, "conf/local.conf")): 119 not in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
118 log.info("Adding: \"include selftest.inc\" in local.conf") 120 log.info("Adding: \"include selftest.inc\" in local.conf")
119 ftools.append_file(os.path.join(builddir, "conf/local.conf"), \ 121 ftools.append_file(os.path.join(builddir, "conf/local.conf"), \
120 "\n#include added by oe-selftest.py\ninclude selftest.inc") 122 "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
121 123
122 if "#include added by oe-selftest.py" \ 124 if "#include added by oe-selftest.py" \
123 not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")): 125 not in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
@@ -133,13 +135,13 @@ def remove_include():
133 in ftools.read_file(os.path.join(builddir, "conf/local.conf")): 135 in ftools.read_file(os.path.join(builddir, "conf/local.conf")):
134 log.info("Removing the include from local.conf") 136 log.info("Removing the include from local.conf")
135 ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \ 137 ftools.remove_from_file(os.path.join(builddir, "conf/local.conf"), \
136 "#include added by oe-selftest.py\ninclude selftest.inc") 138 "\n#include added by oe-selftest.py\ninclude machine.inc\ninclude selftest.inc")
137 139
138 if "#include added by oe-selftest.py" \ 140 if "#include added by oe-selftest.py" \
139 in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")): 141 in ftools.read_file(os.path.join(builddir, "conf/bblayers.conf")):
140 log.info("Removing the include from bblayers.conf") 142 log.info("Removing the include from bblayers.conf")
141 ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \ 143 ftools.remove_from_file(os.path.join(builddir, "conf/bblayers.conf"), \
142 "#include added by oe-selftest.py\ninclude bblayers.inc") 144 "\n#include added by oe-selftest.py\ninclude bblayers.inc")
143 145
144def remove_inc_files(): 146def remove_inc_files():
145 try: 147 try:
@@ -151,10 +153,11 @@ def remove_inc_files():
151 except (AttributeError, OSError,) as e: # AttributeError may happen if BUILDDIR is not set 153 except (AttributeError, OSError,) as e: # AttributeError may happen if BUILDDIR is not set
152 pass 154 pass
153 155
154 try: 156 for incl_file in ['conf/bblayers.inc', 'conf/machine.inc']:
155 os.remove(os.path.join(os.environ.get("BUILDDIR"), "conf/bblayers.inc")) 157 try:
156 except: 158 os.remove(os.path.join(os.environ.get("BUILDDIR"), incl_file))
157 pass 159 except:
160 pass
158 161
159def get_tests(exclusive_modules=[], include_hidden=False): 162def get_tests(exclusive_modules=[], include_hidden=False):
160 testslist = [] 163 testslist = []
@@ -506,7 +509,23 @@ def main():
506 log.error(e) 509 log.error(e)
507 return 1 510 return 1
508 add_include() 511 add_include()
509 result = runner.run(suite) 512
513 if args.machine:
514 # Custom machine sets only weak default values (??=) for MACHINE in machine.inc
515 # This let test cases that require a specific MACHINE to be able to override it, using (?= or =)
516 log.info('Custom machine mode enabled. MACHINE set to %s' % args.machine)
517 if args.machine == 'random':
518 os.environ['CUSTOMMACHINE'] = 'random'
519 result = runner.run(suite)
520 else: # all
521 machines = get_available_machines()
522 for m in machines:
523 log.info('Run tests with custom MACHINE set to: %s' % m)
524 os.environ['CUSTOMMACHINE'] = m
525 result = runner.run(suite)
526 else:
527 result = runner.run(suite)
528
510 log.info("Finished") 529 log.info("Finished")
511 530
512 if result.wasSuccessful(): 531 if result.wasSuccessful():