summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorLeonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com>2015-11-18 15:04:17 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-12-01 21:32:05 +0000
commitcc34104561cd67dd4e4b3534abc59e3e3f1aa0e2 (patch)
treefdd7c44f50b39d9f460e44d7ba46997ed56d2ecd /scripts
parent06859de21bc6eced2bb188fbde7e9481349f8278 (diff)
downloadpoky-cc34104561cd67dd4e4b3534abc59e3e3f1aa0e2.tar.gz
oe-selftest: Enable code coverage on unit tests
Enable code coverage through the library 'python coverage'. In case the environment variable COVERAGE_PROCESS_START is present (one of the requisites for measuring sub-processes; the second one is including some coverage statements into the python sitecustomize.py file) it will be taken into account, otherwise it is exported with value '.coveragerc'. The latter value is a configuration file (also automatically created) with some default settings. Once tests are executed, a coverage report is shown on the log and the coverage output data is stored with name '.coverage.<args>' where '<args>' is the name of the unit tests executed or 'all_tests' when running with --run-all-tests. This output data can be latter used for better reporting using the same tool (coverage). As briefly indicate before, measuring sub-process implies setting the env variable COVERAGE_PROCESS_START (done automatically by the oe-selftest code with this patch if not already set) and creating a sitecustomize.py as explained on [1]. If either one of these is missing, complete coverage will be incomplete. Current measurements for 'oe-selftest --run-all-tests' indicate that current coverage is around 42 % taking into account BBLAYERS, bitbake and scripts folders. More details on [2], indicating the coverage per file/module. This tasks has been done together with Humberto Ibarra <humberto.ibarra.lopez@linux.intel.com> [YOCTO #8679] [1] http://coverage.readthedocs.org/en/latest/subprocess.html [2] https://bugzilla.yoctoproject.org/attachment.cgi?id=2854 (From OE-Core rev: b3feee2cefbbd98b66dc395b651f47c5028c80a0) Signed-off-by: Leonardo Sandoval <leonardo.sandoval.gonzalez@linux.intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/oe-selftest61
1 files changed, 61 insertions, 0 deletions
diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index 91e2dd2824..9679962ec0 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -30,6 +30,7 @@ import sys
30import unittest 30import unittest
31import logging 31import logging
32import argparse 32import argparse
33import subprocess
33 34
34sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib') 35sys.path.insert(0, os.path.dirname(os.path.realpath(__file__)) + '/lib')
35import scriptpath 36import scriptpath
@@ -70,6 +71,7 @@ def get_args_parser():
70 group.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests') 71 group.add_argument('--run-all-tests', required=False, action="store_true", dest="run_all_tests", default=False, help='Run all (unhidden) tests')
71 group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.') 72 group.add_argument('--list-modules', required=False, action="store_true", dest="list_modules", default=False, help='List all available test modules.')
72 group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.') 73 group.add_argument('--list-classes', required=False, action="store_true", dest="list_allclasses", default=False, help='List all available test classes.')
74 parser.add_argument('--coverage', action="store_true", help="Run code coverage when testing")
73 return parser 75 return parser
74 76
75 77
@@ -197,6 +199,42 @@ def main():
197 if not preflight_check(): 199 if not preflight_check():
198 return 1 200 return 1
199 201
202 if args.coverage:
203 try:
204 # check if user can do coverage
205 import coverage
206 log.info("Coverage is enabled")
207 except:
208 log.warn(("python coverage is not installed\n",
209 "Make sure you are also coverage takes into account sub-process\n",
210 "More info on https://pypi.python.org/pypi/coverage\n"))
211
212 # In case the user has not set the variable COVERAGE_PROCESS_START,
213 # create a default one and export it. The COVERAGE_PROCESS_START
214 # value indicates where the coverage configuration file resides
215 # More info on https://pypi.python.org/pypi/coverage
216 coverage_process_start = os.environ.get('COVERAGE_PROCESS_START')
217 if not coverage_process_start:
218 builddir = os.environ.get("BUILDDIR")
219 coveragerc = "%s/.coveragerc" % builddir
220 data_file = "%s/.coverage." % builddir
221 data_file += ((args.run_tests and ".".join(args.run_tests)) or
222 (args.run_all_tests and ".all_tests") or '')
223 if os.path.isfile(data_file):
224 os.remove(data_file)
225 with open(coveragerc, 'w') as cps:
226 cps.write("[run]\n")
227 cps.write("data_file = %s\n" % data_file)
228 cps.write("branch = True\n")
229 # Measure just BBLAYERS, scripts and bitbake folders
230 cps.write("source = \n")
231 for layer in get_bb_var('BBLAYERS').split():
232 cps.write(" %s\n" % layer)
233 cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__)))
234 cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake'))
235
236 coverage_process_start = os.environ["COVERAGE_PROCESS_START"] = coveragerc
237
200 testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False) 238 testslist = get_tests(exclusive_modules=(args.run_tests or []), include_hidden=False)
201 suite = unittest.TestSuite() 239 suite = unittest.TestSuite()
202 loader = unittest.TestLoader() 240 loader = unittest.TestLoader()
@@ -216,6 +254,29 @@ def main():
216 add_include() 254 add_include()
217 result = runner.run(suite) 255 result = runner.run(suite)
218 log.info("Finished") 256 log.info("Finished")
257
258 if args.coverage:
259 with open(coverage_process_start) as ccf:
260 log.info("Coverage configuration file (%s)" % coverage_process_start)
261 log.info("===========================")
262 log.info("\n%s" % "".join(ccf.readlines()))
263
264 try:
265 # depending on the version, coverage command is named 'python-coverage' or 'coverage',
266 # where the latter is for newer versions
267 coverage_cmd = "python-coverage"
268 subprocess.check_call(coverage_cmd, stderr=subprocess.PIPE, shell=True)
269 except subprocess.CalledProcessError:
270 coverage_cmd = "coverage"
271 pass
272
273 log.info("Coverage Report")
274 log.info("===============")
275 p = subprocess.Popen("%s report" % coverage_cmd, shell=True,
276 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
277 cov_output, cov_err = p.communicate()
278 log.info("\n%s" % cov_output)
279
219 if result.wasSuccessful(): 280 if result.wasSuccessful():
220 return 0 281 return 0
221 else: 282 else: