summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorHumberto Ibarra <humberto.ibarra.lopez@intel.com>2015-12-23 17:36:08 -0600
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-01-07 13:40:16 +0000
commit02d259c1f4d9cf19636867ebcabf8695cd79f797 (patch)
tree1cf8ecc7b732342e8c9b3b9a75a25879c4c212a2 /scripts
parent30c06a412e3a85eb2770f6aeb6f9b3ce632ca19d (diff)
downloadpoky-02d259c1f4d9cf19636867ebcabf8695cd79f797.tar.gz
scripts/oe-selftest: Remove extra coverage data added to unittests
Coverage data tracking initiates too early, causing coverage data from the oe-selftest environment setting to be added to each run. Even when no tests are run oe-selftest reports around 24% of coverage due to this extra data. Change the custom resultclass used by the TextTestRunner to one generated from the command arguments. The generated class processes coverage when needed, running coverage setup just before the first testcase is run and reporting after the last one finished. [Yocto #8846] (From OE-Core rev: d66a4c5cb77f745e973daf34b84724f91549f391) Signed-off-by: Humberto Ibarra <humberto.ibarra.lopez@intel.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/oe-selftest172
1 files changed, 103 insertions, 69 deletions
diff --git a/scripts/oe-selftest b/scripts/oe-selftest
index 08a5af3952..bd903f9e68 100755
--- a/scripts/oe-selftest
+++ b/scripts/oe-selftest
@@ -348,6 +348,55 @@ def list_tags():
348 348
349 print 'Tags:\t%s' % ', '.join(str(x) for x in tags) 349 print 'Tags:\t%s' % ', '.join(str(x) for x in tags)
350 350
351def coverage_setup(run_tests, run_all_tests):
352 """ Set up the coverage measurement for the testcases to be run """
353 builddir = os.environ.get("BUILDDIR")
354 coveragerc = "%s/.coveragerc" % builddir
355 data_file = "%s/.coverage." % builddir
356 data_file += ((run_tests and '.'.join(run_tests)) or
357 (run_all_tests and "all_tests") or "")
358 if os.path.isfile(data_file):
359 os.remove(data_file)
360 with open(coveragerc, 'w') as cps:
361 cps.write("[run]\n")
362 cps.write("data_file = %s\n" % data_file)
363 cps.write("branch = True\n")
364 # Measure just BBLAYERS, scripts and bitbake folders
365 cps.write("source = \n")
366 for layer in get_bb_var('BBLAYERS').split():
367 cps.write(" %s\n" % layer)
368 cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__)))
369 cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake'))
370
371 return coveragerc
372
373def coverage_report():
374 """ Loads the coverage data gathered and reports it back """
375 try:
376 # Coverage4 uses coverage.Coverage
377 from coverage import Coverage
378 except:
379 # Coverage under version 4 uses coverage.coverage
380 from coverage import coverage as Coverage
381
382 import cStringIO as StringIO
383 from coverage.misc import CoverageException
384
385 cov_output = StringIO.StringIO()
386 # Creating the coverage data with the setting from the configuration file
387 cov = Coverage(config_file = os.environ.get('COVERAGE_PROCESS_START'))
388 try:
389 # Load data from the data file specified in the configuration
390 cov.load()
391 # Store report data in a StringIO variable
392 cov.report(file = cov_output, show_missing=False)
393 log.info("\n%s" % cov_output.getvalue())
394 except CoverageException as e:
395 # Show problems with the reporting. Since Coverage4 not finding any data to report raises an exception
396 log.warn("%s" % str(e))
397 finally:
398 cov_output.close()
399
351 400
352def main(): 401def main():
353 parser = get_args_parser() 402 parser = get_args_parser()
@@ -415,42 +464,6 @@ def main():
415 if not preflight_check(): 464 if not preflight_check():
416 return 1 465 return 1
417 466
418 if args.coverage:
419 try:
420 # check if user can do coverage
421 import coverage
422 log.info("Coverage is enabled")
423 except:
424 log.warn(("python coverage is not installed\n",
425 "Make sure you are also coverage takes into account sub-process\n",
426 "More info on https://pypi.python.org/pypi/coverage\n"))
427
428 # In case the user has not set the variable COVERAGE_PROCESS_START,
429 # create a default one and export it. The COVERAGE_PROCESS_START
430 # value indicates where the coverage configuration file resides
431 # More info on https://pypi.python.org/pypi/coverage
432 coverage_process_start = os.environ.get('COVERAGE_PROCESS_START')
433 if not coverage_process_start:
434 builddir = os.environ.get("BUILDDIR")
435 coveragerc = "%s/.coveragerc" % builddir
436 data_file = "%s/.coverage." % builddir
437 data_file += ((args.run_tests and ".".join(args.run_tests)) or
438 (args.run_all_tests and ".all_tests") or '')
439 if os.path.isfile(data_file):
440 os.remove(data_file)
441 with open(coveragerc, 'w') as cps:
442 cps.write("[run]\n")
443 cps.write("data_file = %s\n" % data_file)
444 cps.write("branch = True\n")
445 # Measure just BBLAYERS, scripts and bitbake folders
446 cps.write("source = \n")
447 for layer in get_bb_var('BBLAYERS').split():
448 cps.write(" %s\n" % layer)
449 cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__)))
450 cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake'))
451
452 coverage_process_start = os.environ["COVERAGE_PROCESS_START"] = coveragerc
453
454 if args.run_tests_by: 467 if args.run_tests_by:
455 testslist = ts 468 testslist = ts
456 else: 469 else:
@@ -459,7 +472,7 @@ def main():
459 suite = unittest.TestSuite() 472 suite = unittest.TestSuite()
460 loader = unittest.TestLoader() 473 loader = unittest.TestLoader()
461 loader.sortTestMethodsUsing = None 474 loader.sortTestMethodsUsing = None
462 runner = unittest.TextTestRunner(verbosity=2, resultclass=StampedResult) 475 runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args))
463 # we need to do this here, otherwise just loading the tests 476 # we need to do this here, otherwise just loading the tests
464 # will take 2 minutes (bitbake -e calls) 477 # will take 2 minutes (bitbake -e calls)
465 oeSelfTest.testlayer_path = get_test_layer() 478 oeSelfTest.testlayer_path = get_test_layer()
@@ -475,43 +488,64 @@ def main():
475 result = runner.run(suite) 488 result = runner.run(suite)
476 log.info("Finished") 489 log.info("Finished")
477 490
478 if args.coverage:
479 with open(coverage_process_start) as ccf:
480 log.info("Coverage configuration file (%s)" % coverage_process_start)
481 log.info("===========================")
482 log.info("\n%s" % "".join(ccf.readlines()))
483
484 try:
485 # depending on the version, coverage command is named 'python-coverage' or 'coverage',
486 # where the latter is for newer versions
487 coverage_cmd = "python-coverage"
488 subprocess.check_call(coverage_cmd, stderr=subprocess.PIPE, shell=True)
489 except subprocess.CalledProcessError:
490 coverage_cmd = "coverage"
491 pass
492
493 log.info("Coverage Report")
494 log.info("===============")
495 p = subprocess.Popen("%s report" % coverage_cmd, shell=True,
496 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE)
497 cov_output, cov_err = p.communicate()
498 log.info("\n%s" % cov_output)
499
500 if result.wasSuccessful(): 491 if result.wasSuccessful():
501 return 0 492 return 0
502 else: 493 else:
503 return 1 494 return 1
504 495
505class StampedResult(unittest.TextTestResult): 496def buildResultClass(args):
506 """ 497 """Build a Result Class to use in the testcase execution"""
507 Custom TestResult that prints the time when a test starts. As oe-selftest 498
508 can take a long time (ie a few hours) to run, timestamps help us understand 499 class StampedResult(unittest.TextTestResult):
509 what tests are taking a long time to execute. 500 """
510 """ 501 Custom TestResult that prints the time when a test starts. As oe-selftest
511 def startTest(self, test): 502 can take a long time (ie a few hours) to run, timestamps help us understand
512 import time 503 what tests are taking a long time to execute.
513 self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ") 504 If coverage is required, this class executes the coverage setup and reporting.
514 super(StampedResult, self).startTest(test) 505 """
506 def startTest(self, test):
507 import time
508 self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ")
509 super(StampedResult, self).startTest(test)
510
511 def startTestRun(self):
512 """ Setup coverage before running any testcase """
513 if args.coverage:
514 try:
515 # check if user can do coverage
516 import coverage
517 log.info("Coverage is enabled")
518
519 # In case the user has not set the variable COVERAGE_PROCESS_START,
520 # create a default one and export it. The COVERAGE_PROCESS_START
521 # value indicates where the coverage configuration file resides
522 # More info on https://pypi.python.org/pypi/coverage
523 if not os.environ.get('COVERAGE_PROCESS_START'):
524 os.environ['COVERAGE_PROCESS_START'] = coverage_setup(args.run_tests, args.run_all_tests)
525
526 self.coverage_installed = True
527 except:
528 log.warn('\n'.join(["python coverage is not installed",
529 "Make sure your coverage takes into account sub-process",
530 "More info on https://pypi.python.org/pypi/coverage"]))
531 self.coverage_installed = False
532
533 def stopTestRun(self):
534 """ Report coverage data after the testcases are run """
535
536 if args.coverage and self.coverage_installed:
537 with open(os.environ['COVERAGE_PROCESS_START']) as ccf:
538 log.info("Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START'))
539 log.info("===========================")
540 log.info("\n%s" % "".join(ccf.readlines()))
541
542 log.info("Coverage Report")
543 log.info("===============")
544
545 coverage_report()
546
547 return StampedResult
548
515 549
516if __name__ == "__main__": 550if __name__ == "__main__":
517 try: 551 try: