diff options
Diffstat (limited to 'scripts/oe-selftest')
-rwxr-xr-x | scripts/oe-selftest | 172 |
1 files changed, 103 insertions, 69 deletions
diff --git a/scripts/oe-selftest b/scripts/oe-selftest index 08a5af3952..bd903f9e68 100755 --- a/scripts/oe-selftest +++ b/scripts/oe-selftest | |||
@@ -348,6 +348,55 @@ def list_tags(): | |||
348 | 348 | ||
349 | print 'Tags:\t%s' % ', '.join(str(x) for x in tags) | 349 | print 'Tags:\t%s' % ', '.join(str(x) for x in tags) |
350 | 350 | ||
351 | def coverage_setup(run_tests, run_all_tests): | ||
352 | """ Set up the coverage measurement for the testcases to be run """ | ||
353 | builddir = os.environ.get("BUILDDIR") | ||
354 | coveragerc = "%s/.coveragerc" % builddir | ||
355 | data_file = "%s/.coverage." % builddir | ||
356 | data_file += ((run_tests and '.'.join(run_tests)) or | ||
357 | (run_all_tests and "all_tests") or "") | ||
358 | if os.path.isfile(data_file): | ||
359 | os.remove(data_file) | ||
360 | with open(coveragerc, 'w') as cps: | ||
361 | cps.write("[run]\n") | ||
362 | cps.write("data_file = %s\n" % data_file) | ||
363 | cps.write("branch = True\n") | ||
364 | # Measure just BBLAYERS, scripts and bitbake folders | ||
365 | cps.write("source = \n") | ||
366 | for layer in get_bb_var('BBLAYERS').split(): | ||
367 | cps.write(" %s\n" % layer) | ||
368 | cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__))) | ||
369 | cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake')) | ||
370 | |||
371 | return coveragerc | ||
372 | |||
373 | def coverage_report(): | ||
374 | """ Loads the coverage data gathered and reports it back """ | ||
375 | try: | ||
376 | # Coverage4 uses coverage.Coverage | ||
377 | from coverage import Coverage | ||
378 | except: | ||
379 | # Coverage under version 4 uses coverage.coverage | ||
380 | from coverage import coverage as Coverage | ||
381 | |||
382 | import cStringIO as StringIO | ||
383 | from coverage.misc import CoverageException | ||
384 | |||
385 | cov_output = StringIO.StringIO() | ||
386 | # Creating the coverage data with the setting from the configuration file | ||
387 | cov = Coverage(config_file = os.environ.get('COVERAGE_PROCESS_START')) | ||
388 | try: | ||
389 | # Load data from the data file specified in the configuration | ||
390 | cov.load() | ||
391 | # Store report data in a StringIO variable | ||
392 | cov.report(file = cov_output, show_missing=False) | ||
393 | log.info("\n%s" % cov_output.getvalue()) | ||
394 | except CoverageException as e: | ||
395 | # Show problems with the reporting. Since Coverage4 not finding any data to report raises an exception | ||
396 | log.warn("%s" % str(e)) | ||
397 | finally: | ||
398 | cov_output.close() | ||
399 | |||
351 | 400 | ||
352 | def main(): | 401 | def main(): |
353 | parser = get_args_parser() | 402 | parser = get_args_parser() |
@@ -415,42 +464,6 @@ def main(): | |||
415 | if not preflight_check(): | 464 | if not preflight_check(): |
416 | return 1 | 465 | return 1 |
417 | 466 | ||
418 | if args.coverage: | ||
419 | try: | ||
420 | # check if user can do coverage | ||
421 | import coverage | ||
422 | log.info("Coverage is enabled") | ||
423 | except: | ||
424 | log.warn(("python coverage is not installed\n", | ||
425 | "Make sure you are also coverage takes into account sub-process\n", | ||
426 | "More info on https://pypi.python.org/pypi/coverage\n")) | ||
427 | |||
428 | # In case the user has not set the variable COVERAGE_PROCESS_START, | ||
429 | # create a default one and export it. The COVERAGE_PROCESS_START | ||
430 | # value indicates where the coverage configuration file resides | ||
431 | # More info on https://pypi.python.org/pypi/coverage | ||
432 | coverage_process_start = os.environ.get('COVERAGE_PROCESS_START') | ||
433 | if not coverage_process_start: | ||
434 | builddir = os.environ.get("BUILDDIR") | ||
435 | coveragerc = "%s/.coveragerc" % builddir | ||
436 | data_file = "%s/.coverage." % builddir | ||
437 | data_file += ((args.run_tests and ".".join(args.run_tests)) or | ||
438 | (args.run_all_tests and ".all_tests") or '') | ||
439 | if os.path.isfile(data_file): | ||
440 | os.remove(data_file) | ||
441 | with open(coveragerc, 'w') as cps: | ||
442 | cps.write("[run]\n") | ||
443 | cps.write("data_file = %s\n" % data_file) | ||
444 | cps.write("branch = True\n") | ||
445 | # Measure just BBLAYERS, scripts and bitbake folders | ||
446 | cps.write("source = \n") | ||
447 | for layer in get_bb_var('BBLAYERS').split(): | ||
448 | cps.write(" %s\n" % layer) | ||
449 | cps.write(" %s\n" % os.path.dirname(os.path.realpath(__file__))) | ||
450 | cps.write(" %s\n" % os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'bitbake')) | ||
451 | |||
452 | coverage_process_start = os.environ["COVERAGE_PROCESS_START"] = coveragerc | ||
453 | |||
454 | if args.run_tests_by: | 467 | if args.run_tests_by: |
455 | testslist = ts | 468 | testslist = ts |
456 | else: | 469 | else: |
@@ -459,7 +472,7 @@ def main(): | |||
459 | suite = unittest.TestSuite() | 472 | suite = unittest.TestSuite() |
460 | loader = unittest.TestLoader() | 473 | loader = unittest.TestLoader() |
461 | loader.sortTestMethodsUsing = None | 474 | loader.sortTestMethodsUsing = None |
462 | runner = unittest.TextTestRunner(verbosity=2, resultclass=StampedResult) | 475 | runner = unittest.TextTestRunner(verbosity=2, resultclass=buildResultClass(args)) |
463 | # we need to do this here, otherwise just loading the tests | 476 | # we need to do this here, otherwise just loading the tests |
464 | # will take 2 minutes (bitbake -e calls) | 477 | # will take 2 minutes (bitbake -e calls) |
465 | oeSelfTest.testlayer_path = get_test_layer() | 478 | oeSelfTest.testlayer_path = get_test_layer() |
@@ -475,43 +488,64 @@ def main(): | |||
475 | result = runner.run(suite) | 488 | result = runner.run(suite) |
476 | log.info("Finished") | 489 | log.info("Finished") |
477 | 490 | ||
478 | if args.coverage: | ||
479 | with open(coverage_process_start) as ccf: | ||
480 | log.info("Coverage configuration file (%s)" % coverage_process_start) | ||
481 | log.info("===========================") | ||
482 | log.info("\n%s" % "".join(ccf.readlines())) | ||
483 | |||
484 | try: | ||
485 | # depending on the version, coverage command is named 'python-coverage' or 'coverage', | ||
486 | # where the latter is for newer versions | ||
487 | coverage_cmd = "python-coverage" | ||
488 | subprocess.check_call(coverage_cmd, stderr=subprocess.PIPE, shell=True) | ||
489 | except subprocess.CalledProcessError: | ||
490 | coverage_cmd = "coverage" | ||
491 | pass | ||
492 | |||
493 | log.info("Coverage Report") | ||
494 | log.info("===============") | ||
495 | p = subprocess.Popen("%s report" % coverage_cmd, shell=True, | ||
496 | stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.PIPE) | ||
497 | cov_output, cov_err = p.communicate() | ||
498 | log.info("\n%s" % cov_output) | ||
499 | |||
500 | if result.wasSuccessful(): | 491 | if result.wasSuccessful(): |
501 | return 0 | 492 | return 0 |
502 | else: | 493 | else: |
503 | return 1 | 494 | return 1 |
504 | 495 | ||
505 | class StampedResult(unittest.TextTestResult): | 496 | def buildResultClass(args): |
506 | """ | 497 | """Build a Result Class to use in the testcase execution""" |
507 | Custom TestResult that prints the time when a test starts. As oe-selftest | 498 | |
508 | can take a long time (ie a few hours) to run, timestamps help us understand | 499 | class StampedResult(unittest.TextTestResult): |
509 | what tests are taking a long time to execute. | 500 | """ |
510 | """ | 501 | Custom TestResult that prints the time when a test starts. As oe-selftest |
511 | def startTest(self, test): | 502 | can take a long time (ie a few hours) to run, timestamps help us understand |
512 | import time | 503 | what tests are taking a long time to execute. |
513 | self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ") | 504 | If coverage is required, this class executes the coverage setup and reporting. |
514 | super(StampedResult, self).startTest(test) | 505 | """ |
506 | def startTest(self, test): | ||
507 | import time | ||
508 | self.stream.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " - ") | ||
509 | super(StampedResult, self).startTest(test) | ||
510 | |||
511 | def startTestRun(self): | ||
512 | """ Setup coverage before running any testcase """ | ||
513 | if args.coverage: | ||
514 | try: | ||
515 | # check if user can do coverage | ||
516 | import coverage | ||
517 | log.info("Coverage is enabled") | ||
518 | |||
519 | # In case the user has not set the variable COVERAGE_PROCESS_START, | ||
520 | # create a default one and export it. The COVERAGE_PROCESS_START | ||
521 | # value indicates where the coverage configuration file resides | ||
522 | # More info on https://pypi.python.org/pypi/coverage | ||
523 | if not os.environ.get('COVERAGE_PROCESS_START'): | ||
524 | os.environ['COVERAGE_PROCESS_START'] = coverage_setup(args.run_tests, args.run_all_tests) | ||
525 | |||
526 | self.coverage_installed = True | ||
527 | except: | ||
528 | log.warn('\n'.join(["python coverage is not installed", | ||
529 | "Make sure your coverage takes into account sub-process", | ||
530 | "More info on https://pypi.python.org/pypi/coverage"])) | ||
531 | self.coverage_installed = False | ||
532 | |||
533 | def stopTestRun(self): | ||
534 | """ Report coverage data after the testcases are run """ | ||
535 | |||
536 | if args.coverage and self.coverage_installed: | ||
537 | with open(os.environ['COVERAGE_PROCESS_START']) as ccf: | ||
538 | log.info("Coverage configuration file (%s)" % os.environ.get('COVERAGE_PROCESS_START')) | ||
539 | log.info("===========================") | ||
540 | log.info("\n%s" % "".join(ccf.readlines())) | ||
541 | |||
542 | log.info("Coverage Report") | ||
543 | log.info("===============") | ||
544 | |||
545 | coverage_report() | ||
546 | |||
547 | return StampedResult | ||
548 | |||
515 | 549 | ||
516 | if __name__ == "__main__": | 550 | if __name__ == "__main__": |
517 | try: | 551 | try: |