From ed5c12f11f7e18006f2de9eace333523ffb07002 Mon Sep 17 00:00:00 2001 From: Richard Purdie Date: Fri, 2 Nov 2018 13:13:43 +0000 Subject: oeqa/runtime/ptest: Inject results+logs into stored json results file This allows the ptest results from ptest-runner, run in an image to be transferred over to the resulting json results output. Each test is given a pass/skip/fail so individual results can be monitored and the raw log output from the ptest-runner is also dumped into the results json file as this means after the fact debugging becomes much easier. Currently the log output is not split up per test but that would make a good future enhancement. I attempted to implement this as python subTests however it failed as the output was too confusing, subTests don't support any kind of log output handling, subTest successes aren't logged and it was making things far more complex than they needed to be. We mark ptest-runner as "EXPECTEDFAILURE" since its unlikely every ptest will pass currently and we don't want that to fail the whole image test run. Its assumed there would be later analysis of the json output to determine regressions. We do have to change the test runner code so that 'unexpectedsuccess' is not a failure. Also, the test names are manipuated to remove spaces and brackets with "_" used as a replacement and any duplicate occurrences truncated. (From OE-Core rev: a13e088942e2a3c3521e98954a394e61a15234e8) (From OE-Core rev: 526ceab9d0e43f73635bb92e8dd7763ef75ad33b) Signed-off-by: Richard Purdie --- meta/lib/oeqa/core/runner.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'meta/lib/oeqa/core/runner.py') diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py index 7e0b61b587..9a64e494f4 100644 --- a/meta/lib/oeqa/core/runner.py +++ b/meta/lib/oeqa/core/runner.py @@ -107,7 +107,11 @@ class OETestResult(_TestResult): def logDetails(self, json_file_dir=None, configuration=None, result_id=None): self.tc.logger.info("RESULTS:") + result = {} + if hasattr(self.tc, "extraresults"): + result = self.tc.extraresults + for case_name in self.tc._registry['cases']: case = self.tc._registry['cases'][case_name] @@ -126,6 +130,10 @@ class OETestResult(_TestResult): tresultjsonhelper = OETestResultJSONHelper() tresultjsonhelper.dump_testresult_file(json_file_dir, configuration, result_id, result) + def wasSuccessful(self): + # Override as we unexpected successes aren't failures for us + return (len(self.failures) == len(self.errors) == 0) + class OEListTestsResult(object): def wasSuccessful(self): return True -- cgit v1.2.3-54-g00ecf