summaryrefslogtreecommitdiffstats
path: root/meta/lib/oeqa
diff options
context:
space:
mode:
authorNathan Rossi <nathan@nathanrossi.com>2019-09-07 12:55:06 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-09-07 21:56:43 +0100
commit1220faf6659e404f6aa2c3155eb8840ac361c2b2 (patch)
tree4abb88044618505836b00434a3e6ef9609caacbb /meta/lib/oeqa
parent52ba1a3d44c89f4c38d6851504fe8b70c9e8e07e (diff)
downloadpoky-1220faf6659e404f6aa2c3155eb8840ac361c2b2.tar.gz
oeqa/core: Implement proper extra result collection and serialization
Implement handling of extra result (e.g. ptestresult) collection with the addition of a "extraresults" extraction function in OETestResult. In order to be able to serialize and deserialize the extraresults data, allow OETestResult add* calls to take a details kwarg. The subunit module can handle cross-process transfer of binary data for the details kwarg. With a TestResult proxy class to sit inbetween to encode and decode to and from json. (From OE-Core rev: b0831d43606415807af80e2aa1d0566d0b8c209c) Signed-off-by: Nathan Rossi <nathan@nathanrossi.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oeqa')
-rw-r--r--meta/lib/oeqa/core/runner.py41
-rw-r--r--meta/lib/oeqa/core/utils/concurrencytest.py61
2 files changed, 96 insertions, 6 deletions
diff --git a/meta/lib/oeqa/core/runner.py b/meta/lib/oeqa/core/runner.py
index 930620ea19..3060a00fbf 100644
--- a/meta/lib/oeqa/core/runner.py
+++ b/meta/lib/oeqa/core/runner.py
@@ -43,6 +43,7 @@ class OETestResult(_TestResult):
43 self.starttime = {} 43 self.starttime = {}
44 self.endtime = {} 44 self.endtime = {}
45 self.progressinfo = {} 45 self.progressinfo = {}
46 self.extraresults = {}
46 47
47 # Inject into tc so that TestDepends decorator can see results 48 # Inject into tc so that TestDepends decorator can see results
48 tc.results = self 49 tc.results = self
@@ -129,19 +130,51 @@ class OETestResult(_TestResult):
129 130
130 return 'UNKNOWN', None 131 return 'UNKNOWN', None
131 132
132 def addSuccess(self, test): 133 def extractExtraResults(self, test, details = None):
134 extraresults = None
135 if details is not None and "extraresults" in details:
136 extraresults = details.get("extraresults", {})
137 elif hasattr(test, "extraresults"):
138 extraresults = test.extraresults
139
140 if extraresults is not None:
141 for k, v in extraresults.items():
142 # handle updating already existing entries (e.g. ptestresults.sections)
143 if k in self.extraresults:
144 self.extraresults[k].update(v)
145 else:
146 self.extraresults[k] = v
147
148 def addError(self, test, *args, details = None):
149 self.extractExtraResults(test, details = details)
150 return super(OETestResult, self).addError(test, *args)
151
152 def addFailure(self, test, *args, details = None):
153 self.extractExtraResults(test, details = details)
154 return super(OETestResult, self).addFailure(test, *args)
155
156 def addSuccess(self, test, details = None):
133 #Added so we can keep track of successes too 157 #Added so we can keep track of successes too
134 self.successes.append((test, None)) 158 self.successes.append((test, None))
135 super(OETestResult, self).addSuccess(test) 159 self.extractExtraResults(test, details = details)
160 return super(OETestResult, self).addSuccess(test)
161
162 def addExpectedFailure(self, test, *args, details = None):
163 self.extractExtraResults(test, details = details)
164 return super(OETestResult, self).addExpectedFailure(test, *args)
165
166 def addUnexpectedSuccess(self, test, details = None):
167 self.extractExtraResults(test, details = details)
168 return super(OETestResult, self).addUnexpectedSuccess(test)
136 169
137 def logDetails(self, json_file_dir=None, configuration=None, result_id=None, 170 def logDetails(self, json_file_dir=None, configuration=None, result_id=None,
138 dump_streams=False): 171 dump_streams=False):
139 self.tc.logger.info("RESULTS:") 172 self.tc.logger.info("RESULTS:")
140 173
141 result = {} 174 result = self.extraresults
142 logs = {} 175 logs = {}
143 if hasattr(self.tc, "extraresults"): 176 if hasattr(self.tc, "extraresults"):
144 result = self.tc.extraresults 177 result.update(self.tc.extraresults)
145 178
146 for case_name in self.tc._registry['cases']: 179 for case_name in self.tc._registry['cases']:
147 case = self.tc._registry['cases'][case_name] 180 case = self.tc._registry['cases'][case_name]
diff --git a/meta/lib/oeqa/core/utils/concurrencytest.py b/meta/lib/oeqa/core/utils/concurrencytest.py
index 6bf7718863..fa6fa34b0e 100644
--- a/meta/lib/oeqa/core/utils/concurrencytest.py
+++ b/meta/lib/oeqa/core/utils/concurrencytest.py
@@ -21,6 +21,7 @@ import testtools
21import threading 21import threading
22import time 22import time
23import io 23import io
24import json
24import subunit 25import subunit
25 26
26from queue import Queue 27from queue import Queue
@@ -28,6 +29,8 @@ from itertools import cycle
28from subunit import ProtocolTestCase, TestProtocolClient 29from subunit import ProtocolTestCase, TestProtocolClient
29from subunit.test_results import AutoTimingTestResultDecorator 30from subunit.test_results import AutoTimingTestResultDecorator
30from testtools import ThreadsafeForwardingResult, iterate_tests 31from testtools import ThreadsafeForwardingResult, iterate_tests
32from testtools.content import Content
33from testtools.content_type import ContentType
31from oeqa.utils.commands import get_test_layer 34from oeqa.utils.commands import get_test_layer
32 35
33import bb.utils 36import bb.utils
@@ -70,6 +73,58 @@ class BBThreadsafeForwardingResult(ThreadsafeForwardingResult):
70 self.semaphore.release() 73 self.semaphore.release()
71 super(BBThreadsafeForwardingResult, self)._add_result_with_semaphore(method, test, *args, **kwargs) 74 super(BBThreadsafeForwardingResult, self)._add_result_with_semaphore(method, test, *args, **kwargs)
72 75
76class ProxyTestResult:
77 # a very basic TestResult proxy, in order to modify add* calls
78 def __init__(self, target):
79 self.result = target
80
81 def _addResult(self, method, test, *args, **kwargs):
82 return method(test, *args, **kwargs)
83
84 def addError(self, test, *args, **kwargs):
85 self._addResult(self.result.addError, test, *args, **kwargs)
86
87 def addFailure(self, test, *args, **kwargs):
88 self._addResult(self.result.addFailure, test, *args, **kwargs)
89
90 def addSuccess(self, test, *args, **kwargs):
91 self._addResult(self.result.addSuccess, test, *args, **kwargs)
92
93 def addExpectedFailure(self, test, *args, **kwargs):
94 self._addResult(self.result.addExpectedFailure, test, *args, **kwargs)
95
96 def addUnexpectedSuccess(self, test, *args, **kwargs):
97 self._addResult(self.result.addUnexpectedSuccess, test, *args, **kwargs)
98
99 def __getattr__(self, attr):
100 return getattr(self.result, attr)
101
102class ExtraResultsDecoderTestResult(ProxyTestResult):
103 def _addResult(self, method, test, *args, **kwargs):
104 if "details" in kwargs and "extraresults" in kwargs["details"]:
105 if isinstance(kwargs["details"]["extraresults"], Content):
106 kwargs = kwargs.copy()
107 kwargs["details"] = kwargs["details"].copy()
108 extraresults = kwargs["details"]["extraresults"]
109 data = bytearray()
110 for b in extraresults.iter_bytes():
111 data += b
112 extraresults = json.loads(data.decode())
113 kwargs["details"]["extraresults"] = extraresults
114 return method(test, *args, **kwargs)
115
116class ExtraResultsEncoderTestResult(ProxyTestResult):
117 def _addResult(self, method, test, *args, **kwargs):
118 if hasattr(test, "extraresults"):
119 extras = lambda : [json.dumps(test.extraresults).encode()]
120 kwargs = kwargs.copy()
121 if "details" not in kwargs:
122 kwargs["details"] = {}
123 else:
124 kwargs["details"] = kwargs["details"].copy()
125 kwargs["details"]["extraresults"] = Content(ContentType("application", "json", {'charset': 'utf8'}), extras)
126 return method(test, *args, **kwargs)
127
73# 128#
74# We have to patch subunit since it doesn't understand how to handle addError 129# We have to patch subunit since it doesn't understand how to handle addError
75# outside of a running test case. This can happen if classSetUp() fails 130# outside of a running test case. This can happen if classSetUp() fails
@@ -116,7 +171,9 @@ class ConcurrentTestSuite(unittest.TestSuite):
116 result.threadprogress = {} 171 result.threadprogress = {}
117 for i, (test, testnum) in enumerate(tests): 172 for i, (test, testnum) in enumerate(tests):
118 result.threadprogress[i] = [] 173 result.threadprogress[i] = []
119 process_result = BBThreadsafeForwardingResult(result, semaphore, i, testnum, totaltests) 174 process_result = BBThreadsafeForwardingResult(
175 ExtraResultsDecoderTestResult(result),
176 semaphore, i, testnum, totaltests)
120 # Force buffering of stdout/stderr so the console doesn't get corrupted by test output 177 # Force buffering of stdout/stderr so the console doesn't get corrupted by test output
121 # as per default in parent code 178 # as per default in parent code
122 process_result.buffer = True 179 process_result.buffer = True
@@ -231,7 +288,7 @@ def fork_for_tests(concurrency_num, suite):
231 # as per default in parent code 288 # as per default in parent code
232 subunit_client.buffer = True 289 subunit_client.buffer = True
233 subunit_result = AutoTimingTestResultDecorator(subunit_client) 290 subunit_result = AutoTimingTestResultDecorator(subunit_client)
234 process_suite.run(subunit_result) 291 process_suite.run(ExtraResultsEncoderTestResult(subunit_result))
235 if ourpid != os.getpid(): 292 if ourpid != os.getpid():
236 os._exit(0) 293 os._exit(0)
237 if newbuilddir: 294 if newbuilddir: