diff options
author | Ross Burton <ross.burton@arm.com> | 2023-07-20 16:51:50 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-07-21 11:52:26 +0100 |
commit | b398c7653ec6272178dd6403dfcadf475f677bf5 (patch) | |
tree | 03500f603fb1ce3c115530cd868eade0d716bac6 | |
parent | aceb924845e0a77c878c3aea2941e2d3548b5a61 (diff) | |
download | poky-b398c7653ec6272178dd6403dfcadf475f677bf5.tar.gz |
oeqa/ltp: rewrote LTP testcase and parser
The LTP test reporting appears to be a little fragile so I tried to make
it more reliable.
Primarily this is done by not passing -p to runltp, which results in
machine-readable logfiles instead of human-readable. These are easier
to parse and have more context in, so we can also report correctly
skipped tests.
(From OE-Core rev: d585c6062fcf452e7288f6f8fb540fd92cbf5ea2)
Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r-- | meta/lib/oeqa/runtime/cases/ltp.py | 17 | ||||
-rw-r--r-- | meta/lib/oeqa/utils/logparser.py | 62 |
2 files changed, 51 insertions, 28 deletions
diff --git a/meta/lib/oeqa/runtime/cases/ltp.py b/meta/lib/oeqa/runtime/cases/ltp.py index a66d5d13d7..29c26d7d32 100644 --- a/meta/lib/oeqa/runtime/cases/ltp.py +++ b/meta/lib/oeqa/runtime/cases/ltp.py | |||
@@ -65,29 +65,34 @@ class LtpTest(LtpTestBase): | |||
65 | ltp_groups += ltp_fs | 65 | ltp_groups += ltp_fs |
66 | 66 | ||
67 | def runltp(self, ltp_group): | 67 | def runltp(self, ltp_group): |
68 | cmd = '/opt/ltp/runltp -f %s -p -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) | 68 | # LTP appends to log files, so ensure we start with a clean log |
69 | self.target.deleteFiles("/opt/ltp/results/", ltp_group) | ||
70 | |||
71 | cmd = '/opt/ltp/runltp -f %s -q -r /opt/ltp -l /opt/ltp/results/%s -I 1 -d /opt/ltp' % (ltp_group, ltp_group) | ||
72 | |||
69 | starttime = time.time() | 73 | starttime = time.time() |
70 | (status, output) = self.target.run(cmd) | 74 | (status, output) = self.target.run(cmd) |
71 | endtime = time.time() | 75 | endtime = time.time() |
72 | 76 | ||
77 | # Write the console log to disk for convenience | ||
73 | with open(os.path.join(self.ltptest_log_dir, "%s-raw.log" % ltp_group), 'w') as f: | 78 | with open(os.path.join(self.ltptest_log_dir, "%s-raw.log" % ltp_group), 'w') as f: |
74 | f.write(output) | 79 | f.write(output) |
75 | 80 | ||
81 | # Also put the console log into the test result JSON | ||
76 | self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output | 82 | self.extras['ltpresult.rawlogs']['log'] = self.extras['ltpresult.rawlogs']['log'] + output |
77 | 83 | ||
78 | # copy nice log from DUT | 84 | # Copy the machine-readable test results locally so we can parse it |
79 | dst = os.path.join(self.ltptest_log_dir, "%s" % ltp_group ) | 85 | dst = os.path.join(self.ltptest_log_dir, ltp_group) |
80 | remote_src = "/opt/ltp/results/%s" % ltp_group | 86 | remote_src = "/opt/ltp/results/%s" % ltp_group |
81 | (status, output) = self.target.copyFrom(remote_src, dst, True) | 87 | (status, output) = self.target.copyFrom(remote_src, dst, True) |
82 | msg = 'File could not be copied. Output: %s' % output | ||
83 | if status: | 88 | if status: |
89 | msg = 'File could not be copied. Output: %s' % output | ||
84 | self.target.logger.warning(msg) | 90 | self.target.logger.warning(msg) |
85 | 91 | ||
86 | parser = LtpParser() | 92 | parser = LtpParser() |
87 | results, sections = parser.parse(dst) | 93 | results, sections = parser.parse(dst) |
88 | 94 | ||
89 | runtime = int(endtime-starttime) | 95 | sections['duration'] = int(endtime-starttime) |
90 | sections['duration'] = runtime | ||
91 | self.sections[ltp_group] = sections | 96 | self.sections[ltp_group] = sections |
92 | 97 | ||
93 | failed_tests = {} | 98 | failed_tests = {} |
diff --git a/meta/lib/oeqa/utils/logparser.py b/meta/lib/oeqa/utils/logparser.py index 8054acc853..496d9e0c90 100644 --- a/meta/lib/oeqa/utils/logparser.py +++ b/meta/lib/oeqa/utils/logparser.py | |||
@@ -4,7 +4,7 @@ | |||
4 | # SPDX-License-Identifier: MIT | 4 | # SPDX-License-Identifier: MIT |
5 | # | 5 | # |
6 | 6 | ||
7 | import sys | 7 | import enum |
8 | import os | 8 | import os |
9 | import re | 9 | import re |
10 | 10 | ||
@@ -106,30 +106,48 @@ class PtestParser(object): | |||
106 | f.write(status + ": " + test_name + "\n") | 106 | f.write(status + ": " + test_name + "\n") |
107 | 107 | ||
108 | 108 | ||
109 | # ltp log parsing | 109 | class LtpParser: |
110 | class LtpParser(object): | 110 | """ |
111 | def __init__(self): | 111 | Parse the machine-readable LTP log output into a ptest-friendly data structure. |
112 | self.results = {} | 112 | """ |
113 | self.section = {'duration': "", 'log': ""} | ||
114 | |||
115 | def parse(self, logfile): | 113 | def parse(self, logfile): |
116 | test_regex = {} | 114 | results = {} |
117 | test_regex['PASSED'] = re.compile(r"PASS") | 115 | # Aaccumulate the duration here but as the log rounds quick tests down |
118 | test_regex['FAILED'] = re.compile(r"FAIL") | 116 | # to 0 seconds this is very much a lower bound. The caller can replace |
119 | test_regex['SKIPPED'] = re.compile(r"SKIP") | 117 | # the value. |
120 | 118 | section = {"duration": 0, "log": ""} | |
121 | with open(logfile, errors='replace') as f: | 119 | |
120 | class LtpExitCode(enum.IntEnum): | ||
121 | # Exit codes as defined in ltp/include/tst_res_flags.h | ||
122 | TPASS = 0 # Test passed flag | ||
123 | TFAIL = 1 # Test failed flag | ||
124 | TBROK = 2 # Test broken flag | ||
125 | TWARN = 4 # Test warning flag | ||
126 | TINFO = 16 # Test information flag | ||
127 | TCONF = 32 # Test not appropriate for configuration flag | ||
128 | |||
129 | with open(logfile, errors="replace") as f: | ||
130 | # Lines look like this: | ||
131 | # tag=cfs_bandwidth01 stime=1689762564 dur=0 exit=exited stat=32 core=no cu=0 cs=0 | ||
122 | for line in f: | 132 | for line in f: |
123 | for t in test_regex: | 133 | if not line.startswith("tag="): |
124 | result = test_regex[t].search(line) | 134 | continue |
125 | if result: | ||
126 | self.results[line.split()[0].strip()] = t | ||
127 | |||
128 | for test in self.results: | ||
129 | result = self.results[test] | ||
130 | self.section['log'] = self.section['log'] + ("%s: %s\n" % (result.strip()[:-2], test.strip())) | ||
131 | 135 | ||
132 | return self.results, self.section | 136 | values = dict(s.split("=") for s in line.strip().split()) |
137 | |||
138 | section["duration"] += int(values["dur"]) | ||
139 | exitcode = int(values["stat"]) | ||
140 | if values["exit"] == "exited" and exitcode == LtpExitCode.TCONF: | ||
141 | # Exited normally with the "invalid configuration" code | ||
142 | results[values["tag"]] = "SKIPPED" | ||
143 | elif exitcode == LtpExitCode.TPASS: | ||
144 | # Successful exit | ||
145 | results[values["tag"]] = "PASSED" | ||
146 | else: | ||
147 | # Other exit | ||
148 | results[values["tag"]] = "FAILED" | ||
149 | |||
150 | return results, section | ||
133 | 151 | ||
134 | 152 | ||
135 | # ltp Compliance log parsing | 153 | # ltp Compliance log parsing |