diff options
author | Ross Burton <ross.burton@arm.com> | 2023-09-23 14:04:11 +0100 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2023-09-26 10:25:42 +0100 |
commit | f15f38065023f3c375df96ca9773354cd54337ca (patch) | |
tree | 43689596b5f96fa34f0b414944215834f1341ad6 /meta/lib/oeqa/runtime | |
parent | 048e56f3b3d8be9213d90f89c9998477129c6e6b (diff) | |
download | poky-f15f38065023f3c375df96ca9773354cd54337ca.tar.gz |
oeqa/runtime/parselogs: parse the logs with Python, not grep
Instead of constructing huge grep statements, we can simply open the logs
in Python and do the relevant string operations directly.
The trick is to remember to casefold() all of the strings, so that the
"in" operator can be used.
Just one of the ignores needs to be adjusted because it uses a regular
expression and the new logic doesn't support that. This is handled
by simply reducing the size of the ignore match.
(From OE-Core rev: 78ae254c4a78a025a712281ce9de373cdccf5472)
Signed-off-by: Ross Burton <ross.burton@arm.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oeqa/runtime')
-rw-r--r-- | meta/lib/oeqa/runtime/cases/parselogs.py | 91 |
1 files changed, 41 insertions, 50 deletions
diff --git a/meta/lib/oeqa/runtime/cases/parselogs.py b/meta/lib/oeqa/runtime/cases/parselogs.py index 3980582dda..a805edd79d 100644 --- a/meta/lib/oeqa/runtime/cases/parselogs.py +++ b/meta/lib/oeqa/runtime/cases/parselogs.py | |||
@@ -4,9 +4,9 @@ | |||
4 | # SPDX-License-Identifier: MIT | 4 | # SPDX-License-Identifier: MIT |
5 | # | 5 | # |
6 | 6 | ||
7 | import collections | ||
7 | import os | 8 | import os |
8 | 9 | ||
9 | from subprocess import check_output | ||
10 | from shutil import rmtree | 10 | from shutil import rmtree |
11 | from oeqa.runtime.case import OERuntimeTestCase | 11 | from oeqa.runtime.case import OERuntimeTestCase |
12 | from oeqa.core.decorator.depends import OETestDepends | 12 | from oeqa.core.decorator.depends import OETestDepends |
@@ -64,6 +64,7 @@ common_errors = [ | |||
64 | "xf86OpenConsole: Switching VT failed", | 64 | "xf86OpenConsole: Switching VT failed", |
65 | "Failed to read LoaderConfigTimeoutOneShot variable, ignoring: Operation not supported", | 65 | "Failed to read LoaderConfigTimeoutOneShot variable, ignoring: Operation not supported", |
66 | "Failed to read LoaderEntryOneShot variable, ignoring: Operation not supported", | 66 | "Failed to read LoaderEntryOneShot variable, ignoring: Operation not supported", |
67 | "invalid BAR (can't size)", | ||
67 | ] | 68 | ] |
68 | 69 | ||
69 | x86_common = [ | 70 | x86_common = [ |
@@ -97,16 +98,13 @@ ignore_errors = { | |||
97 | 'default' : common_errors, | 98 | 'default' : common_errors, |
98 | 'qemux86' : [ | 99 | 'qemux86' : [ |
99 | 'Failed to access perfctr msr (MSR', | 100 | 'Failed to access perfctr msr (MSR', |
100 | 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', | ||
101 | ] + qemux86_common, | 101 | ] + qemux86_common, |
102 | 'qemux86-64' : qemux86_common, | 102 | 'qemux86-64' : qemux86_common, |
103 | 'qemumips' : [ | 103 | 'qemumips' : [ |
104 | 'Failed to load module "glx"', | 104 | 'Failed to load module "glx"', |
105 | 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', | ||
106 | 'cacheinfo: Failed to find cpu0 device node', | 105 | 'cacheinfo: Failed to find cpu0 device node', |
107 | ] + common_errors, | 106 | ] + common_errors, |
108 | 'qemumips64' : [ | 107 | 'qemumips64' : [ |
109 | 'pci 0000:00:00.0: [Firmware Bug]: reg 0x..: invalid BAR (can\'t size)', | ||
110 | 'cacheinfo: Failed to find cpu0 device node', | 108 | 'cacheinfo: Failed to find cpu0 device node', |
111 | ] + common_errors, | 109 | ] + common_errors, |
112 | 'qemuppc' : [ | 110 | 'qemuppc' : [ |
@@ -218,11 +216,13 @@ class ParseLogsTest(OERuntimeTestCase): | |||
218 | 'Ordering cycle found, skipping', | 216 | 'Ordering cycle found, skipping', |
219 | ]) | 217 | ]) |
220 | 218 | ||
219 | cls.errors = [s.casefold() for s in cls.errors] | ||
220 | |||
221 | try: | 221 | try: |
222 | cls.ignore_errors = ignore_errors[cls.td.get('MACHINE')] | 222 | cls.ignore_errors = [s.casefold() for s in ignore_errors[cls.td.get('MACHINE')]] |
223 | except KeyError: | 223 | except KeyError: |
224 | cls.logger.info('No ignore list found for this machine, using default') | 224 | cls.logger.info('No ignore list found for this machine, using default') |
225 | cls.ignore_errors = ignore_errors['default'] | 225 | cls.ignore_errors = [s.casefold() for s in ignore_errors['default']] |
226 | 226 | ||
227 | # Go through the log locations provided and if it's a folder | 227 | # Go through the log locations provided and if it's a folder |
228 | # create a list with all the .log files in it, if it's a file | 228 | # create a list with all the .log files in it, if it's a file |
@@ -263,30 +263,33 @@ class ParseLogsTest(OERuntimeTestCase): | |||
263 | logs = [f for f in dir_files if os.path.isfile(f)] | 263 | logs = [f for f in dir_files if os.path.isfile(f)] |
264 | return logs | 264 | return logs |
265 | 265 | ||
266 | # Build the grep command to be used with filters and exclusions | 266 | def get_context(self, lines, index, before=6, after=3): |
267 | def build_grepcmd(self, log): | 267 | """ |
268 | grepcmd = 'grep ' | 268 | Given a set of lines and the index of the line that is important, return |
269 | grepcmd += '-Ei "' | 269 | a number of lines surrounding that line. |
270 | for error in self.errors: | 270 | """ |
271 | grepcmd += r'\<' + error + r'\>' + '|' | 271 | last = len(lines) |
272 | grepcmd = grepcmd[:-1] | ||
273 | grepcmd += '" ' + str(log) + " | grep -Eiv \'" | ||
274 | 272 | ||
273 | start = index - before | ||
274 | end = index + after + 1 | ||
275 | 275 | ||
276 | for ignore_error in self.ignore_errors: | 276 | if start < 0: |
277 | ignore_error = ignore_error.replace('(', r'\(') | 277 | end -= start |
278 | ignore_error = ignore_error.replace(')', r'\)') | 278 | start = 0 |
279 | ignore_error = ignore_error.replace("'", '.') | 279 | if end > last: |
280 | ignore_error = ignore_error.replace('?', r'\?') | 280 | start -= end - last |
281 | ignore_error = ignore_error.replace('[', r'\[') | 281 | end = last |
282 | ignore_error = ignore_error.replace(']', r'\]') | ||
283 | ignore_error = ignore_error.replace('*', r'\*') | ||
284 | ignore_error = ignore_error.replace('0-9', '[0-9]') | ||
285 | grepcmd += ignore_error + '|' | ||
286 | grepcmd = grepcmd[:-1] | ||
287 | grepcmd += "\'" | ||
288 | 282 | ||
289 | return grepcmd | 283 | return lines[start:end] |
284 | |||
285 | def test_get_context(self): | ||
286 | """ | ||
287 | A test case for the test case. | ||
288 | """ | ||
289 | lines = list(range(0,10)) | ||
290 | self.assertEqual(self.get_context(lines, 0, 2, 1), [0, 1, 2, 3]) | ||
291 | self.assertEqual(self.get_context(lines, 5, 2, 1), [3, 4, 5, 6]) | ||
292 | self.assertEqual(self.get_context(lines, 9, 2, 1), [6, 7, 8, 9]) | ||
290 | 293 | ||
291 | def parse_logs(self, logs, lines_before=10, lines_after=10): | 294 | def parse_logs(self, logs, lines_before=10, lines_after=10): |
292 | """ | 295 | """ |
@@ -296,31 +299,19 @@ class ParseLogsTest(OERuntimeTestCase): | |||
296 | Returns a dictionary of log filenames to a dictionary of error lines to | 299 | Returns a dictionary of log filenames to a dictionary of error lines to |
297 | the error context (controlled by @lines_before and @lines_after). | 300 | the error context (controlled by @lines_before and @lines_after). |
298 | """ | 301 | """ |
299 | results = {} | 302 | results = collections.defaultdict(dict) |
300 | rez = [] | ||
301 | grep_output = '' | ||
302 | 303 | ||
303 | for log in logs: | 304 | for log in logs: |
304 | result = None | 305 | with open(log) as f: |
305 | thegrep = self.build_grepcmd(log) | 306 | lines = f.readlines() |
306 | |||
307 | try: | ||
308 | result = check_output(thegrep, shell=True).decode('utf-8') | ||
309 | except: | ||
310 | pass | ||
311 | 307 | ||
312 | if result is not None: | 308 | for i, line in enumerate(lines): |
313 | results[log] = {} | 309 | line = line.strip() |
314 | rez = result.splitlines() | 310 | line_lower = line.casefold() |
315 | 311 | ||
316 | for xrez in rez: | 312 | if any(keyword in line_lower for keyword in self.errors): |
317 | try: | 313 | if not any(ignore in line_lower for ignore in self.ignore_errors): |
318 | cmd = ['grep', '-F', xrez, '-B', str(lines_before)] | 314 | results[log][line] = "".join(self.get_context(lines, i, lines_before, lines_after)) |
319 | cmd += ['-A', str(lines_after), log] | ||
320 | grep_output = check_output(cmd).decode('utf-8') | ||
321 | except: | ||
322 | pass | ||
323 | results[log][xrez]=grep_output | ||
324 | 315 | ||
325 | return results | 316 | return results |
326 | 317 | ||
@@ -342,9 +333,9 @@ class ParseLogsTest(OERuntimeTestCase): | |||
342 | self.msg += '-----------------------\n' | 333 | self.msg += '-----------------------\n' |
343 | for error in result[log]: | 334 | for error in result[log]: |
344 | errcount += 1 | 335 | errcount += 1 |
345 | self.msg += 'Central error: ' + str(error) + '\n' | 336 | self.msg += 'Central error: ' + error + '\n' |
346 | self.msg += '***********************\n' | 337 | self.msg += '***********************\n' |
347 | self.msg += result[str(log)][str(error)] + '\n' | 338 | self.msg += result[log][error] + '\n' |
348 | self.msg += '***********************\n' | 339 | self.msg += '***********************\n' |
349 | self.msg += '%s errors found in logs.' % errcount | 340 | self.msg += '%s errors found in logs.' % errcount |
350 | self.assertEqual(errcount, 0, msg=self.msg) | 341 | self.assertEqual(errcount, 0, msg=self.msg) |