From 58155ea1a346dc5f9fcd65ed5c82584a3e6e6bec Mon Sep 17 00:00:00 2001 From: Martin Jansa Date: Mon, 5 Feb 2024 16:42:32 +0100 Subject: bitbake: bitbake-diffsigs: fix walking the task dependencies and show better error * when comparing 2 tmp/stamps/*do_configure.sigdata* I got TypeError("filename must be a str or bytes object, or a file") but both files I was comparing were Zstandard compressed data (v0.8+), Dictionary ID: None according to "file" with TypeError catched to show which file it failed to open I got better error which shows it was trying to read "do_prepare_recipe_sysroot.sigdata" file now and after a while you might notice that it's not just the expected file, but a dict with 'path', 'sstate', 'time'. Fix that in bitbake-diffsigs but keep the TypeError and add OSError in case it will eventually walk on file which isn't zstd compressed pipecompress throws CompressionError. ERROR: Failed to open {'path': '5.15.do_prepare_recipe_sysroot.sigdata.99b12a401341a0df7c3553cb00c87a7674295496bd5c25ed71764ee0d0fb8eb8', 'sstate': False, 'time': 1707136354.991718}: filename must be a str or bytes object, or a file Traceback (most recent call last): File "bitbake/bin/bitbake-diffsigs", line 192, in output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, recursecb, color=color) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/lib/bb/siggen.py", line 1039, in compare_sigfiles recout = recursecb(dep, a[dep], b[dep]) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/bin/bitbake-diffsigs", line 102, in recursecb out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/lib/bb/siggen.py", line 857, in compare_sigfiles raise err File "bitbake/lib/bb/siggen.py", line 853, in compare_sigfiles with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/lib/bb/compress/zstd.py", line 12, in open return bb.compress._pipecompress.open_wrap(ZstdFile, *args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/lib/bb/compress/_pipecompress.py", line 59, in open_wrap raise TypeError("filename must be a str or bytes object, or a file") TypeError: filename must be a str or bytes object, or a file * if I replace zstd file with just plaintext it fails with: $ echo foo > foo $ echo foo > bar $ bitbake-diffsigs foo bar zstd: /*stdin*\: unsupported format ERROR: Process died with 1 sys:1: ResourceWarning: unclosed file <_io.BufferedReader name='foo'> with this change it shows the name of the file which it failed to uncompress: $ bitbake-diffsigs foo bar zstd: /*stdin*\: unsupported format ERROR: Failed to open sigdata file 'foo': Process died with 1 ERROR: Process died with 1 sys:1: ResourceWarning: unclosed file <_io.BufferedReader name='foo'> (Bitbake rev: f3f843a4fd6e0a9e8f6edef5dd3cf1fce29c50ba) Signed-off-by: Richard Purdie --- bitbake/bin/bitbake-diffsigs | 2 +- bitbake/lib/bb/siggen.py | 24 ++++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs index a8f49191b0..8202c78623 100755 --- a/bitbake/bin/bitbake-diffsigs +++ b/bitbake/bin/bitbake-diffsigs @@ -99,7 +99,7 @@ def recursecb(key, hash1, hash2): elif hash2 not in hashfiles: recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2)) else: - out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color) + out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb, color=color) for change in out2: for line in change.splitlines(): recout.append(' ' + line) diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 5a584cadf9..58854aee76 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -849,10 +849,18 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False): formatparams.update(values) return formatstr.format(**formatparams) - with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: - a_data = json.load(f, object_hook=SetDecoder) - with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: - b_data = json.load(f, object_hook=SetDecoder) + try: + with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: + a_data = json.load(f, object_hook=SetDecoder) + except (TypeError, OSError) as err: + bb.error("Failed to open sigdata file '%s': %s" % (a, str(err))) + raise err + try: + with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: + b_data = json.load(f, object_hook=SetDecoder) + except (TypeError, OSError) as err: + bb.error("Failed to open sigdata file '%s': %s" % (b, str(err))) + raise err for data in [a_data, b_data]: handle_renames(data) @@ -1090,8 +1098,12 @@ def calc_taskhash(sigdata): def dump_sigfile(a): output = [] - with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: - a_data = json.load(f, object_hook=SetDecoder) + try: + with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: + a_data = json.load(f, object_hook=SetDecoder) + except (TypeError, OSError) as err: + bb.error("Failed to open sigdata file '%s': %s" % (a, str(err))) + raise err handle_renames(a_data) -- cgit v1.2.3-54-g00ecf