summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Jansa <martin.jansa@gmail.com>2024-02-05 16:42:32 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2024-02-08 10:59:32 +0000
commit58155ea1a346dc5f9fcd65ed5c82584a3e6e6bec (patch)
tree8418a12562b42c4f8795466eff4c593fc0b27c52
parent3ab7f0af56d0a12fa890faec4f24506669eb7e39 (diff)
downloadpoky-58155ea1a346dc5f9fcd65ed5c82584a3e6e6bec.tar.gz
bitbake: bitbake-diffsigs: fix walking the task dependencies and show better error
* when comparing 2 tmp/stamps/*do_configure.sigdata* I got TypeError("filename must be a str or bytes object, or a file") but both files I was comparing were Zstandard compressed data (v0.8+), Dictionary ID: None according to "file" with TypeError catched to show which file it failed to open I got better error which shows it was trying to read "do_prepare_recipe_sysroot.sigdata" file now and after a while you might notice that it's not just the expected file, but a dict with 'path', 'sstate', 'time'. Fix that in bitbake-diffsigs but keep the TypeError and add OSError in case it will eventually walk on file which isn't zstd compressed pipecompress throws CompressionError. ERROR: Failed to open {'path': '5.15.do_prepare_recipe_sysroot.sigdata.99b12a401341a0df7c3553cb00c87a7674295496bd5c25ed71764ee0d0fb8eb8', 'sstate': False, 'time': 1707136354.991718}: filename must be a str or bytes object, or a file Traceback (most recent call last): File "bitbake/bin/bitbake-diffsigs", line 192, in <module> output = bb.siggen.compare_sigfiles(options.sigdatafile1, options.sigdatafile2, recursecb, color=color) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/lib/bb/siggen.py", line 1039, in compare_sigfiles recout = recursecb(dep, a[dep], b[dep]) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/bin/bitbake-diffsigs", line 102, in recursecb out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/lib/bb/siggen.py", line 857, in compare_sigfiles raise err File "bitbake/lib/bb/siggen.py", line 853, in compare_sigfiles with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/lib/bb/compress/zstd.py", line 12, in open return bb.compress._pipecompress.open_wrap(ZstdFile, *args, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "bitbake/lib/bb/compress/_pipecompress.py", line 59, in open_wrap raise TypeError("filename must be a str or bytes object, or a file") TypeError: filename must be a str or bytes object, or a file * if I replace zstd file with just plaintext it fails with: $ echo foo > foo $ echo foo > bar $ bitbake-diffsigs foo bar zstd: /*stdin*\: unsupported format ERROR: Process died with 1 sys:1: ResourceWarning: unclosed file <_io.BufferedReader name='foo'> with this change it shows the name of the file which it failed to uncompress: $ bitbake-diffsigs foo bar zstd: /*stdin*\: unsupported format ERROR: Failed to open sigdata file 'foo': Process died with 1 ERROR: Process died with 1 sys:1: ResourceWarning: unclosed file <_io.BufferedReader name='foo'> (Bitbake rev: f3f843a4fd6e0a9e8f6edef5dd3cf1fce29c50ba) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xbitbake/bin/bitbake-diffsigs2
-rw-r--r--bitbake/lib/bb/siggen.py24
2 files changed, 19 insertions, 7 deletions
diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs
index a8f49191b0..8202c78623 100755
--- a/bitbake/bin/bitbake-diffsigs
+++ b/bitbake/bin/bitbake-diffsigs
@@ -99,7 +99,7 @@ def recursecb(key, hash1, hash2):
99 elif hash2 not in hashfiles: 99 elif hash2 not in hashfiles:
100 recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2)) 100 recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
101 else: 101 else:
102 out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color) 102 out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb, color=color)
103 for change in out2: 103 for change in out2:
104 for line in change.splitlines(): 104 for line in change.splitlines():
105 recout.append(' ' + line) 105 recout.append(' ' + line)
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 5a584cadf9..58854aee76 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -849,10 +849,18 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
849 formatparams.update(values) 849 formatparams.update(values)
850 return formatstr.format(**formatparams) 850 return formatstr.format(**formatparams)
851 851
852 with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: 852 try:
853 a_data = json.load(f, object_hook=SetDecoder) 853 with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
854 with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f: 854 a_data = json.load(f, object_hook=SetDecoder)
855 b_data = json.load(f, object_hook=SetDecoder) 855 except (TypeError, OSError) as err:
856 bb.error("Failed to open sigdata file '%s': %s" % (a, str(err)))
857 raise err
858 try:
859 with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f:
860 b_data = json.load(f, object_hook=SetDecoder)
861 except (TypeError, OSError) as err:
862 bb.error("Failed to open sigdata file '%s': %s" % (b, str(err)))
863 raise err
856 864
857 for data in [a_data, b_data]: 865 for data in [a_data, b_data]:
858 handle_renames(data) 866 handle_renames(data)
@@ -1090,8 +1098,12 @@ def calc_taskhash(sigdata):
1090def dump_sigfile(a): 1098def dump_sigfile(a):
1091 output = [] 1099 output = []
1092 1100
1093 with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f: 1101 try:
1094 a_data = json.load(f, object_hook=SetDecoder) 1102 with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
1103 a_data = json.load(f, object_hook=SetDecoder)
1104 except (TypeError, OSError) as err:
1105 bb.error("Failed to open sigdata file '%s': %s" % (a, str(err)))
1106 raise err
1095 1107
1096 handle_renames(a_data) 1108 handle_renames(a_data)
1097 1109