summaryrefslogtreecommitdiffstats
path: root/meta/lib
diff options
context:
space:
mode:
authorHenry Kleynhans <hkleynhans@fb.com>2021-10-12 09:56:16 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2021-10-14 22:37:43 +0100
commit3579782a4415685063b28ba00e39271309106917 (patch)
tree853a7ee5a9f097405e82940e2ed800dce91795e8 /meta/lib
parentca8ef6bc38b8bfd772ff19931b49a076de5c1ec4 (diff)
downloadpoky-3579782a4415685063b28ba00e39271309106917.tar.gz
sstate: Switch to ZStandard compressor support
This patch switches the compressor from Gzip to ZStandard for ssate cache files. Zstandard compression provides a significant improvement in decompression speed as well as improvement in compression speed and disk usage over the 'tgz' format in use. Furthermore, its configurable compression level offers a trade-off between time spent compressing sstate cache files and disk space used by those files. The reduced disk usage also contributes to saving network traffic for those sharing their sstate cache with others. Zstandard should therefore be a good choice when: * disk space is at a premium * network speed / resources are limited * the CI server can sstate packages can be created at high compression * less CPU on the build server should be used for sstate decompression (From OE-Core rev: 0710e98f40ec7394567ede1f88f7c7bae306b925) Signed-off-by: Henry Kleynhans <hkleynhans@fb.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib')
-rw-r--r--meta/lib/oeqa/selftest/cases/signing.py8
-rw-r--r--meta/lib/oeqa/selftest/cases/sstatetests.py24
2 files changed, 16 insertions, 16 deletions
diff --git a/meta/lib/oeqa/selftest/cases/signing.py b/meta/lib/oeqa/selftest/cases/signing.py
index af7a0b8b45..6f3d4aeae9 100644
--- a/meta/lib/oeqa/selftest/cases/signing.py
+++ b/meta/lib/oeqa/selftest/cases/signing.py
@@ -159,13 +159,13 @@ class Signing(OESelftestTestCase):
159 bitbake('-c clean %s' % test_recipe) 159 bitbake('-c clean %s' % test_recipe)
160 bitbake('-c populate_lic %s' % test_recipe) 160 bitbake('-c populate_lic %s' % test_recipe)
161 161
162 recipe_sig = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz.sig') 162 recipe_sig = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tar.zst.sig')
163 recipe_tgz = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tgz') 163 recipe_archive = glob.glob(sstatedir + '/*/*/*:ed:*_populate_lic.tar.zst')
164 164
165 self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.') 165 self.assertEqual(len(recipe_sig), 1, 'Failed to find .sig file.')
166 self.assertEqual(len(recipe_tgz), 1, 'Failed to find .tgz file.') 166 self.assertEqual(len(recipe_archive), 1, 'Failed to find .tar.zst file.')
167 167
168 ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_tgz[0])) 168 ret = runCmd('gpg --homedir %s --verify %s %s' % (self.gpg_dir, recipe_sig[0], recipe_archive[0]))
169 # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30 169 # gpg: Signature made Thu 22 Oct 2015 01:45:09 PM EEST using RSA key ID 61EEFB30
170 # gpg: Good signature from "testuser (nocomment) <testuser@email.com>" 170 # gpg: Good signature from "testuser (nocomment) <testuser@email.com>"
171 self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.') 171 self.assertIn('gpg: Good signature from', ret.output, 'Package signed incorrectly.')
diff --git a/meta/lib/oeqa/selftest/cases/sstatetests.py b/meta/lib/oeqa/selftest/cases/sstatetests.py
index 874f439282..3dab607eeb 100644
--- a/meta/lib/oeqa/selftest/cases/sstatetests.py
+++ b/meta/lib/oeqa/selftest/cases/sstatetests.py
@@ -68,7 +68,7 @@ class SStateTests(SStateBase):
68 results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific) 68 results = self.search_sstate('|'.join(map(str, targets)), distro_specific, distro_nonspecific)
69 if distro_nonspecific: 69 if distro_nonspecific:
70 for r in results: 70 for r in results:
71 if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo", "_fetch.tgz.siginfo", "_unpack.tgz.siginfo", "_patch.tgz.siginfo")): 71 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo", "_fetch.tar.zst.siginfo", "_unpack.tar.zst.siginfo", "_patch.tar.zst.siginfo")):
72 continue 72 continue
73 file_tracker.append(r) 73 file_tracker.append(r)
74 else: 74 else:
@@ -98,15 +98,15 @@ class SStateTests(SStateBase):
98 bitbake(['-ccleansstate'] + targets) 98 bitbake(['-ccleansstate'] + targets)
99 99
100 bitbake(targets) 100 bitbake(targets)
101 tgz_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) 101 archives_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific)
102 self.assertTrue(tgz_created, msg="Could not find sstate .tgz files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_created))) 102 self.assertTrue(archives_created, msg="Could not find sstate .tar.zst files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_created)))
103 103
104 siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific) 104 siginfo_created = self.search_sstate('|'.join(map(str, [s + r'.*?\.siginfo$' for s in targets])), distro_specific, distro_nonspecific)
105 self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created))) 105 self.assertTrue(siginfo_created, msg="Could not find sstate .siginfo files for: %s (%s)" % (', '.join(map(str, targets)), str(siginfo_created)))
106 106
107 bitbake(['-ccleansstate'] + targets) 107 bitbake(['-ccleansstate'] + targets)
108 tgz_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific, distro_nonspecific) 108 archives_removed = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific, distro_nonspecific)
109 self.assertTrue(not tgz_removed, msg="do_cleansstate didn't remove .tgz sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(tgz_removed))) 109 self.assertTrue(not archives_removed, msg="do_cleansstate didn't remove .tar.zst sstate files for: %s (%s)" % (', '.join(map(str, targets)), str(archives_removed)))
110 110
111 def test_cleansstate_task_distro_specific_nonspecific(self): 111 def test_cleansstate_task_distro_specific_nonspecific(self):
112 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native'] 112 targets = ['binutils-cross-'+ self.tune_arch, 'binutils-native']
@@ -129,14 +129,14 @@ class SStateTests(SStateBase):
129 bitbake(['-ccleansstate'] + targets) 129 bitbake(['-ccleansstate'] + targets)
130 130
131 bitbake(targets) 131 bitbake(targets)
132 results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=False, distro_nonspecific=True) 132 results = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=False, distro_nonspecific=True)
133 filtered_results = [] 133 filtered_results = []
134 for r in results: 134 for r in results:
135 if r.endswith(("_populate_lic.tgz", "_populate_lic.tgz.siginfo")): 135 if r.endswith(("_populate_lic.tar.zst", "_populate_lic.tar.zst.siginfo")):
136 continue 136 continue
137 filtered_results.append(r) 137 filtered_results.append(r)
138 self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results))) 138 self.assertTrue(filtered_results == [], msg="Found distro non-specific sstate for: %s (%s)" % (', '.join(map(str, targets)), str(filtered_results)))
139 file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=True, distro_nonspecific=False) 139 file_tracker_1 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False)
140 self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) 140 self.assertTrue(len(file_tracker_1) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets)))
141 141
142 self.track_for_cleanup(self.distro_specific_sstate + "_old") 142 self.track_for_cleanup(self.distro_specific_sstate + "_old")
@@ -145,7 +145,7 @@ class SStateTests(SStateBase):
145 145
146 bitbake(['-cclean'] + targets) 146 bitbake(['-cclean'] + targets)
147 bitbake(targets) 147 bitbake(targets)
148 file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tgz$' for s in targets])), distro_specific=True, distro_nonspecific=False) 148 file_tracker_2 = self.search_sstate('|'.join(map(str, [s + r'.*?\.tar.zst$' for s in targets])), distro_specific=True, distro_nonspecific=False)
149 self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets))) 149 self.assertTrue(len(file_tracker_2) >= len(targets), msg = "Not all sstate files were created for: %s" % ', '.join(map(str, targets)))
150 150
151 not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2] 151 not_recreated = [x for x in file_tracker_1 if x not in file_tracker_2]
@@ -188,18 +188,18 @@ class SStateTests(SStateBase):
188 if not sstate_arch in sstate_archs_list: 188 if not sstate_arch in sstate_archs_list:
189 sstate_archs_list.append(sstate_arch) 189 sstate_archs_list.append(sstate_arch)
190 if target_config[idx] == target_config[-1]: 190 if target_config[idx] == target_config[-1]:
191 target_sstate_before_build = self.search_sstate(target + r'.*?\.tgz$') 191 target_sstate_before_build = self.search_sstate(target + r'.*?\.tar.zst$')
192 bitbake("-cclean %s" % target) 192 bitbake("-cclean %s" % target)
193 result = bitbake(target, ignore_status=True) 193 result = bitbake(target, ignore_status=True)
194 if target_config[idx] == target_config[-1]: 194 if target_config[idx] == target_config[-1]:
195 target_sstate_after_build = self.search_sstate(target + r'.*?\.tgz$') 195 target_sstate_after_build = self.search_sstate(target + r'.*?\.tar.zst$')
196 expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)] 196 expected_remaining_sstate += [x for x in target_sstate_after_build if x not in target_sstate_before_build if not any(pattern in x for pattern in ignore_patterns)]
197 self.remove_config(global_config[idx]) 197 self.remove_config(global_config[idx])
198 self.remove_recipeinc(target, target_config[idx]) 198 self.remove_recipeinc(target, target_config[idx])
199 self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output)) 199 self.assertEqual(result.status, 0, msg = "build of %s failed with %s" % (target, result.output))
200 200
201 runCmd("sstate-cache-management.sh -y --cache-dir=%s --remove-duplicated --extra-archs=%s" % (self.sstate_path, ','.join(map(str, sstate_archs_list)))) 201 runCmd("sstate-cache-management.sh -y --cache-dir=%s --remove-duplicated --extra-archs=%s" % (self.sstate_path, ','.join(map(str, sstate_archs_list))))
202 actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tgz$') if not any(pattern in x for pattern in ignore_patterns)] 202 actual_remaining_sstate = [x for x in self.search_sstate(target + r'.*?\.tar.zst$') if not any(pattern in x for pattern in ignore_patterns)]
203 203
204 actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate] 204 actual_not_expected = [x for x in actual_remaining_sstate if x not in expected_remaining_sstate]
205 self.assertFalse(actual_not_expected, msg="Files should have been removed but were not: %s" % ', '.join(map(str, actual_not_expected))) 205 self.assertFalse(actual_not_expected, msg="Files should have been removed but were not: %s" % ', '.join(map(str, actual_not_expected)))