summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Jansa <martin.jansa@gmail.com>2014-03-01 21:52:00 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-03-05 15:50:19 +0000
commit6bfa52afee2eb439ac049b7cf573b515cd74871e (patch)
tree31f9ce35898f77a278ef13ee37ab5c03c989b912
parent8ae183d517f9061457e73f94028acb55d9e9a620 (diff)
downloadpoky-6bfa52afee2eb439ac049b7cf573b515cd74871e.tar.gz
sstate-cache-management: improve shown numbers of removed files
* deleted files were counting .siginfo and .done files, but total_files were counting only .tgz archives so it could show confusing output like: Figuring out the sstate:xxx_packagedata.tgz ... (21098 from 15162 files for packagedata suffix will be removed) * split the counts and show both * fix .sigdata sometimes used instead of .siginfo * this will also show new "suffixes" which have only .siginfo files but no .tgz, e.g.: Figuring out the sstate:xxx_rm_work.tgz ... (0 from 0 .tgz files for rm_work suffix will be removed or 0 from 107630 when counting also .siginfo and .done files) because now we're storing siginfo files from all intermediate tasks, but they are not being removed by this script (From OE-Core rev: ec881997c748ed8bfb3fc75797367ce3599bd5b4) Signed-off-by: Martin Jansa <Martin.Jansa@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xscripts/sstate-cache-management.sh48
1 files changed, 26 insertions, 22 deletions
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index 34483d0b51..a4f66115e3 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -172,7 +172,7 @@ remove_duplicated () {
172 local ava_archs 172 local ava_archs
173 local arch 173 local arch
174 local file_names 174 local file_names
175 local sstate_list 175 local sstate_files_list
176 local fn_tmp 176 local fn_tmp
177 local list_suffix=`mktemp` || exit 1 177 local list_suffix=`mktemp` || exit 1
178 178
@@ -195,14 +195,14 @@ remove_duplicated () {
195 | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs" 195 | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) $extra_archs"
196 echo "Done" 196 echo "Done"
197 197
198 # Total number of files including sstate-, sigdata and .done files 198 # Total number of files including sstate-, .siginfo and .done files
199 total_files=`find $cache_dir -name 'sstate*' | wc -l` 199 total_files=`find $cache_dir -name 'sstate*' | wc -l`
200 # Save all the sstate files in a file 200 # Save all the sstate files in a file
201 sstate_list=`mktemp` || exit 1 201 sstate_files_list=`mktemp` || exit 1
202 find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz' >$sstate_list 202 find $cache_dir -name 'sstate:*:*:*:*:*:*:*.tgz*' >$sstate_files_list
203 203
204 echo -n "Figuring out the suffixes in the sstate cache dir ... " 204 echo -n "Figuring out the suffixes in the sstate cache dir ... "
205 sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz$%\1%g' $sstate_list | sort -u`" 205 sstate_suffixes="`sed 's%.*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^_]*_\([^:]*\)\.tgz.*%\1%g' $sstate_files_list | sort -u`"
206 echo "Done" 206 echo "Done"
207 echo "The following suffixes have been found in the cache dir:" 207 echo "The following suffixes have been found in the cache dir:"
208 echo $sstate_suffixes 208 echo $sstate_suffixes
@@ -211,10 +211,10 @@ remove_duplicated () {
211 # Using this SSTATE_PKGSPEC definition it's 6th colon separated field 211 # Using this SSTATE_PKGSPEC definition it's 6th colon separated field
212 # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" 212 # SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:"
213 for arch in $all_archs; do 213 for arch in $all_archs; do
214 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_list 214 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
215 [ $? -eq 0 ] && ava_archs="$ava_archs $arch" 215 [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
216 # ${builder_arch}_$arch used by toolchain sstate 216 # ${builder_arch}_$arch used by toolchain sstate
217 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_list 217 grep -q ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:${builder_arch}_$arch:[^:]*:[^:]*\.tgz$" $sstate_files_list
218 [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch" 218 [ $? -eq 0 ] && ava_archs="$ava_archs ${builder_arch}_$arch"
219 done 219 done
220 echo "Done" 220 echo "Done"
@@ -226,11 +226,13 @@ remove_duplicated () {
226 local remove_listdir=`mktemp -d` || exit 1 226 local remove_listdir=`mktemp -d` || exit 1
227 227
228 for suffix in $sstate_suffixes; do 228 for suffix in $sstate_suffixes; do
229 # Total number of files including sigdata and .done files 229 # Total number of files including .siginfo and .done files
230 total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.*" $sstate_list | wc -l 2>/dev/null` 230 total_files_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.*" $sstate_files_list | wc -l 2>/dev/null`
231 total_tgz_suffix=`grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.*.tgz$" $sstate_files_list | wc -l 2>/dev/null`
231 # Save the file list to a file, some suffix's file may not exist 232 # Save the file list to a file, some suffix's file may not exist
232 grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.tgz" $sstate_list >$list_suffix 2>/dev/null 233 grep ".*/sstate:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:]*:[^:_]*_$suffix.tgz$" $sstate_files_list >$list_suffix 2>/dev/null
233 local deleted=0 234 local deleted_tgz=0
235 local deleted_files=0
234 echo -n "Figuring out the sstate:xxx_$suffix.tgz ... " 236 echo -n "Figuring out the sstate:xxx_$suffix.tgz ... "
235 # Uniq BPNs 237 # Uniq BPNs
236 file_names=`for arch in $ava_archs ""; do 238 file_names=`for arch in $ava_archs ""; do
@@ -272,21 +274,23 @@ remove_duplicated () {
272 fi 274 fi
273 done 275 done
274 done 276 done
275 [ ! -s "$rm_list" ] || deleted=`cat $rm_list | wc -l` 277 [ -s "$rm_list" ] && deleted_tgz=`cat $rm_list | grep ".tgz$" | wc -l`
278 [ -s "$rm_list" ] && deleted_files=`cat $rm_list | wc -l`
276 [ -s "$rm_list" -a $debug -gt 0 ] && cat $rm_list 279 [ -s "$rm_list" -a $debug -gt 0 ] && cat $rm_list
277 echo "($deleted from $total_files_suffix files for $suffix suffix will be removed)" 280 echo "($deleted_tgz from $total_tgz_suffix .tgz files for $suffix suffix will be removed or $deleted_files from $total_files_suffix when counting also .siginfo and .done files)"
278 let total_deleted=$total_deleted+$deleted 281 let total_deleted=$total_deleted+$deleted_files
279 done 282 done
280 deleted=0 283 deleted_tgz=0
281 rm_old_list=$remove_listdir/sstate-old-filenames 284 rm_old_list=$remove_listdir/sstate-old-filenames
282 find $cache_dir -name 'sstate-*.tgz' >$rm_old_list 285 find $cache_dir -name 'sstate-*.tgz' >$rm_old_list
283 [ ! -s "$rm_old_list" ] || deleted=`cat $rm_old_list | wc -l` 286 [ -s "$rm_old_list" ] && deleted_tgz=`cat $rm_old_list | grep ".tgz$" | wc -l`
287 [ -s "$rm_old_list" ] && deleted_files=`cat $rm_old_list | wc -l`
284 [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list 288 [ -s "$rm_old_list" -a $debug -gt 0 ] && cat $rm_old_list
285 echo "($deleted files with old sstate-* filenames will be removed)" 289 echo "($deleted_tgz .tgz files with old sstate-* filenames will be removed or $deleted_files when counting also .siginfo and .done files)"
286 let total_deleted=$total_deleted+$deleted 290 let total_deleted=$total_deleted+$deleted_files
287 291
288 rm -f $list_suffix 292 rm -f $list_suffix
289 rm -f $sstate_list 293 rm -f $sstate_files_list
290 if [ $total_deleted -gt 0 ]; then 294 if [ $total_deleted -gt 0 ]; then
291 read_confirm 295 read_confirm
292 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then 296 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
@@ -325,17 +329,17 @@ rm_by_stamps (){
325 # Figure out all the md5sums in the stamps dir. 329 # Figure out all the md5sums in the stamps dir.
326 echo -n "Figuring out all the md5sums in stamps dir ... " 330 echo -n "Figuring out all the md5sums in stamps dir ... "
327 for i in $suffixes; do 331 for i in $suffixes; do
328 # There is no "\.sigdata" but "_setcene" when it is mirrored 332 # There is no "\.siginfo" but "_setcene" when it is mirrored
329 # from the SSTATE_MIRRORS, use them to figure out the sum. 333 # from the SSTATE_MIRRORS, use them to figure out the sum.
330 sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \ 334 sums=`find $stamps -maxdepth 3 -name "*.do_$i.*" \
331 -o -name "*.do_${i}_setscene.*" | \ 335 -o -name "*.do_${i}_setscene.*" | \
332 sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \ 336 sed -ne 's#.*_setscene\.##p' -e 's#.*\.siginfo\.##p' | \
333 sed -e 's#\..*##' | sort -u` 337 sed -e 's#\..*##' | sort -u`
334 all_sums="$all_sums $sums" 338 all_sums="$all_sums $sums"
335 done 339 done
336 echo "Done" 340 echo "Done"
337 341
338 # Total number of files including sstate-, sigdata and .done files 342 # Total number of files including sstate-, .siginfo and .done files
339 total_files=`find $cache_dir -name 'sstate*' | wc -l` 343 total_files=`find $cache_dir -name 'sstate*' | wc -l`
340 # Save all the state file list to a file 344 # Save all the state file list to a file
341 find $cache_dir -name 'sstate*.tgz' | sort -u -o $cache_list 345 find $cache_dir -name 'sstate*.tgz' | sort -u -o $cache_list