summaryrefslogtreecommitdiffstats
path: root/scripts/sstate-cache-management.sh
diff options
context:
space:
mode:
authorRobert Yang <liezhi.yang@windriver.com>2012-08-20 08:29:46 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-08-20 15:53:17 (GMT)
commit6163148c1798586f9b0543fc0f3b149fa15d1fc8 (patch)
treed66a3ad44378564f75348f0ab304c6f499740fcf /scripts/sstate-cache-management.sh
parentba8ba42b9f84499f9e514c7fb3e9dfacc7a7c0dd (diff)
downloadpoky-6163148c1798586f9b0543fc0f3b149fa15d1fc8.tar.gz
sstate-cache-management.sh: update for the SSTATE_MIRRORS
Several fixes: * We have put the sstate file to SSTATE_DIR/??/ currently, but the sstate file on the SSTATE_MIRRORS or the obsolete one is still in SSTATE_DIR/ (no subdir), update the script to support manage them. * Remove the related ".done" file in the SSTATE_DIR. * Add a "-L, --follow-symlink" which will remove both the symbol link and the destination file * Change the "ls -u file_list" (access time) to "ls -t file_list" (change tiem), since the "ls -u" and readlink will change the symlink's access time, which would make the result inconsistent. A solution is save the access time before every "ls -u" and "readlink", save it back after the command, but this would cause performance lost since it needs check each file and modify the symlink's status. Use the "-t" doesn't cause much different. [YOCTO #2897] (From OE-Core rev: 209ec08787981345a7f62b10a8a5c2ace0887c8e) Signed-off-by: Robert Yang <liezhi.yang@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/sstate-cache-management.sh')
-rwxr-xr-xscripts/sstate-cache-management.sh82
1 files changed, 68 insertions, 14 deletions
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index c3791d2..3a5980c 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -19,6 +19,7 @@
19# Global vars 19# Global vars
20cache_dir= 20cache_dir=
21confirm= 21confirm=
22fsym=
22total_deleted=0 23total_deleted=0
23verbose= 24verbose=
24 25
@@ -58,6 +59,9 @@ Options:
58 59
59 Conflicts with --remove-duplicated. 60 Conflicts with --remove-duplicated.
60 61
62 -L, --follow-symlink
63 Rmove both the symbol link and the destination file, default: no.
64
61 -y, --yes 65 -y, --yes
62 Automatic yes to prompts; assume "yes" as answer to all prompts 66 Automatic yes to prompts; assume "yes" as answer to all prompts
63 and run non-interactively. 67 and run non-interactively.
@@ -104,6 +108,47 @@ echo_error () {
104 exit 1 108 exit 1
105} 109}
106 110
111# Generate the remove list:
112#
113# * Add .done/.siginfo to the remove list
114# * Add destination of symlink to the remove list
115#
116# $1: output file, others: sstate cache file (.tgz)
117gen_rmlist (){
118 local rmlist_file="$1"
119 shift
120 local files="$@"
121 for i in $files; do
122 echo $i >> $rmlist_file
123 # Add the ".siginfo"
124 if [ -e $i.siginfo ]; then
125 echo $i.siginfo >> $rmlist_file
126 fi
127 # Add the destination of symlink
128 if [ -L "$i" ]; then
129 if [ "$fsym" = "y" ]; then
130 dest="`readlink -e $i`"
131 if [ -n "$dest" ]; then
132 echo $dest >> $rmlist_file
133 # Remove the .siginfo when .tgz is removed
134 if [ -f "$dest.siginfo" ]; then
135 echo $dest.siginfo >> $rmlist_file
136 fi
137 fi
138 fi
139 # Add the ".tgz.done" and ".siginfo.done" (may exist in the future)
140 base_fn="${i##/*/}"
141 t_fn="$base_fn.done"
142 s_fn="$base_fn.siginfo.done"
143 for d in $t_fn $s_fn; do
144 if [ -f $cache_dir/$d ]; then
145 echo $cache_dir/$d >> $rmlist_file
146 fi
147 done
148 fi
149 done
150}
151
107# Remove the duplicated cache files for the pkg, keep the newest one 152# Remove the duplicated cache files for the pkg, keep the newest one
108remove_duplicated () { 153remove_duplicated () {
109 154
@@ -134,7 +179,7 @@ remove_duplicated () {
134 179
135 # Save all the sstate files in a file 180 # Save all the sstate files in a file
136 sstate_list=`mktemp` || exit 1 181 sstate_list=`mktemp` || exit 1
137 find $cache_dir -path '*/??/sstate-*.tgz' >$sstate_list 182 find $cache_dir -name 'sstate-*.tgz' >$sstate_list
138 echo -n "Figuring out the archs in the sstate cache dir ... " 183 echo -n "Figuring out the archs in the sstate cache dir ... "
139 for arch in $all_archs; do 184 for arch in $all_archs; do
140 grep -q "\-$arch-" $sstate_list 185 grep -q "\-$arch-" $sstate_list
@@ -156,21 +201,22 @@ remove_duplicated () {
156 # There are at list 6 dashes (-) after arch, use this to avoid the 201 # There are at list 6 dashes (-) after arch, use this to avoid the
157 # greedy match of sed. 202 # greedy match of sed.
158 file_names=`for arch in $ava_archs; do 203 file_names=`for arch in $ava_archs; do
159 sed -ne 's#.*/../\(sstate-.*\)-'"$arch"'-.*-.*-.*-.*-.*-.*#\1#p' $list_suffix 204 sed -ne 's#.*/\(sstate-.*\)-'"$arch"'-.*-.*-.*-.*-.*-.*#\1#p' $list_suffix
160 done | sort -u` 205 done | sort -u`
161 206
162 fn_tmp=`mktemp` || exit 1 207 fn_tmp=`mktemp` || exit 1
208 rm_list="$remove_listdir/sstate-xxx_$suffix"
163 for fn in $file_names; do 209 for fn in $file_names; do
164 [ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz" 210 [ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz"
165 for arch in $ava_archs; do 211 for arch in $ava_archs; do
166 grep -h "/../$fn-$arch-" $list_suffix >>$fn_tmp 212 grep -h "/$fn-$arch-" $list_suffix >>$fn_tmp
167 done 213 done
168 # Use the access time, also delete the .siginfo file 214 # Use the access time, also delete the .siginfo file
169 to_del=$(ls -u $(cat $fn_tmp) | sed -n '1!p' | sed -e 'p' -e 's/$/.siginfo/') 215 to_del=$(ls -t $(cat $fn_tmp) | sed -n '1!p')
170 [ "$to_del" = "" ] || echo $to_del >>$remove_listdir/sstate-xxx_$suffix
171 let deleted=$deleted+`echo $to_del | wc -w`
172 rm -f $fn_tmp 216 rm -f $fn_tmp
217 gen_rmlist $rm_list "$to_del"
173 done 218 done
219 [ ! -s "$rm_list" ] || deleted=`cat $rm_list | wc -l`
174 echo "($deleted files will be removed)" 220 echo "($deleted files will be removed)"
175 let total_deleted=$total_deleted+$deleted 221 let total_deleted=$total_deleted+$deleted
176 done 222 done
@@ -213,33 +259,37 @@ rm_by_stamps (){
213 # Figure out all the md5sums in the stamps dir. 259 # Figure out all the md5sums in the stamps dir.
214 echo -n "Figuring out all the md5sums in stamps dir ... " 260 echo -n "Figuring out all the md5sums in stamps dir ... "
215 for i in $suffixes; do 261 for i in $suffixes; do
216 sums=`find $stamps -maxdepth 2 -name "*\.do_$i\.sigdata.*" | \ 262 # There is no "\.sigdata" but "_setcene" when it is mirrored
217 sed 's#.*\.sigdata\.##' | sort -u` 263 # from the SSTATE_MIRRORS, use them to figure out the sum.
264 sums=`find $stamps -maxdepth 2 -name "*.do_$i.*" \
265 -o -name "*.do_${i}_setscene.*" | \
266 sed -ne 's#.*_setscene\.##p' -e 's#.*\.sigdata\.##p' | \
267 sed -e 's#\..*##' | sort -u`
218 all_sums="$all_sums $sums" 268 all_sums="$all_sums $sums"
219 done 269 done
220 echo "Done" 270 echo "Done"
221 271
222 # Save all the state file list to a file 272 # Save all the state file list to a file
223 find $cache_dir -path '*/??/sstate-*.tgz' | sort -u -o $cache_list 273 find $cache_dir -name 'sstate-*.tgz' | sort -u -o $cache_list
224 274
225 echo -n "Figuring out the files which will be removed ... " 275 echo -n "Figuring out the files which will be removed ... "
226 for i in $all_sums; do 276 for i in $all_sums; do
227 grep ".*-$i.*" $cache_list >>$keep_list 277 grep ".*-${i}_*" $cache_list >>$keep_list
228 done 278 done
229 echo "Done" 279 echo "Done"
230 280
231 if [ -s $keep_list ]; then 281 if [ -s $keep_list ]; then
232 sort -u $keep_list -o $keep_list 282 sort -u $keep_list -o $keep_list
233 comm -1 -3 $keep_list $cache_list > $rm_list 283 to_del=`comm -1 -3 $keep_list $cache_list`
234 let total_deleted=(`cat $rm_list | wc -w`)*2 284 gen_rmlist $rm_list "$to_del"
235 285 let total_deleted=(`cat $rm_list | wc -w`)
236 if [ $total_deleted -gt 0 ]; then 286 if [ $total_deleted -gt 0 ]; then
237 read_confirm 287 read_confirm
238 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then 288 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
239 echo "Removing sstate cache files ... ($total_deleted files)" 289 echo "Removing sstate cache files ... ($total_deleted files)"
240 # Remove them one by one to avoid the argument list too long error 290 # Remove them one by one to avoid the argument list too long error
241 for i in `cat $rm_list`; do 291 for i in `cat $rm_list`; do
242 rm -f $verbose $i $i.siginfo 292 rm -f $verbose $i
243 done 293 done
244 echo "$total_deleted files have been removed" 294 echo "$total_deleted files have been removed"
245 else 295 else
@@ -273,6 +323,10 @@ while [ -n "$1" ]; do
273 confirm="y" 323 confirm="y"
274 shift 324 shift
275 ;; 325 ;;
326 --follow-symlink|-L)
327 fsym="y"
328 shift
329 ;;
276 --extra-layer=*) 330 --extra-layer=*)
277 extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'` 331 extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
278 [ -n "$extra_layers" ] || echo_error "Invalid extra layer $i" 332 [ -n "$extra_layers" ] || echo_error "Invalid extra layer $i"