summaryrefslogtreecommitdiffstats
path: root/scripts/sstate-cache-management.sh
diff options
context:
space:
mode:
authorRobert Yang <liezhi.yang@windriver.com>2012-03-30 06:44:31 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-03-30 16:20:23 (GMT)
commit24f5fe8210dcd8b9d792ebb3a9302b7102f91497 (patch)
treebe960932f302b8a5a0ffa052ff92bee43d16c254 /scripts/sstate-cache-management.sh
parentdebd5d5bb1eba943830e1cbf3753712bbaa2ce35 (diff)
downloadpoky-24f5fe8210dcd8b9d792ebb3a9302b7102f91497.tar.gz
sstate-cache-management.sh: fixes and enhancement
Fix a few problems and enhance its functions, it shoud be more useful than before. * Search in meta and meta-* for archs, and grep AVAILTUNES for archs, (only search meta, and grep DEFAULTTUNE before), add the host arch. and also can search in extra layers with --extra-layer. * Reduce the analyzing time when remove duplicated files. It would cost more than 10 minutes to analyze 11,000 files before, now only needs about 50 seconds. * Check the access time rather than create time. * Need the user's confirm before really remove the file, or use --yes to assume yes. * Add --stamps-dir to keep sstate files which are used by the build directory, and remove others this can make the sstate cache dir clean, it is faster and should be useful than the --remove-duplicated. * Add --verbose to explain what is being done. * Add "-d" which is short for --remove-duplicated [YOCTO #2198] (From OE-Core rev: 769a000428e4b2462a4e6d8f179b5816b8ec2417) Signed-off-by: Robert Yang <liezhi.yang@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/sstate-cache-management.sh')
-rwxr-xr-xscripts/sstate-cache-management.sh289
1 files changed, 240 insertions, 49 deletions
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh
index d0e3abc..be185bb 100755
--- a/scripts/sstate-cache-management.sh
+++ b/scripts/sstate-cache-management.sh
@@ -16,22 +16,54 @@
16# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 16# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
17# 17#
18 18
19# Global vars
20cache_dir=
21confirm=
22total_deleted=0
23verbose=
24
19usage () { 25usage () {
20 cat << EOF 26 cat << EOF
21Welcome to sstate cache management utilities. 27Welcome to sstate cache management utilities.
22sstate-cache-management.sh <OPTION> 28sstate-cache-management.sh <OPTION>
23 29
24Options: 30Options:
25 --help, -h 31 -h, --help
26 Display this help and exit. 32 Display this help and exit.
27 33
28 --cache-dir=<sstate cache dir> 34 --cache-dir=<sstate cache dir>
29 Specify sstate cache directory, will use the environment 35 Specify sstate cache directory, will use the environment
30 variable SSTATE_CACHE_DIR if it is not specified. 36 variable SSTATE_CACHE_DIR if it is not specified.
31 37
32 --remove-duplicated 38 --extra-layer=<layer1>,<layer2>...<layern>
39 Specify the layer which will be used for searching the archs,
40 it will search the meta and meta-* layers in the top dir by
41 default, and will search meta, meta-*, <layer1>, <layer2>,
42 ...<layern> when specified. Use "," as the separator.
43
44 This is useless for --stamps-dir.
45
46 -d, --remove-duplicated
33 Remove the duplicated sstate cache files of one package, only 47 Remove the duplicated sstate cache files of one package, only
34 the newest one would be kept. 48 the newest one will be kept.
49
50 Conflicts with --stamps-dir.
51
52 --stamps-dir=<dir1>,<dir2>...<dirn>
53 Specify the build directory's stamps directories, the sstate
54 cache file which IS USED by these build diretories will be KEPT,
55 other sstate cache files in cache-dir will be removed. Use ","
56 as the separator. For example:
57 --stamps-dir=build1/tmp/stamps,build2/tmp/stamps
58
59 Conflicts with --remove-duplicated.
60
61 -y, --yes
62 Automatic yes to prompts; assume "yes" as answer to all prompts
63 and run non-interactively.
64
65 -v, --verbose
66 explain what is being done
35 67
36EOF 68EOF
37} 69}
@@ -41,6 +73,31 @@ if [ $# -lt 1 ]; then
41 exit 0 73 exit 0
42fi 74fi
43 75
76# Echo no files to remove
77no_files () {
78 echo No files to remove
79}
80
81# Echo nothing to do
82do_nothing () {
83 echo Nothing to do
84}
85
86# Read the input "y"
87read_confirm () {
88 echo -n "$total_deleted files will be removed! "
89 if [ "$confirm" != "y" ]; then
90 echo -n "Do you want to continue (y/n)? "
91 while read confirm; do
92 [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \
93 -o "$confirm" = "N" ] && break
94 echo -n "Invalid input \"$confirm\", please input 'y' or 'n': "
95 done
96 else
97 echo
98 fi
99}
100
44# Print error information and exit. 101# Print error information and exit.
45echo_error () { 102echo_error () {
46 echo "ERROR: $1" >&2 103 echo "ERROR: $1" >&2
@@ -49,45 +106,197 @@ echo_error () {
49 106
50# Remove the duplicated cache files for the pkg, keep the newest one 107# Remove the duplicated cache files for the pkg, keep the newest one
51remove_duplicated () { 108remove_duplicated () {
52 local all_suffixes="$1" 109
53 local ava_archs="$2" 110 local topdir
54 local total_deleted=0 111 local tunedirs
55 for suffix in $all_suffixes; do 112 local all_archs
113 local ava_archs
114 local arch
115 local file_names
116 local sstate_list
117 local fn_tmp
118 local list_suffix=`mktemp` || exit 1
119
120 # Find out the archs in all the layers
121 echo -n "Figuring out the archs in the layers ... "
122 topdir=$(dirname $(dirname $(readlink -e $0)))
123 tunedirs="`find $topdir/meta* $layers -path '*/meta*/conf/machine/include'`"
124 [ -n "$tunedirs" ] || echo_error "Can't find the tune directory"
125 all_archs=`grep -r -h "^AVAILTUNES " $tunedirs | sed -e 's/.*=//' -e 's/\"//g'`
126 # Add the qemu and native archs
127 # Use the "_" to substitute "-", e.g., x86-64 to x86_64
128 # Sort to remove the duplicated ones
129 all_archs=$(echo $all_archs qemuarm qemux86 qemumips qemuppc qemux86_64 $(uname -m) \
130 | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u)
131 echo "Done"
132
133 sstate_suffixes="deploy-rpm deploy-ipk deploy-deb deploy package populate-lic populate-sysroot"
134
135 cd $cache_dir || exit 1
136 # Save all the sstate files in a file
137 sstate_list=`mktemp` || exit 1
138 ls sstate-*.tgz >$sstate_list
139 echo -n "Figuring out the archs in the sstate cache dir ... "
140 for arch in $all_archs; do
141 grep -q -w $arch $sstate_list
142 [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
143 done
144 echo "Done"
145 echo "The following archs have been found in the cache dir:"
146 echo $ava_archs
147 echo ""
148
149 # Save the file list which needs to be removed
150 local remove_listdir=`mktemp -d` || exit 1
151
152 for suffix in $sstate_suffixes; do
153 # Save the file list to a file, some suffix's file may not exist
154 ls *_$suffix.tgz >$list_suffix 2>/dev/null
56 local deleted=0 155 local deleted=0
57 echo -n "Removing the sstate-xxx_$suffix.tgz ... " 156 echo -n "Figuring out the sstate-xxx_$suffix.tgz ... "
58 # sed twice to avoid the greedy match 157 # There are at list 6 dashes (-) after arch, use this to avoid the
158 # greedy match of sed.
59 file_names=`for arch in $ava_archs; do 159 file_names=`for arch in $ava_archs; do
60 ls sstate-*-$arch-*_$suffix.tgz 2>/dev/null | \ 160 sed -ne 's/^\(sstate-.*\)-'"$arch"'-.*-.*-.*-.*-.*-.*/\1/p' $list_suffix
61 sed -e 's/\(.*\)-'"$arch"'-.*/\1/' \
62 -e 's/\(.*\)-'"$arch"'-.*/\1/'
63 done | sort -u` 161 done | sort -u`
64 162
163 fn_tmp=`mktemp` || exit 1
65 for fn in $file_names; do 164 for fn in $file_names; do
165 [ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz"
66 for arch in $ava_archs; do 166 for arch in $ava_archs; do
67 ls $fn-$arch-*_$suffix.tgz 2>/dev/null >>/tmp/$fn 167 grep -h "^$fn-$arch-" $list_suffix >>$fn_tmp
68 done 168 done
69 # Also delete the .siginfo file 169 # Use the access time, also delete the .siginfo file
70 to_del=$(ls -t $(cat /tmp/$fn) | sed -n '1!p' | sed -e 'p' -e 's/$/.siginfo/') 170 to_del=$(ls -u $(cat $fn_tmp) | sed -n '1!p' | sed -e 'p' -e 's/$/.siginfo/')
71 rm -f $to_del 171 echo $to_del >>$remove_listdir/sstate-xxx_$suffix
72 let deleted=$deleted+`echo $to_del | wc -w` 172 let deleted=$deleted+`echo $to_del | wc -w`
73 rm -f /tmp/$fn 173 rm -f $fn_tmp
74 done 174 done
75 echo "($deleted files)" 175 echo "($deleted files will be removed)"
76 let total_deleted=$total_deleted+$deleted 176 let total_deleted=$total_deleted+$deleted
77 done 177 done
78 echo "$total_deleted files have been removed" 178 rm -f $list_suffix
179 if [ $total_deleted -gt 0 ]; then
180 read_confirm
181 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
182 for list in `ls $remove_listdir/`; do
183 echo -n "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... "
184 rm -f $verbose `cat $remove_listdir/$list`
185 echo "Done"
186 done
187 echo "$total_deleted files have been removed!"
188 else
189 do_nothing
190 fi
191 else
192 no_files
193 fi
194 [ -d $remove_listdir ] && rm -fr $remove_listdir
195}
196
197# Remove the sstate file by stamps dir, the file not used by the stamps dir
198# will be removed.
199rm_by_stamps (){
200
201 local cache_list=`mktemp` || exit 1
202 local keep_list=`mktemp` || exit 1
203 local mv_to_dir=`mktemp -d -p $cache_dir` || exit 1
204 local suffixes
205 local sums
206 local all_sums
207
208 suffixes="populate_sysroot populate_lic package_write_ipk \
209 package_write_rpm package_write_deb package deploy"
210
211 # Figure out all the md5sums in the stamps dir.
212 echo -n "Figuring out all the md5sums in stamps dir ... "
213 for i in $suffixes; do
214 sums=`find $stamps -maxdepth 2 -name "*\.do_$i\.sigdata.*" | \
215 sed 's#.*\.sigdata\.##' | sort -u`
216 all_sums="$all_sums $sums"
217 done
218 echo "Done"
219
220 # Save all the state file list to a file
221 ls $cache_dir/sstate-*.tgz >$cache_list
222
223 echo -n "Figuring out the files which will be removed ... "
224 for i in $all_sums; do
225 grep ".*-$i.*" $cache_list >>$keep_list
226 done
227 echo "Done"
228
229 if [ -s $keep_list ]; then
230 let total_deleted=(`cat $cache_list | wc -w` - `cat $keep_list | wc -l`)*2
231
232 if [ $total_deleted -gt 0 ]; then
233 read_confirm
234 if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then
235 echo "Removing sstate cache files ... ($total_deleted files)"
236 # Save the file which needs to be kept, remove the others,
237 # then move it back
238 for i in `cat $keep_list`; do
239 mv $i $mv_to_dir
240 mv $i.siginfo $mv_to_dir || true
241 done
242 rm -f $verbose $cache_dir/sstate-*.tgz
243 rm -f $verbose $cache_dir/sstate-*.tgz.siginfo
244 mv $mv_to_dir/* $cache_dir/
245 echo "$total_deleted files have been removed"
246 else
247 do_nothing
248 fi
249 else
250 no_files
251 fi
252 else
253 echo_error "All files in cache dir will be removed! Abort!"
254 fi
255
256 rm -f $cache_list
257 rm -f $keep_list
258 rmdir $mv_to_dir
79} 259}
80 260
81# Parse arguments 261# Parse arguments
82while [ -n "$1" ]; do 262while [ -n "$1" ]; do
83 case $1 in 263 case $1 in
84 --cache-dir=*) 264 --cache-dir=*)
85 cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' -e 's#/*$##' | xargs readlink -f` 265 cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e`
86 [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir" 266 [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir"
87 shift 267 shift
88 ;; 268 ;;
89 --remove-duplicated) 269 --remove-duplicated|-d)
90 rm_duplicated="yes" 270 rm_duplicated="y"
271 shift
272 ;;
273 --yes|-y)
274 confirm="y"
275 shift
276 ;;
277 --extra-layer=*)
278 extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'`
279 [ -n "$extra_layers" ] || echo_error "Invalid extra layer $i"
280 for i in $extra_layers; do
281 l=`readlink -e $i`
282 if [ -d "$l" ]; then
283 layers="$layers $l"
284 else
285 echo_error "Can't find layer $i"
286 fi
287 done
288 shift
289 ;;
290 --stamps-dir=*)
291 stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'`
292 [ -n "$stamps" ] || echo_error "Invalid stamps dir $i"
293 for i in $stamps; do
294 [ -d "$i" ] || echo_error "Invalid stamps dir $i"
295 done
296 shift
297 ;;
298 --verbose|-v)
299 verbose="-v"
91 shift 300 shift
92 ;; 301 ;;
93 --help|-h) 302 --help|-h)
@@ -95,8 +304,8 @@ while [ -n "$1" ]; do
95 exit 0 304 exit 0
96 ;; 305 ;;
97 *) 306 *)
98 echo "Invalid arguments $*" 307 echo "Invalid arguments $*"
99 echo_error "Try 'sstate-cache-management.sh -h' for more information." 308 echo_error "Try 'sstate-cache-management.sh -h' for more information."
100 ;; 309 ;;
101 esac 310 esac
102done 311done
@@ -104,32 +313,14 @@ done
104# sstate cache directory, use environment variable SSTATE_CACHE_DIR 313# sstate cache directory, use environment variable SSTATE_CACHE_DIR
105# if it was not specified, otherwise, error. 314# if it was not specified, otherwise, error.
106[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR 315[ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR
316[ -n "$cache_dir" ] || echo_error "No cache dir found!"
107[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\"" 317[ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\""
108 318
109cache_dir=`readlink -f $cache_dir` 319[ -n "$rm_duplicated" -a -n "$stamps" ] && \
320 echo_error "Can not use both --remove-duplicated and --stamps-dir"
110 321
111topdir=$(dirname $(dirname $(readlink -f $0))) 322[ "$rm_duplicated" = "y" ] && remove_duplicated
112tunedir=$topdir/meta/conf/machine/include 323[ -n "$stamps" ] && rm_by_stamps
113[ -d $tunedir ] || echo_error "Can't find the tune directory" 324[ -z "$rm_duplicated" -a -z "$stamps" ] && \
325 echo "What do you want to do?"
114 326
115# Use the "_" to substitute "-", e.g., x86-64 to x86_64
116all_archs=`grep -r DEFAULTTUNE $tunedir | \
117 sed -e 's/.*\"\(.*\)\"/\1/' -e 's/-/_/g' | sort -u`
118# Add the qemu archs
119all_archs="$all_archs qemuarm qemux86 qemumips qemuppc"
120
121all_suffixes="deploy-rpm deploy-ipk deploy-deb deploy package populate-lic populate-sysroot"
122
123cd $cache_dir
124
125echo "Figuring out the archs in the sstate cache dir ..."
126for arch in $all_archs; do
127 ls | grep -q -w $arch
128 [ $? -eq 0 ] && ava_archs="$ava_archs $arch"
129done
130echo "The following archs have been found in the sstate cache dir:"
131echo $ava_archs
132
133if [ "$rm_duplicated" == "yes" -a -n "$ava_archs" ]; then
134 remove_duplicated "$all_suffixes" "$ava_archs"
135fi