diff options
| -rwxr-xr-x | scripts/sstate-cache-management.sh | 289 |
1 files changed, 240 insertions, 49 deletions
diff --git a/scripts/sstate-cache-management.sh b/scripts/sstate-cache-management.sh index d0e3abc82..be185bbb4 100755 --- a/scripts/sstate-cache-management.sh +++ b/scripts/sstate-cache-management.sh @@ -16,22 +16,54 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # +# Global vars +cache_dir= +confirm= +total_deleted=0 +verbose= + usage () { cat << EOF Welcome to sstate cache management utilities. sstate-cache-management.sh <OPTION> Options: - --help, -h + -h, --help Display this help and exit. --cache-dir=<sstate cache dir> Specify sstate cache directory, will use the environment variable SSTATE_CACHE_DIR if it is not specified. - --remove-duplicated + --extra-layer=<layer1>,<layer2>...<layern> + Specify the layer which will be used for searching the archs, + it will search the meta and meta-* layers in the top dir by + default, and will search meta, meta-*, <layer1>, <layer2>, + ...<layern> when specified. Use "," as the separator. + + This is useless for --stamps-dir. + + -d, --remove-duplicated Remove the duplicated sstate cache files of one package, only - the newest one would be kept. + the newest one will be kept. + + Conflicts with --stamps-dir. + + --stamps-dir=<dir1>,<dir2>...<dirn> + Specify the build directory's stamps directories, the sstate + cache file which IS USED by these build diretories will be KEPT, + other sstate cache files in cache-dir will be removed. Use "," + as the separator. For example: + --stamps-dir=build1/tmp/stamps,build2/tmp/stamps + + Conflicts with --remove-duplicated. + + -y, --yes + Automatic yes to prompts; assume "yes" as answer to all prompts + and run non-interactively. + + -v, --verbose + explain what is being done EOF } @@ -41,6 +73,31 @@ if [ $# -lt 1 ]; then exit 0 fi +# Echo no files to remove +no_files () { + echo No files to remove +} + +# Echo nothing to do +do_nothing () { + echo Nothing to do +} + +# Read the input "y" +read_confirm () { + echo -n "$total_deleted files will be removed! " + if [ "$confirm" != "y" ]; then + echo -n "Do you want to continue (y/n)? " + while read confirm; do + [ "$confirm" = "Y" -o "$confirm" = "y" -o "$confirm" = "n" \ + -o "$confirm" = "N" ] && break + echo -n "Invalid input \"$confirm\", please input 'y' or 'n': " + done + else + echo + fi +} + # Print error information and exit. echo_error () { echo "ERROR: $1" >&2 @@ -49,45 +106,197 @@ echo_error () { # Remove the duplicated cache files for the pkg, keep the newest one remove_duplicated () { - local all_suffixes="$1" - local ava_archs="$2" - local total_deleted=0 - for suffix in $all_suffixes; do + + local topdir + local tunedirs + local all_archs + local ava_archs + local arch + local file_names + local sstate_list + local fn_tmp + local list_suffix=`mktemp` || exit 1 + + # Find out the archs in all the layers + echo -n "Figuring out the archs in the layers ... " + topdir=$(dirname $(dirname $(readlink -e $0))) + tunedirs="`find $topdir/meta* $layers -path '*/meta*/conf/machine/include'`" + [ -n "$tunedirs" ] || echo_error "Can't find the tune directory" + all_archs=`grep -r -h "^AVAILTUNES " $tunedirs | sed -e 's/.*=//' -e 's/\"//g'` + # Add the qemu and native archs + # Use the "_" to substitute "-", e.g., x86-64 to x86_64 + # Sort to remove the duplicated ones + all_archs=$(echo $all_archs qemuarm qemux86 qemumips qemuppc qemux86_64 $(uname -m) \ + | sed -e 's/-/_/g' -e 's/ /\n/g' | sort -u) + echo "Done" + + sstate_suffixes="deploy-rpm deploy-ipk deploy-deb deploy package populate-lic populate-sysroot" + + cd $cache_dir || exit 1 + # Save all the sstate files in a file + sstate_list=`mktemp` || exit 1 + ls sstate-*.tgz >$sstate_list + echo -n "Figuring out the archs in the sstate cache dir ... " + for arch in $all_archs; do + grep -q -w $arch $sstate_list + [ $? -eq 0 ] && ava_archs="$ava_archs $arch" + done + echo "Done" + echo "The following archs have been found in the cache dir:" + echo $ava_archs + echo "" + + # Save the file list which needs to be removed + local remove_listdir=`mktemp -d` || exit 1 + + for suffix in $sstate_suffixes; do + # Save the file list to a file, some suffix's file may not exist + ls *_$suffix.tgz >$list_suffix 2>/dev/null local deleted=0 - echo -n "Removing the sstate-xxx_$suffix.tgz ... " - # sed twice to avoid the greedy match + echo -n "Figuring out the sstate-xxx_$suffix.tgz ... " + # There are at list 6 dashes (-) after arch, use this to avoid the + # greedy match of sed. file_names=`for arch in $ava_archs; do - ls sstate-*-$arch-*_$suffix.tgz 2>/dev/null | \ - sed -e 's/\(.*\)-'"$arch"'-.*/\1/' \ - -e 's/\(.*\)-'"$arch"'-.*/\1/' + sed -ne 's/^\(sstate-.*\)-'"$arch"'-.*-.*-.*-.*-.*-.*/\1/p' $list_suffix done | sort -u` + fn_tmp=`mktemp` || exit 1 for fn in $file_names; do + [ -z "$verbose" ] || echo "Analyzing $fn-xxx_$suffix.tgz" for arch in $ava_archs; do - ls $fn-$arch-*_$suffix.tgz 2>/dev/null >>/tmp/$fn + grep -h "^$fn-$arch-" $list_suffix >>$fn_tmp done - # Also delete the .siginfo file - to_del=$(ls -t $(cat /tmp/$fn) | sed -n '1!p' | sed -e 'p' -e 's/$/.siginfo/') - rm -f $to_del + # Use the access time, also delete the .siginfo file + to_del=$(ls -u $(cat $fn_tmp) | sed -n '1!p' | sed -e 'p' -e 's/$/.siginfo/') + echo $to_del >>$remove_listdir/sstate-xxx_$suffix let deleted=$deleted+`echo $to_del | wc -w` - rm -f /tmp/$fn + rm -f $fn_tmp done - echo "($deleted files)" + echo "($deleted files will be removed)" let total_deleted=$total_deleted+$deleted done - echo "$total_deleted files have been removed" + rm -f $list_suffix + if [ $total_deleted -gt 0 ]; then + read_confirm + if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then + for list in `ls $remove_listdir/`; do + echo -n "Removing $list.tgz (`cat $remove_listdir/$list | wc -w` files) ... " + rm -f $verbose `cat $remove_listdir/$list` + echo "Done" + done + echo "$total_deleted files have been removed!" + else + do_nothing + fi + else + no_files + fi + [ -d $remove_listdir ] && rm -fr $remove_listdir +} + +# Remove the sstate file by stamps dir, the file not used by the stamps dir +# will be removed. +rm_by_stamps (){ + + local cache_list=`mktemp` || exit 1 + local keep_list=`mktemp` || exit 1 + local mv_to_dir=`mktemp -d -p $cache_dir` || exit 1 + local suffixes + local sums + local all_sums + + suffixes="populate_sysroot populate_lic package_write_ipk \ + package_write_rpm package_write_deb package deploy" + + # Figure out all the md5sums in the stamps dir. + echo -n "Figuring out all the md5sums in stamps dir ... " + for i in $suffixes; do + sums=`find $stamps -maxdepth 2 -name "*\.do_$i\.sigdata.*" | \ + sed 's#.*\.sigdata\.##' | sort -u` + all_sums="$all_sums $sums" + done + echo "Done" + + # Save all the state file list to a file + ls $cache_dir/sstate-*.tgz >$cache_list + + echo -n "Figuring out the files which will be removed ... " + for i in $all_sums; do + grep ".*-$i.*" $cache_list >>$keep_list + done + echo "Done" + + if [ -s $keep_list ]; then + let total_deleted=(`cat $cache_list | wc -w` - `cat $keep_list | wc -l`)*2 + + if [ $total_deleted -gt 0 ]; then + read_confirm + if [ "$confirm" = "y" -o "$confirm" = "Y" ]; then + echo "Removing sstate cache files ... ($total_deleted files)" + # Save the file which needs to be kept, remove the others, + # then move it back + for i in `cat $keep_list`; do + mv $i $mv_to_dir + mv $i.siginfo $mv_to_dir || true + done + rm -f $verbose $cache_dir/sstate-*.tgz + rm -f $verbose $cache_dir/sstate-*.tgz.siginfo + mv $mv_to_dir/* $cache_dir/ + echo "$total_deleted files have been removed" + else + do_nothing + fi + else + no_files + fi + else + echo_error "All files in cache dir will be removed! Abort!" + fi + + rm -f $cache_list + rm -f $keep_list + rmdir $mv_to_dir } # Parse arguments while [ -n "$1" ]; do case $1 in --cache-dir=*) - cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' -e 's#/*$##' | xargs readlink -f` + cache_dir=`echo $1 | sed -e 's#^--cache-dir=##' | xargs readlink -e` [ -d "$cache_dir" ] || echo_error "Invalid argument to --cache-dir" shift ;; - --remove-duplicated) - rm_duplicated="yes" + --remove-duplicated|-d) + rm_duplicated="y" + shift + ;; + --yes|-y) + confirm="y" + shift + ;; + --extra-layer=*) + extra_layers=`echo $1 | sed -e 's#^--extra-layer=##' -e 's#,# #g'` + [ -n "$extra_layers" ] || echo_error "Invalid extra layer $i" + for i in $extra_layers; do + l=`readlink -e $i` + if [ -d "$l" ]; then + layers="$layers $l" + else + echo_error "Can't find layer $i" + fi + done + shift + ;; + --stamps-dir=*) + stamps=`echo $1 | sed -e 's#^--stamps-dir=##' -e 's#,# #g'` + [ -n "$stamps" ] || echo_error "Invalid stamps dir $i" + for i in $stamps; do + [ -d "$i" ] || echo_error "Invalid stamps dir $i" + done + shift + ;; + --verbose|-v) + verbose="-v" shift ;; --help|-h) @@ -95,8 +304,8 @@ while [ -n "$1" ]; do exit 0 ;; *) - echo "Invalid arguments $*" - echo_error "Try 'sstate-cache-management.sh -h' for more information." + echo "Invalid arguments $*" + echo_error "Try 'sstate-cache-management.sh -h' for more information." ;; esac done @@ -104,32 +313,14 @@ done # sstate cache directory, use environment variable SSTATE_CACHE_DIR # if it was not specified, otherwise, error. [ -n "$cache_dir" ] || cache_dir=$SSTATE_CACHE_DIR +[ -n "$cache_dir" ] || echo_error "No cache dir found!" [ -d "$cache_dir" ] || echo_error "Invalid cache directory \"$cache_dir\"" -cache_dir=`readlink -f $cache_dir` +[ -n "$rm_duplicated" -a -n "$stamps" ] && \ + echo_error "Can not use both --remove-duplicated and --stamps-dir" -topdir=$(dirname $(dirname $(readlink -f $0))) -tunedir=$topdir/meta/conf/machine/include -[ -d $tunedir ] || echo_error "Can't find the tune directory" +[ "$rm_duplicated" = "y" ] && remove_duplicated +[ -n "$stamps" ] && rm_by_stamps +[ -z "$rm_duplicated" -a -z "$stamps" ] && \ + echo "What do you want to do?" -# Use the "_" to substitute "-", e.g., x86-64 to x86_64 -all_archs=`grep -r DEFAULTTUNE $tunedir | \ - sed -e 's/.*\"\(.*\)\"/\1/' -e 's/-/_/g' | sort -u` -# Add the qemu archs -all_archs="$all_archs qemuarm qemux86 qemumips qemuppc" - -all_suffixes="deploy-rpm deploy-ipk deploy-deb deploy package populate-lic populate-sysroot" - -cd $cache_dir - -echo "Figuring out the archs in the sstate cache dir ..." -for arch in $all_archs; do - ls | grep -q -w $arch - [ $? -eq 0 ] && ava_archs="$ava_archs $arch" -done -echo "The following archs have been found in the sstate cache dir:" -echo $ava_archs - -if [ "$rm_duplicated" == "yes" -a -n "$ava_archs" ]; then - remove_duplicated "$all_suffixes" "$ava_archs" -fi |