巧用shell+rsync实现日志自动过滤处理压缩上传分类(在斯凯的最终版本下周离职)

2021-11-25 06:48:49

上次的版本http://blog.chinaunix.net/uid-21142030-id-5613029.html


点击(此处)折叠或打开

  1. [10:58 t ~]$ cat rsync.t
  2. #!/usr/bin/env bash
  3. #set -x
  4. #2015-08-18:    修复直接压缩201目录的bug
  5. #2015-09-28:    新加项目分类目录,新加多重判断
  6. #2015-10-19:    修复与glusterfs冲突问题.
  7. #2016-04-08:    处理磁盘空紧张时无法处理日志压缩上传导致空间满问题。
  8. #2016-04-11:    解决目录非单独挂载时识别为/无限循环问题。
  9. #2016-04-12:    优化搜索逻辑，添加输出日志。

  11. LOGPATH=$1
  12. PROJECT=$2

  14. SLEEPTIME=1s
  15. DATETIME=$(date +%F_%T)
  16. DATE=$(date +%F)
  17. IP=$(LC_ALL=C /sbin/ifconfig|grep "inet addr:"|grep -v "127.0.0.1"|cut -d: -f2|awk '{print $1}'|sed ':a;$!N;s/\n/_/g;ta;P;D')
  18. LINE=`seq -s= 80|sed 's/[0-9]*//g'`

  20. USER=admin
  21. export RSYNC_PASSWORD=skymobi
  22. RSYNCIP=192.168.1.2
  23. RSYNCMODULE=chss-t
  24. RSYNCCONFMODULE=chss-tc

  26. RSYNCDIR=${PROJECT}/"$IP"
  27. FINDLOGFILE=/tmp/find_"$IP".log.$DATE
  28. RSYNCLOGFILE=/tmp/rsync_"$IP".log.$DATE
  29. RSYNCEXELOGFILE=/tmp/rsync.exe.log.$DATE

  31. REGEX="-regex .*logs/\(.*log\)?.*201.*\(.*log\)?\(.gz\)?"
  32. REGEX2="-regex .*tar.gz"
  33. TIME="-mtime +${DAYS:=1}"
  34. DEL="-exec rm -rfv {} ;"
  35. COMPRESSION="-exec tar -zcvf `basename {}`_${RSYNCMODULE}.tar.gz {} --remove-files --exclude=*tar.gz ;"

  37. #创建目录
  38. echo -e "$LINE\n$DATETIME : Create directory for upload."
  39. mkdir -p /tmp/$RSYNCDIR
  40. echo -e "$DATETIME : Upload create directory.\n$LINE"
  41. for rspath in $RSYNCMODULE $RSYNCCONFMODULE
  42. do
  43.     rsync -atv --log-file=${RSYNCLOGFILE} /tmp/${PROJECT} ${USER}@"$RSYNCIP"::${rspath}
  44. done

  46. #扫描日志
  47. for Path in $LOGPATH
  48. do
  49.   echo -e "$LINE \n$DATETIME : Operation on the [ $Path ] directory "
  50.   #1.是否存在目录
  51.   if [ -d /$Path ];then
  52. #########################################
  53.     RSYNC(){
  54.       if test $# -ne 0;then
  55.         rsync -aERtvz --no-p --chmod=Do+rX,Fo-X --log-file=${RSYNCLOGFILE} --remove-source-files --skip-compress=*.gz "$@" ${USER}@"$RSYNCIP"::${ttype:=$RSYNCMODULE}/${RSYNCDIR}
  56.       else
  57.         echo "$DATETIME : no $RSYNCMODULE.tar.gz file found."
  58.         return 1
  59.       fi
  60.     }
  61.     FINDER(){
  62.       find -L /${Path} $@
  63.     }
  64.     SPACECOUNT(){
  65.     df -P|grep /${Path} |tail -n 1 |awk '{print $5}'|awk -F% '{print $1}'
  66.     }
  67. ##########################################


  70.     #2.是否空间紧张
  71.     echo "$DATETIME : Start while."
  72.     starttime=`date`
  73.     SPACE=`SPACECOUNT`
  74.     Count=0
  75.     while test -n "$SPACE" -a "${SPACE:=0}" -gt 20 -a $Count -lt 2
  76.     do
  77.         
  78.         
  79.       if [ $SPACE -gt 80 ];then
  80.     echo "$DATETIME : Start del."
  81.         TIME="-mmin +720"
  82.         FINDER "$REGEX" "-a" "$TIME" "$DEL" >> $FINDLOGFILE
  83.         SPACE=`SPACECOUNT`
  84.         continue
  85.       elif [ $SPACE -gt 60 ];then
  86.     echo "$DATETIME : Start upload file direct."
  87.         echo '60%
  88.         TIME="-mmin +60"
  89.         RSYNC `FINDER "$REGEX" "-a" "$TIME"`
  90.         SPACE=`SPACECOUNT`
  91.         ((Count+=1))
  92.         continue
  93.       #2.
  94.       else
  95.     echo "$DATETIME : Start compression."
  96.         echo 'space
  97.         TIME="-mmin +0"
  98.         FINDER "$REGEX" "-a ! $REGEX2" "-a" "$TIME" "$COMPRESSION" >> $FINDLOGFILE
  99.         SPACE=`SPACECOUNT`
  100.     echo "new space is $SPACE"
  101.         break
  102.       fi
  103.     done

  105.     endtime=`date`
  106.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
  107.     echo "haldertime:$haldertime"
  108.     #3.执行上传操作
  109.     echo "$DATETIME : Start upload tar.gz file."
  110.     starttime=`date`
  111. # for ttype in ${RSYNCMODULE} #${RSYNCCONFMODULE}
  112. # do
  113.           RSYNC $(find /$Path -type f -name "*_${RSYNCMODULE}.tar.gz" -a ! -regex '.*/brick/.*')
  114.           #RSYNC $(find /$Path -type f -name "*_$ttype.tar.gz" -a ! -regex '.*/brick/.*')
  115. # done
  116.     endtime=`date`
  117.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
  118.     echo "upload_tar_gz_haldertime:$haldertime"
  119.   #1.
  120.   else
  121.     echo /${Path} does not exist...
  122.   fi
  123.   echo -e "$DATETIME : [ $Path ] To the end of the directory operation. \n$LINE"
  124.   #1.是否存在目录
  125. done >>$RSYNCEXELOGFILE


点击(此处)折叠或打开

  1. [10:58 t ~]$ cat rsync.t
  2. #!/usr/bin/env bash
  3. #set -x
  4. #2015-08-18:    修复直接压缩201目录的bug
  5. #2015-09-28:    新加项目分类目录,新加多重判断
  6. #2015-10-19:    修复与glusterfs冲突问题.
  7. #2016-04-08:    处理磁盘空紧张时无法处理日志压缩上传导致空间满问题。
  8. #2016-04-11:    解决目录非单独挂载时识别为/无限循环问题。
  9. #2016-04-12:    优化搜索逻辑，添加输出日志。

  11. LOGPATH=$1
  12. PROJECT=$2

  14. SLEEPTIME=1s
  15. DATETIME=$(date +%F_%T)
  16. DATE=$(date +%F)
  17. IP=$(LC_ALL=C /sbin/ifconfig|grep "inet addr:"|grep -v "127.0.0.1"|cut -d: -f2|awk '{print $1}'|sed ':a;$!N;s/\n/_/g;ta;P;D')
  18. LINE=`seq -s= 80|sed 's/[0-9]*//g'`

  20. USER=admin
  21. export RSYNC_PASSWORD=skymobi
  22. RSYNCIP=192.168.165.88
  23. RSYNCMODULE=chss-t
  24. RSYNCCONFMODULE=chss-tc

  26. RSYNCDIR=${PROJECT}/"$IP"
  27. FINDLOGFILE=/tmp/find_"$IP".log.$DATE
  28. RSYNCLOGFILE=/tmp/rsync_"$IP".log.$DATE
  29. RSYNCEXELOGFILE=/tmp/rsync.exe.log.$DATE

  31. REGEX="-regex .*logs/\(.*log\)?.*201.*\(.*log\)?\(.gz\)?"
  32. REGEX2="-regex .*tar.gz"
  33. TIME="-mtime +${DAYS:=1}"
  34. DEL="-exec rm -rfv {} ;"
  35. COMPRESSION="-exec tar -zcvf `basename {}`_${RSYNCMODULE}.tar.gz {} --remove-files --exclude=*tar.gz ;"

  37. #创建目录
  38. echo -e "$LINE\n$DATETIME : Create directory for upload."
  39. mkdir -p /tmp/$RSYNCDIR
  40. echo -e "$DATETIME : Upload create directory.\n$LINE"
  41. for rspath in $RSYNCMODULE $RSYNCCONFMODULE
  42. do
  43.     rsync -atv --log-file=${RSYNCLOGFILE} /tmp/${PROJECT} ${USER}@"$RSYNCIP"::${rspath}
  44. done

  46. #扫描日志
  47. for Path in $LOGPATH
  48. do
  49.   echo -e "$LINE \n$DATETIME : Operation on the [ $Path ] directory "
  50.   #1.是否存在目录
  51.   if [ -d /$Path ];then
  52. #########################################
  53.     RSYNC(){
  54.       if test $# -ne 0;then
  55.         rsync -aERtvz --no-p --chmod=Do+rX,Fo-X --log-file=${RSYNCLOGFILE} --remove-source-files --skip-compress=*.gz "$@" ${USER}@"$RSYNCIP"::${ttype:=$RSYNCMODULE}/${RSYNCDIR}
  56.       else
  57.         echo "$DATETIME : no $RSYNCMODULE.tar.gz file found."
  58.         return 1
  59.       fi
  60.     }
  61.     FINDER(){
  62.       find -L /${Path} $@
  63.     }
  64.     SPACECOUNT(){
  65.     df -P|grep /${Path} |tail -n 1 |awk '{print $5}'|awk -F% '{print $1}'
  66.     }
  67. ##########################################


  70.     #2.是否空间紧张
  71.     echo "$DATETIME : Start while."
  72.     starttime=`date`
  73.     SPACE=`SPACECOUNT`
  74.     Count=0
  75.     while test -n "$SPACE" -a "${SPACE:=0}" -gt 20 -a $Count -lt 2
  76.     do
  77.         
  78.         
  79.       if [ $SPACE -gt 80 ];then
  80.     echo "$DATETIME : Start del."
  81.         TIME="-mmin +720"
  82.         FINDER "$REGEX" "-a" "$TIME" "$DEL" >> $FINDLOGFILE
  83.         SPACE=`SPACECOUNT`
  84.         continue
  85.       elif [ $SPACE -gt 60 ];then
  86.     echo "$DATETIME : Start upload file direct."
  87.         echo '60%
  88.         TIME="-mmin +60"
  89.         RSYNC `FINDER "$REGEX" "-a" "$TIME"`
  90.         SPACE=`SPACECOUNT`
  91.         ((Count+=1))
  92.         continue
  93.       #2.
  94.       else
  95.     echo "$DATETIME : Start compression."
  96.         echo 'space
  97.         TIME="-mmin +0"
  98.         FINDER "$REGEX" "-a ! $REGEX2" "-a" "$TIME" "$COMPRESSION" >> $FINDLOGFILE
  99.         SPACE=`SPACECOUNT`
  100.     echo "new space is $SPACE"
  101.         break
  102.       fi
  103.     done

  105.     endtime=`date`
  106.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
  107.     echo "haldertime:$haldertime"
  108.     #3.执行上传操作
  109.     echo "$DATETIME : Start upload tar.gz file."
  110.     starttime=`date`
  111. # for ttype in ${RSYNCMODULE} #${RSYNCCONFMODULE}
  112. # do
  113.           RSYNC $(find /$Path -type f -name "*_${RSYNCMODULE}.tar.gz" -a ! -regex '.*/brick/.*')
  114.           #RSYNC $(find /$Path -type f -name "*_$ttype.tar.gz" -a ! -regex '.*/brick/.*')
  115. # done
  116.     endtime=`date`
  117.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
  118.     echo "upload_tar_gz_haldertime:$haldertime"
  119.   #1.
  120.   else
  121.     echo /${Path} does not exist...
  122.   fi
  123.   echo -e "$DATETIME : [ $Path ] To the end of the directory operation. \n$LINE"
  124.   #1.是否存在目录
  125. done >>$RSYNCEXELOGFILE

点击(此处)折叠或打开

  1. [11:00 t ~]$ cat while.t
  2. #!/bin/bash
  3. set -x
  4. LOGPATH='opt app data app1'
  5. PROJECT=talentest
  6. sleeptime=0.5h

  8. while :
  9. do
  10. time /bin/bash /bin/rsync.t "$LOGPATH" $PROJECT
  11. sleep $sleeptime
  12. done


上一篇：几个简单步骤可以提高网站浏览体验


下一篇：centOS 下安装Nginx，访问/data/www目录时出现file not found