脚本说明：


日志切割与上传类脚本的功能：脚本自身的运行日志记录与清理、日志切割与打包、ftp上传、传送失败自动重传、断电自动补传、清理超期旧打包等

--------------------

#!/bin/bash#Cutanduploadacloglog#bycolin#revisionon2016-06-15#########################################功能说明：该脚本运用于上传aclog日志##使用说明：#+./aclog_backup_and_upload.sh-f/var/log/host/jnxac.log-ttar-hSDQD-TS-JNX-HLS#更新说明：##########################################sleep60#延时60秒运行PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/binRUN_LOG='/var/log/cron_scripts_run.log'[!-f${RUN_LOG}]&&touch${RUN_LOG}echoGoodLog(){echo-e"\033[32m`date+%F""%T":"%N`$*\033[0m">>${RUN_LOG}}echoBadLog(){echo-e"\033[31m`date+%F""%T":"%N`$*\033[0m">>${RUN_LOG}}echoGoodLog"Now,Script:`basename$0`run."RUNLOG_MAX_NUM=100000RUNLOG_MAX_DELNUM=5000SEND_MAX_TIME=6SEND_WHILE_SLEEP_TIME=30BACK_SAVE_MAX_DAY=180LOG_CUT_MIN=60#日志文件30分钟切割一次cleanRunLog(){CLEANLOGFILE=${1?"Usage:$FUNCNAMElog_file_name"}TEMP_WC=`cat${CLEANLOGFILE}|wc-l`["${TEMP_WC}"-gt"${RUNLOG_MAX_NUM}"]&&{sed-i"1,${RUNLOG_MAX_DELNUM}d"${CLEANLOGFILE}&&echoGoodLog"Cleanupthe${CLEANLOGFILE}..."}echoGoodLog"Script:`basename$0`rundone."exit}###省份数组#PROVINCE=(sdhljsc)###脚本帮助提示函数#scriptsHelp(){echoBadLog"======================================="echoGoodLog"Usageparameters:"echoGoodLog"./`basename$0`-f/--file/var/log/host/qfgw.log[-t/--tartar][-h/--hostnameSDQD-TS-CL-WIN]"echoGoodLog"Options:"echoGoodLog"-f/--file)"echoGoodLog"必须的参数：需要切割与上传的日志文件"echoGoodLog"-t/--tar)"echoGoodLog"可选参数：若需打包，使用方法：-ttar"echoGoodLog"-h/--hostname)"echoGoodLog"可选参数：若一个站点存放了多个地方的相同类型日志，此时就需要设定每个日志所属站点"echoGoodLog"比如：昌乐的网关日志，没有HLS，日志存储在青岛北，那加参数如下：-hSDQD-TS-CL-WIN"echoGoodLog"-c)"echoGoodLog"可选参数：用于检查日志服务器上是否有存储日志的文件夹，没有就创建，需使用except命令，会自动安装"echoGoodLog"注意：日志服务器必须要开放ssh远程登录，或者防火墙需要放行"echoGoodLog"使用方法：./`basename$0`-c；-c后面不需要加选项"echoGoodLog"Example:"echoGoodLog"./`basename$0`-f/var/log/host/qfgw.log-ttar-hSDQD-TS-CL-WIN"echoBadLog"======================================="}checkParameter(){PARAMETER=${1:-null}PARAMETER_STATUS=`echo"${PARAMETER}"|grep"^-"|wc-l`if["${PARAMETER_STATUS}"-eq1-o"${PARAMETER}"="null"];thenscriptsHelpechoBadLog"参数错误，请重新输入。"exitfi}###判断是否带参数#if[-z"$*"];thenscriptsHelpexitelse###脚本传参数，调用相应的函数功能#whiletest-n"$1";docase"$1"in--file|-f)shiftcheckParameter$1LOG_NAME=$1;;--tar|-t)shiftcheckParameter$1CTAR=$1;;--hostname|-h)shiftcheckParameter$1CHOSTNAME=$1;;-c)CCHECK_LOG_SERVER_DIR=$1;;*)echoBadLog"Unknownargument:$1"scriptsHelpexit;;esacshiftdonefi###需要切割的日志为必须要的参数#checkParameter$LOG_NAMEIS_TAR=${CTAR:-notar}HOSTNAME=${CHOSTNAME:-`hostname`}TEMP_PRO=`echo${PROVINCE[@]}|sed"s//|/g"`STATION=`echo${HOSTNAME}|awk-F-'{print$3}'|tr[A-Z][a-z]`STATION_TYPE=`echo${HOSTNAME}|awk-F-'{print$2}'|tr[A-Z][a-z]`STATION_SITE=`echo${HOSTNAME}|awk-F-'{print$1}'|tr[A-Z][a-z]|grep-Eo"\b${TEMP_PRO}"`##############################definetheftpclientLOG_TYPE="aclog"LOG_TYPEB="aclog"LCD_DIR="/data/${STATION}_log/${STATION}_${LOG_TYPE}"#日志服务器上的保存文件夹特例#----------------------------case${STATION_SITE}${STATION_TYPE}in#sdbs)CD_DIR="/${LOG_TYPEB}/${STATION_SITE}/qdjy/${STATION}/";;sdbs)CD_DIR="/${LOG_TYPEB}/${STATION_SITE}/qdjy/";;*)CD_DIR="/${LOG_TYPEB}/${STATION_SITE}/${STATION}/";;esac##############################definetheftpserverFTPSERVER='IP地址'FTPUSER='用户名'FTPPASSWD='密码'SSHPORT='端口'#############################T=`echo$(date+%k)|sed's///g'`#LAST_T=`echo$(date-d"1hourago"+%k)|sed's///g'`DAY_TIME=`date+%Y-%m-%d`LAST_DAY_TIME=`date-d"yesterday"+%Y-%m-%d`LAST_HOUR_TIME=`date+%Y-%m-%d-%H`FILENUM=`expr$(date+%M|sed's/^0//g')/${LOG_CUT_MIN}`DAY_LOG_TAR_NUM="${FILENUM}"[["${FILENUM}"-eq0]]&&{FILENUM=`expr60/${LOG_CUT_MIN}`LAST_HOUR_TIME=`date-d"1hourago"+%Y-%m-%d-%H`}###为兼容旧版本，当60分钟切割一次时，就和之前旧版本文件名一样咯#if[${LOG_CUT_MIN}-eq60];thenLOG_HOUR_NAME="${LOG_TYPE}${LAST_HOUR_TIME}.txt"elseLOG_HOUR_NAME="${LOG_TYPE}${LAST_HOUR_TIME}-${FILENUM}.txt"fiLOG_DAY_NAME="${LOG_TYPE}${LAST_DAY_TIME}.txt"case"${IS_TAR}"in"tar")PUT_LOG_DAY_NAME="${LOG_DAY_NAME%.txt}.tar.gz"PUT_LOG_HOUR_NAME="${LOG_HOUR_NAME%.txt}.tar.gz";;"notar")PUT_LOG_DAY_NAME="${LOG_DAY_NAME}"PUT_LOG_HOUR_NAME="${LOG_HOUR_NAME}";;esactarLogFile(){TARTONAME=${1?"Usage:$FUNCNAMEtar_file_nameneed_tar_file_name"}FROMLOG=$2tar-czf${TARTONAME}--remove-files${FROMLOG}if[-e${TARTONAME}];thenechoGoodLog"Tar:${TARTONAME}issuccessfully."elseechoBadLog"Tar:${TARTONAME}wasfailed,Pleasecheck..."fi}checkProgramExist(){PROGRAMNAME=${1?"Usage:$FUNCNAMEprogram_install_name"}PROGRAMEXIST=`dpkg-l|grep-wo${PROGRAMNAME}|wc-l`if["${PROGRAMEXIST}"-ge1];thenreturn0;else/usr/bin/apt-getinstall${PROGRAMNAME}-yif[$?-eq0];thenechoGoodLog"Install${PROGRAMNAME}issuccessfully."return0;elseechoBadLog"Install${PROGRAMNAME}wasfailed,Pleasecheck..."return1;fifi}checkLogServerDir(){checkProgramExistexpectpasswd=${FTPPASSWD}/usr/bin/expect<<-EOFsettime1spawnssh-p${SSHPORT}${FTPUSER}@${FTPSERVER}expect{"*yes/no"{send"yes\r";exp_continue}"*password:"{send"$passwd\r"}}expect"*~$"send"cd/home/upload${CD_DIR}\r"expect{"*Nosuchfileordirectory"{send"mkdir-p/home/upload${CD_DIR}\r"}}expect"*~$"send"exit\r"interactexpecteofEOFecho-e"\r"}[!-d${LCD_DIR}]&&mkdir-p${LCD_DIR}###若切割后的日志文件存在时，就退出切割命令等#cd${LCD_DIR}&&[!-f${PUT_LOG_HOUR_NAME}]&&{if[-s${LOG_NAME}];thenuntil[-f${LOG_HOUR_NAME}]docp${LOG_NAME}${LOG_HOUR_NAME}doneif[$?-eq0-a-f"${LOG_HOUR_NAME}"];thenechoGoodLog"Create${LOG_HOUR_NAME}issuccessfully."elseechoBadLog"Create${LOG_HOUR_NAME}wasfailed,Pleasecheck..."fi["${IS_TAR}"="tar"]&&tarLogFile${PUT_LOG_HOUR_NAME}${LOG_HOUR_NAME}if["${T}"-eq0-a"${DAY_LOG_TAR_NUM}"-eq0];thenTEMP_CHECK_LOG=`ls-l${LOG_DAY_NAME}|awk'{print$5}'`cat${LOG_NAME}>>${LOG_DAY_NAME}[$?-eq0]||cat${LOG_NAME}>>${LOG_DAY_NAME}CHECK_LOG=`ls-l${LOG_DAY_NAME}|awk'{print$5}'`else[-f${LOG_TYPE}${DAY_TIME}.txt]||touch${LOG_TYPE}${DAY_TIME}.txtTEMP_CHECK_LOG=`ls-l${LOG_TYPE}${DAY_TIME}.txt|awk'{print$5}'`cat${LOG_NAME}>>${LOG_TYPE}${DAY_TIME}.txt[$?-eq0]||cat${LOG_NAME}>>${LOG_TYPE}${DAY_TIME}.txtCHECK_LOG=`ls-l${LOG_TYPE}${DAY_TIME}.txt|awk'{print$5}'`fiechoGoodLog"CHECK:CHECK_LOG=${CHECK_LOG},TEMP_CHECK_LOG=${TEMP_CHECK_LOG}..."if["${CHECK_LOG}"-gt"${TEMP_CHECK_LOG}"];thencat/dev/null>${LOG_NAME}[$?-eq0]&&echoGoodLog"Append:${LOG_HOUR_NAME}to${LOG_TYPE}_day_logissuccessfully."elseechoBadLog"Append:${LOG_HOUR_NAME}to${LOG_TYPE}_day_logwasfailed,Pleasecheck..."fi["${T}"-eq0-a"${IS_TAR}"="tar"-a"${DAY_LOG_TAR_NUM}"-eq0]&&tarLogFile${PUT_LOG_DAY_NAME}${LOG_DAY_NAME}elseechoBadLog"Log:${LOG_NAME}isnullornotexist,Pleasecheck..."fi}###检查保存文件夹目录是否存在#CHECK_LOG_SERVER_DIR=${CCHECK_LOG_SERVER_DIR:-"f.ck"}["${CHECK_LOG_SERVER_DIR}"="-c"]&&checkLogServerDirFTP_LOG_DIR="/tmp/ftp_err"[-d${FTP_LOG_DIR}]||mkdir-p${FTP_LOG_DIR}FTP_ERROR_LOG="${FTP_LOG_DIR}/ftp_temp_${LOG_TYPE}_err$$.log"###FTP自动化上传函数#sendLog(){SENDLOGFILE=$1ftp-ivn${FTPSERVER}21>${FTP_ERROR_LOG}<<_EOF_user${FTPUSER}${FTPPASSWD}passivebinlcd${LCD_DIR}cd${CD_DIR}put${SENDLOGFILE}bye_EOF_###统计前面FTP运行输出的错误日志记录行数#LOG_COUNT=`grep-w"^226"${FTP_ERROR_LOG}|wc-l`if["${LOG_COUNT}"-eq1];thenechoGoodLog"Send:${SENDLOGFILE}toftp_serverwassuccessfully."TEMP_SEND_STATUS=0return0elseechoBadLog"Send:${SENDLOGFILE}morethan$xtime."TEMP_SEND_STATUS=1sleep${SEND_WHILE_SLEEP_TIME}return1fi}REUPLOADLIST="/var/log/reupload_list_${LOG_TYPEB}_${STATION}.log"TEMP_REUPLOADLIST="/var/log/temp_reupload_list_${LOG_TYPEB}_${STATION}.log"[-f${TEMP_REUPLOADLIST}]&&rm${TEMP_REUPLOADLIST}runSendLog(){SENDLOGNAME=$1x=1;i=1until["$i"-eq0];do["$x"-gt"${SEND_MAX_TIME}"]&&{echoBadLog"Send:${SENDLOGNAME}toftp_serverwasfailed,Pleasecheck..."echo"${LCD_DIR};;${CD_DIR};;${SENDLOGFILE}">>${TEMP_REUPLOADLIST}break}sendLog"${SENDLOGNAME}"i=`echo$?`x=`expr$x+1`done}["${T}"-eq0-a-f"${LCD_DIR}/${PUT_LOG_DAY_NAME}"-a"${DAY_LOG_TAR_NUM}"-eq0]&&runSendLog${PUT_LOG_DAY_NAME}[-f"${LCD_DIR}/${PUT_LOG_HOUR_NAME}"]&&runSendLog${PUT_LOG_HOUR_NAME}###把上面两种发送成功的记录更新到这个临时文件#TEMP_SENDSUCCESFILE="/var/log/temp_send_succes_${LOG_TYPEB}_${STATION}.txt"SENDSUCCESFILE="/var/log/send_succes_${LOG_TYPEB}_${STATION}.txt"["${TEMP_SEND_STATUS}"-eq0]&&echo"${LCD_DIR};;${CD_DIR};;${SENDLOGFILE};;$(date+%s)">${TEMP_SENDSUCCESFILE}[-e${SENDSUCCESFILE}]&&{LCD_DIR=`cat${SENDSUCCESFILE}|awk-F";;"'{print$1}'`CD_DIR=`cat${SENDSUCCESFILE}|awk-F";;"'{print$2}'`LAST_DATE_SUCCESS=`date-d@"$(cat${SENDSUCCESFILE}|awk-F";;"'{print$4}')"+%Y-%m-%d`INTERVAL_TIME=`expr$(date+%s)-$(cat${SENDSUCCESFILE}|awk-F";;"'{print$4}')`LAST_DAY=`echo$(date-d@"$(cat${SENDSUCCESFILE}|awk-F";;"'{print$4}')"+%d)|sed's/^0//g'`NOW_DAY=`echo$(date+%d)|sed's/^0//g'`["${LAST_DAY}"-ne"${NOW_DAY}"-a"${INTERVAL_TIME}"-gt7000]&&{if["${IS_TAR}"="tar"];thencd${LCD_DIR}&&{TEMP_SEND_FILES="${LOG_TYPE}${LAST_DATE_SUCCESS}.tar.gz"[-e${TEMP_SEND_FILES}]||tarLogFile${TEMP_SEND_FILES}${LOG_TYPE}${LAST_DATE_SUCCESS}.txt}elseTEMP_SEND_FILES="${LOG_TYPE}${LAST_DATE_SUCCESS}.txt"firunSendLog${TEMP_SEND_FILES}}rm${SENDSUCCESFILE}}[-f${TEMP_SENDSUCCESFILE}]&&mv${TEMP_SENDSUCCESFILE}${SENDSUCCESFILE}###重传上次发送失败的文件#+说明：重新上传函数的执行段，最好是放在后面，因为它依赖前面的FTP发送函数生成的TEMP_REUPLOADLIST清单#reUploadFile(){TEMP_NEED_DO_FILE=$1REUPLOADLIST_NUM=`cat${TEMP_NEED_DO_FILE}|wc-l`["${REUPLOADLIST_NUM}"-ge1]&&{whilereadlinedoLCD_DIR=`echo${line}|awk-F";;"'{print$1}'`CD_DIR=`echo${line}|awk-F";;"'{print$2}'`REUPLOADFILENAME=`echo${line}|awk-F";;"'{print$3}'`[-f"${LCD_DIR}/${REUPLOADFILENAME}"]&&runSendLog${REUPLOADFILENAME}done<${TEMP_NEED_DO_FILE}}[-e${TEMP_NEED_DO_FILE}]&&rm${TEMP_NEED_DO_FILE}}[-s${REUPLOADLIST}]&&reUploadFile${REUPLOADLIST}[-f${TEMP_REUPLOADLIST}]&&mv${TEMP_REUPLOADLIST}${REUPLOADLIST}[-f${FTP_ERROR_LOG}]&&rm${FTP_ERROR_LOG}###清理超过90天的备份日志#[-d${LCD_DIR}]&&cd${LCD_DIR}&&{forFILENAMEin`find.-typef-ctime+"${BACK_SAVE_MAX_DAY}"|awk-F/'{print$2}'`dorm${FILENAME}&&echoGoodLog"Clear:${LCD_DIR}/${FILENAME}..."done}cleanRunLog${RUN_LOG}

--------------------