inotify监控Nginx目录且将Log文件放入Flume监控文件中

最新推荐文章于 2024-08-08 15:36:47 发布

辉哥大数据

最新推荐文章于 2024-08-08 15:36:47 发布

阅读量1.4k

点赞数

CC 4.0 BY-SA版权

分类专栏：【工具】Linux 文章标签：脚本 nginx

本文链接：https://blog.youkuaiyun.com/silentwolfyh/article/details/59097737

【工具】Linux 专栏收录该内容

26 篇文章

订阅专栏

本文介绍了一种使用inotify监控Nginx日志文件，并将其传输至Flume进行进一步处理的方法。该方案包括了自定义脚本实现文件完整性检查、解压缩及移动等功能。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

#!/bin/bash    
  
src=/home/bigdataservice/yuhui/monitorfile # directory to monitor  
/usr/local/bin/inotifywait -rmq  -e close_write $src |  while read k_dir mon_event k_file
do  
echo "new file"
pd=".gz"
end=${k_file:0-3}
#echo $end
if [ "$end" == "$pd" ]; then  
 echo $k_dir$k_file
 sh check_md5_decompression.sh $k_dir$k_file /home/bigdataservice/yuhui/monitorTemp/  /home/bigdataservice/yuhui/push.log
 mv /home/bigdataservice/yuhui/monitorTemp/* /home/bigdataservice/yuhui/monitorResult
 echo "file unzip done"
fi  
done

start.sh

nohup /bin/bash monitor.sh &

check_md5_decompression.sh

#!/bin/bash

## =====================================================================
# Script Name : check_md5_decompression.sh
# Description : check md5/unzip file/modify file
# Author      : ddl
# Date        : 2016-09-23
#             ./check_md5_decompression.sh
#
# Revision    : 1.0
# Organization: donews
## =====================================================================

#The default variable
CurrentDate=`date "+%Y%m%d_%H%M%S"`
ScriptOwnDir=$(cd "$(/usr/bin/dirname "$0")"; pwd)
check_file=$1
unzip_dir=$2
output_log=$3
check_file_dir=$(cd "$(/usr/bin/dirname "$check_file")";pwd)

# =========colour========
blue="\033[0;34m"
green="\033[0;32m"
red="\033[0;31m"
yellow="\033[1;33m"
End="\033[0m"
#===================

usage () {
  echo -n -e "$green Usage: check_md5_decompression.sh check_file unzip_dir output_log $End \n"
  echo -n -e "$red exp:$End \n"
  echo -n -e "	check_md5_decompression.sh /data/temp/m.donews.com_access.log_20160922_000000.tar.gz /tmp/001 /tmp/a.log \n"
  exit 1
}

if [ "$#" -ne 3 ]; then
  usage
fi

# Set record log function
RecordLog(){
    local ShellName LogFile

    ShellName=${0##*/}
    LogFile=${ScriptOwnDir}/${ShellName}.log
    echo "[ `date "+%F %T"` ] | SysUser : $USER | ${ShellName} | ${*:-"Null Log"}" >> ${LogFile}

    output_writelog=$output_log
    if [ -f "$output_writelog" ];then
        echo "[ `date "+%F %T"` ] | SysUser : $USER | ${ShellName} | ${*:-"Null Log"}" >> ${output_writelog}
    else
        touch $output_writelog 
        echo "[ `date "+%F %T"` ] | SysUser : $USER | ${ShellName} | ${*:-"Null Log"}" >> ${output_writelog}
    fi

}

check_md5(){
    #echo -e "$green begin check MD5.... $End"
    RecordLog "begin check MD5...."
    check_file_md5=$check_file.MD5 
    if [ -f "$check_file_md5" ];then
	  #echo -e "$check_file_md5 is exits.....................................[$green OK $End]"
	  RecordLog "$check_file_md5 is exits OK"
          cd $check_file_dir
          /usr/bin/md5sum -c $check_file_md5 > /dev/null 2>&1
          if [ $? -eq 0 ];then
              #echo -e "check MD5 file is ....................................................................................[$green OK $End]"
              RecordLog "check $check_file MD5 file is success!"
          else
              #echo -e "check MD5 file is ....................................................................................[$red FAILD $End]"
              RecordLog "check $check_file MD5 file is faild!"
              exit 0
          fi 
    else
	  #echo -e "$check_file_md5 is not exits........................................[$red FAILD $End]"
	  RecordLog "$check_file_md5 is not exits is FAILD"
          min=0
          max=600
          while [ $min -le $max ]
          do
              #echo $min
              sleep 5
              min=`expr $min + 5`
              if [ -f "$check_file_md5" ];then 
                  cd $check_file_dir
                  /usr/bin/md5sum -c $check_file_md5 > /dev/null 2>&1
                  if [ $? -eq 0 ];then
                      #echo -e "check MD5 file is .................................................................................................[$green OK $End]"
                      RecordLog "check $check_file MD5 file is success!"
                  else
                      #echo -e "check MD5 file is .................................................................................................[$red FAILD $End]"
                      RecordLog "check $check_file MD5 file is faild!"
                      exit 0
                  fi
                  break
	      else
                  continue 
              fi
          done  
    fi


}

decompressing_files(){
    #echo -e "$green begin decompressing_files $check_file.... $End"
    RecordLog "begin decompressing_files $check_file...."
    cd $check_file_dir
    if [ -d $unzip_dir ];then
        #echo -e "unzip_dir $unzip_dir is exit ...............................................................................[$green OK $End]"
        RecordLog "unzip_dir $unzip_dir is exit!"
    else
        #echo -e "$unzip_dir is not exit .....................................................................................[$red FAILD $End]"
        RecordLog "$unzip_dir is not exit!"
        /bin/mkdir -p $unzip_dir
        #echo -e "create dir $unzip_dir!  .....................................................................................[$green OK $End]"
        RecordLog "create dir $unzip_dir!" 
    fi
    if [ -f "$check_file" ];then
        /bin/tar -zxvf $check_file -C $unzip_dir 2>& 1 > /dev/null
        if [ $? -eq 0 ];then
            #echo -e "decompressing_files $check_file is .............................................[$green OK $End]"
            RecordLog "decompressing_files $check_file is success!"
        else
            #echo -e "decompressing_files $check_file is .............................................[$red FAILD $End]"
            RecordLog "decompressing_files $check_file is faild!"
            exit 0
        fi       
    else
        #echo -e "$check_file is not exits"
        RecordLog "$check_file is not exits"
        exit 0
    fi


}



check_md5
decompressing_files