Linux日志处理

LINUX 下定时任务删除N天前的文件

一、模版：find 目录　-mtime +天数　-name "文件名"　-exec rm -rf {} \;
示例：将/usr/local/log目录下所有3天前带".log"的文件删除
find /usr/local/log-mtime +3-name "*.log*"-exec rm -rf {} \;
二、利用 crontab 执行定时任务
写脚本
0 0 */2 * * find /home/work/spiders/logs/hmd -mtime +2 -name "*.log*" -exec rm -rf {} \;
保存到 crontest.cron
执行 "crontab crontest.cron" 添加定时任务
执行"crontab -l" 查看定时任务是否成功或者检测/var/spool/cron下是否生成对应cron脚本
 

生成日志报告

#!/bin/bash
cd /xxx

start=$(date --date '12 apr 2019 00:00' +%s)
stop=$(date --date '15 apr 2019 23:00' +%s)

for t in $(seq ${start} 3600 ${stop})
do
Y=`date --date @${t} +'%d %b %Y %H:'`

REQ1=`grep "xxx- xxx" *.log | grep "xxx" | grep "$Y"  | wc -l`
REQ2=`grep "xxx- xxx" *.log.8 *.log.9 RTEServer*.log.1[0-9]| grep "xxx" | grep "$Y"  | wc -l`
RES1=`grep "xxx" *.log | grep "SlotId" | grep "rdpdt" | grep "$Y"  | wc -l`
RES2=`grep "xxx " *.log.8 *.log.9 *.log.1[0-9] | grep "xxx" | grep "xxx" | grep "$Y"  | wc -l`

REQ=`expr $REQ1 + $REQ2`
RES=`expr $RES1 + $RES2`

echo  $Y    $REQ    $RES >> /xxx/result.txt

done
 

日志批量打包到目标服务器

rm -rf /xxx/IVR.zip
rm -rf /xxx/IVR
mkdir -p /xxx/IVR
find /xxx/log -maxdepth 1 -type f -mtime -2 -name "*xxx.log*" | sort | xargs cat > /xxx/IVR/LIVRHSA01_xxx_all.log
find /xxx/log -maxdepth 1 -type f -mtime -2 -name "*xxx.log*" | sort | xargs cat > /xxx/IVR/LIVRHSA02_xxx_all.log
find /xxx/log -maxdepth 1 -type f -mtime -2 -name "*xxx.log*" | sort | xargs cat > /xxx/IVR/LIVRHSA03_xxx_all.log
find /xxx/log -maxdepth 1 -type f -mtime -2 -name "*xxx.log*" | sort | xargs cat > /xxx/IVR/LIVRHSA04_xxx_all.log
find /xxx/log -maxdepth 1 -type f -mtime -2 -name "*xxx.log*" | sort | xargs cat > /xxx/IVR/LIVRHSA05_xxx_all.log
zip -r /xxx/IVR-27.zip /xxx/IVR
find /xxx/ -maxdepth 1 -type f -mtime -2 -name "*.zip" | xargs -i scp {} xxx@xxx:/home/xxx/IVR

 

scp在命令行中带密码远程拷贝文件

sshpass -p 密码 scp -P 端口 源文件 目的文件

 

scp自动输入密码

#!/usr/bin/expect -f
spawn scp 10.10.10.10:/www/aaa.txt /www/
set timeout 20
expect "information"
exec sleep 2
send "password\r"
interact

 

解压日志压缩包并根据日期顺序合并日志文件

cd /home/zhssfe1a/demo
mkdir -p ./dealfile
find . -name 'weblog*tar.gz' | xargs -i tar xzvf {} -C ./dealfile
find -name "*appprofile*StdErr.log.gz" | sort | xargs gunzip -c > ./dealfile/native_stderr_all.log
java -jar -Xmx1000m ga458.jar -I native_stderr_all.log output.html PB

过程分析

find . -name "*.jpg" | xargs tar -rf picture.tgz  命令打包
find -name "*native_stderr.log" -exec ls -r {} \;
find -name "*native_stderr.log" | sort | xargs cat > native_stderr_all.log

tar xzvf weblog_20190615.tar.gz

find -maxdepth 2 -name "weblog*tar.gz"|xargs -i tar xzvf {} -C /home/zhssfe1a/demo/dealfile
mkdir -p /home/zhssfe1a/demo/dealfile && cp /tmp/test.txt "$_"

---
mkdir -p ./dealfile && find . -name 'weblog*tar.gz' | xargs -i tar xzvf {} -C ./dealfilejava  && find -name "*appprofile*StdErr.log.gz" | sort | xargs gunzip -c > ./dealfile/native_stderr_all.log

cd /home/zhssfe1a/demo
mkdir -p ./dealfile
find . -name 'weblog*tar.gz' | xargs -i tar xzvf {} -C ./dealfile
find -name "*appprofile*StdErr.log.gz" | sort | xargs gunzip -c > ./dealfile/native_stderr_all.log
java -jar -Xmx1000m ga458.jar -I native_stderr_all.log output.html PB

---
find . -name '*appprofile*StdErr.log.gz' | xargs -n1 gunzip 
find . -name '*wascell*StdErr*.gz' | xargs -n1 gunzip -k

#!/bin/bash
# a script to recursively find and copy files to a desired location
find -name "weblog*tar.gz" -print0 |
while IFS= read -r -d '' f; 
do 
echo "$f"
tar zxf $f
done

# a script to recursively find and copy files to a desired location
find -name "*appprofile*StdErr.log.gz" -print0 |
while IFS= read -r -d '' f; 
do 
echo "$f"
tar zxf $f
done

# a script to recursively find and copy files to a desired location
find -name "*native_stderr.log" -print0 |
while IFS= read -r -d '' f; 
do 
echo "$f"
find -name "*native_stderr.log" | sort | xargs cat > native_stderr_all.log
done

find -name "*native_stderr.log" | sort | xargs cat > native_stderr_all.log

#!bin/sh
for file in /data/xxx/
 do
    if test -d $file 
   then
        echo $file
       for i in `ls $file/*.png`; do cp "$i" `echo "$i" |awk -F '.' '{print $1 "_zip." $2}'`; done
     fi
 done

find -name "*native_stderr.log" | sort | xargs cat > native_stderr_all4.log
find -name "was85*FSAB*StdErr.log.gz"|sort | xargs cp -t ./dealfile/

mkdir -p ./files && cp /tmp/test.txt "$_"
gzip -d  压缩文件
gunzip  压缩文件

find -name "*native_stderr.log" | sort | xargs cat > native_stderr_all4.log
find -name "*native_stderr.log" | xargs ls -lrt | xargs cat > native_stderr_all4.log
find -name "*native_stderr.log" -exec 'cat' {} \; > native_stderr_all3.log