shell sed 积累

xshell

    a, 远程登录ssh 等；  ssh 可以证书或者  用户名密码登录

    b， 文件上传下载：

          配置；

           rz   上传文件到unix 服务器

           sz  下载

find and 统计
find ./ -type d -name "*v1*"|wc -l
find ./ -type d -name "*wan*"|wc -l



list：
 for i in `find ./ -type d -name "*tonn*"`;do echo $i;done
 
 

查询 拷贝

for i in `find ./ -type d -name "*tonn*"`;do rsync -avP $i /data/aa/;done

1， 生成全目录的list：

[root@10-10-159-137 ip_all_my_used]# pwd
/data/actlog/ip_all_my_used

[root@10-10-159-137 ip_all_my_used]# find /data/actlog/ip_all_my_used/ -name "*.tar.gz"  > service.list

/data/actlog/ip_all_my_used/wefasdierfxoede1/act.log.2015-10-06.tar.gz
/data/actlog/ip_all_my_used/wefasdierfxoede1/act.log.2015-08-31.tar.gz

2， 全list文件修改内容：

#!/bin/bash

i=0
echo "current time is: "$(date)
for tarfile in `cat ./service.list`
do
        echo $tarfile
        tar -zxvf $tarfile
        rm -fr $tarfile
        logfile=`ls act.log.*`
        echo "here is my: "$logfile
        echo `ll $logfile`
        sed  -i 's/S771/S257/g' $logfile
        tar czf $tarfile $logfile
        echo "current time is: "$(date)
        rm ./act.log.*
done

service.list 内容如下：

/data/actlog/ip_all_my_used/wefasdierfxoede1/act.log.2015-11-15.tar.gz

3， 几个 用过的sed 替换 或者行删除

   锚定行的结束 如：/sed$/匹配所有以sed结尾的行

   删除 reason=65 结束的行；

   sed  -i 's/reason=65$//g'   act.log.2015-11-15
   sed  -i 's/reason=74$//g'   act.log.2015-11-15
   
   删除所有 && 之间， 含有 banlance的 列；   &balance=14044& ---》》》  &&
   sed  -i 's/&[^&]*balance[^&]*//g'   act.log.2015-11-15
 
   删除所有 && 之间， 含有 banlance的 列（包括前面的&）；   &balance=14044& ---》》》  &
   sed  -i 's/&balance[^&]*//g'   act.log.2015-11-15

4，  shell 脚本例子

[root@10-10-159-137 ip_all_my_used]# find /data/actlog/ip_all_my_used/ -name "*.tar.gz"  > service.list

循环遍历 目录中的tar.gz 文件， 解压， 搜索文件， 然后满足一定条件的， 输出到指定文件（目录拼接而成）中。

#!/bin/bash

i=0
echo "current time is: "$(date)
for tarfile in `cat ./service.list`
do
        a=${tarfile%".tar.gz"}
        path1=${a#"/data/actlog/"}
        path=${path1//"/"/"_"}
        tar -zxvf $tarfile
        for line in `cat ./act.log.*`
        do
                #echo $line
                if [[ $line == *interface=yb_expend* ]]
                then
                        echo $line >> ./yb_expend/$path"yb_expend".txt
                elif [[ $line == *interface=shop_buy* ]]
                then
                        echo $line >> ./shop_buy/$path"shop_buy".txt
                elif [[ $line == *interface=recharge* ]]
                then
                        echo $line >> ./recharge/$path"recharge".txt
                elif [[ $line == *interface=yb_income* ]]
                then
                        echo $line >> ./yb_income/$path"yb_income".txt
                elif [[ $line == *interface=logout* ]]
                then
                        echo $line >> ./logout/$path"logout".txt
                fi
        done
        i=$(($i+1))
        echo "current time is: "$(date)
        echo "is first parseing log file of: "$i" and name is: "$tarfile
        rm ./act.log.*
done

5， 批量更改当前目录：

#!/bin/bash

echo "current time is: "$(date)
i=0
for tarfile in `ls .`
do
        mv $tarfile "bf4c52cbd9cd_"$i
        i=$(($i+1))
done

6， 批量运行 一系列脚本：

# cat  run_parse.sh

echo "here make dir" >> timeinfo.txt
date >> timeinfo.txt
./mymkdir.sh

echo "here firstparse" >> timeinfo.txt
date >> timeinfo.txt
./firstparse.sh


echo "here make dir" >> timeinfo.txt
date >> timeinfo.txt
./recharge.sh


echo "here  shop_buy" >> timeinfo.txt
date >> timeinfo.txt
./shop_buy.sh


echo "here parse yb_expend" >> timeinfo.txt
date >> timeinfo.txt
./yb_expend.sh


echo "here parse yb_income" >> timeinfo.txt
date >> timeinfo.txt
./yb_income.sh


echo "here parse logout" >> timeinfo.txt
date >> timeinfo.txt
./logout.sh

7，   去除 某些 表头的 列名；

#!/bin/bash

rm -fr tt.txt
ls  yb_expend  > tt.txt
#find ./yb_expend/ -name "*.txt" > tt.txt


let i=0
for tarfile in `cat tt.txt`
do
  echo ./"yb_expend"/$tarfile
  sed -i 's/interface=/\ /g' ./"yb_expend"/$tarfile
  sed  -i 's/&gname=/, /g'   ./"yb_expend"/$tarfile
  sed   -i 's/&gid=/, /g'   ./"yb_expend"/$tarfile
  sed  -i 's/&sid=/, /g'    ./"yb_expend"/$tarfile
  sed  -i 's/&oldsid=/, /g'  ./"yb_expend"/$tarfile
  sed   -i 's/&user=/, /g'  ./"yb_expend"/$tarfile
  sed   -i 's/&roleid=/, /g'  ./"yb_expend"/$tarfile
  sed   -i 's/&dept=/, /g'  ./"yb_expend"/$tarfile
  sed  -i 's/&time=/, /g'  ./"yb_expend"/$tarfile
  sed  -i 's/&amount=/, /g'  ./"yb_expend"/$tarfile
  sed   -i 's/&balance=/, /g' ./"yb_expend"/$tarfile
  sed  -i 's/&level=/, /g'  ./"yb_expend"/$tarfile
  sed  -i 's/&reason=/, /g' ./"yb_expend"/$tarfile

  cat ./"yb_expend"/$tarfile >> yb_expend_after.txt
  i=$(($i+1))
done

echo "yb_expend file number is"$i
echo "done yb_expend  , and yb_expend file number is "$i >> result_log.txt

rm -fr tt.txt

#!/bin/bash

rm -fr tt.txt
ls  yb_expend  > tt.txt
#find ./yb_expend/ -name "*.txt" > tt.txt


let i=0
for tarfile in `cat tt.txt`
do
  echo ./"yb_expend"/$tarfile
  sed  -i 's/reason=65$//g'  ./"yb_expend"/$tarfile
  sed  -i 's/reason=74$//g'  ./"yb_expend"/$tarfile  #删除reason=74的行列
  sed  -i 's/&balance[^&]*//g'  ./"yb_expend"/$tarfile  #删除balance 列
  sed -i 's/interface=/\ /g' ./"yb_expend"/$tarfile
  sed  -i 's/&gname=/, /g'   ./"yb_expend"/$tarfile
  sed   -i 's/&gid=/, /g'   ./"yb_expend"/$tarfile
  sed  -i 's/&sid=/, /g'    ./"yb_expend"/$tarfile
  sed  -i 's/&oldsid=/, /g'  ./"yb_expend"/$tarfile
  sed   -i 's/&user=/, /g'  ./"yb_expend"/$tarfile
  sed   -i 's/&roleid=/, /g'  ./"yb_expend"/$tarfile
  sed   -i 's/&dept=/, /g'  ./"yb_expend"/$tarfile
  sed  -i 's/&time=/, /g'  ./"yb_expend"/$tarfile
  sed  -i 's/&amount=/, /g'  ./"yb_expend"/$tarfile
  sed   -i 's/&balance=/, /g' ./"yb_expend"/$tarfile
  sed  -i 's/&level=/, /g'  ./"yb_expend"/$tarfile
  sed  -i 's/&reason=/, /g' ./"yb_expend"/$tarfile

  cat ./"yb_expend"/$tarfile >> yb_expend_after.txt
  i=$(($i+1))
done

echo "yb_expend file number is"$i
echo "done yb_expend  , and yb_expend file number is "$i >> result_log.txt

rm -fr tt.txt

#!/bin/bash

rm -fr tt.txt
ls  shop_buy  > tt.txt
#find ./shop_buy/ -name "*.txt" > tt.txt

let i=0
for tarfile in `cat tt.txt`
do
  echo ./"shop_buy"/$tarfile
  sed -i 's/interface=/\ /g' ./"shop_buy"/$tarfile
  sed  -i 's/&gname=/, /g'   ./"shop_buy"/$tarfile
  sed   -i 's/&gid=/, /g'   ./"shop_buy"/$tarfile
  sed  -i 's/&sid=/, /g'    ./"shop_buy"/$tarfile
  sed  -i 's/&oldsid=/, /g'  ./"shop_buy"/$tarfile
  sed   -i 's/&user=/, /g'  ./"shop_buy"/$tarfile
  sed   -i 's/&roleid=/, /g'  ./"shop_buy"/$tarfile
  sed   -i 's/&dept=/, /g'  ./"shop_buy"/$tarfile
  sed  -i 's/&time=/, /g'  ./"shop_buy"/$tarfile
  sed   -i 's/&balance=/, /g' ./"shop_buy"/$tarfile
  sed  -i 's/&itemid=/, /g' ./"shop_buy"/$tarfile
  sed  -i 's/&amount=/, /g'  ./"shop_buy"/$tarfile
  sed   -i 's/&money=/, /g'  ./"shop_buy"/$tarfile
  sed  -i 's/&level=/, /g'  ./"shop_buy"/$tarfile
  sed  -i 's/&remain=/, /g' ./"shop_buy"/$tarfile
  sed  -i 's/&map_id=/, /g' ./"shop_buy"/$tarfile

  cat ./"shop_buy"/$tarfile >> shop_buy_after.txt
  i=$(($i+1))
done

echo "shop_buy file number is"$i
echo "done shop_buy , and shop_buy file number is "$i >> result_log.txt
rm -fr tt.txt

#!/bin/bash

rm -fr tt.txt
ls  yb_income  > tt.txt
#find ./yb_income/ -name "*.txt" > tt.txt

let i=0
for tarfile in `cat tt.txt`
do
  echo ./"yb_income"/$tarfile
  sed -i 's/interface=/\ /g' ./"yb_income"/$tarfile
  sed  -i 's/&gname=/, /g'   ./"yb_income"/$tarfile
  sed   -i 's/&gid=/, /g'   ./"yb_income"/$tarfile
  sed  -i 's/&sid=/, /g'    ./"yb_income"/$tarfile
  sed  -i 's/&oldsid=/, /g'  ./"yb_income"/$tarfile
  sed   -i 's/&user=/, /g'  ./"yb_income"/$tarfile
  sed   -i 's/&roleid=/, /g'  ./"yb_income"/$tarfile
  sed   -i 's/&dept=/, /g'  ./"yb_income"/$tarfile
  sed  -i 's/&time=/, /g'  ./"yb_income"/$tarfile
  sed  -i 's/&amount=/, /g'  ./"yb_income"/$tarfile
  sed   -i 's/&balance=/, /g' ./"yb_income"/$tarfile
  sed  -i 's/&level=/, /g'  ./"yb_income"/$tarfile
  sed  -i 's/&reason=/, /g'  ./"yb_income"/$tarfile

  cat ./"yb_income"/$tarfile >> yb_income_after.txt
  i=$(($i+1))
done

echo "yb_income file number is"$i
echo "done yb_income, and yb_income file number is "$i  >> result_log.txt
rm -fr tt.txt

#!/bin/bash

rm -fr tt.txt
ls  logout  > tt.txt
#find ./logout/ -name "*.txt" > tt.txt

let i=0
for tarfile in `cat tt.txt`
do
  echo ./"logout"/$tarfile
  sed -i 's/interface=/\ /g' ./"logout"/$tarfile
  sed  -i 's/&gname=/, /g'   ./"logout"/$tarfile
  sed   -i 's/&gid=/, /g'   ./"logout"/$tarfile
  sed  -i 's/&sid=/, /g'    ./"logout"/$tarfile
  sed  -i 's/&oldsid=/, /g'  ./"logout"/$tarfile
  sed   -i 's/&user=/, /g'  ./"logout"/$tarfile
  sed   -i 's/&roleid=/, /g'  ./"logout"/$tarfile
  sed   -i 's/&dept=/, /g'  ./"logout"/$tarfile
  sed  -i 's/&time=/, /g'  ./"logout"/$tarfile
  sed  -i 's/&level=/, /g'  ./"logout"/$tarfile
  sed  -i 's/&map_id=/, /g'  ./"logout"/$tarfile
  sed   -i 's/&ip=/, /g'  ./"logout"/$tarfile
  sed   -i 's/&onlinetime=/, /g' ./"logout"/$tarfile
  sed  -i 's/&group=/, /g'  ./"logout"/$tarfile

  cat ./"logout"/$tarfile >> logout_after.txt
  i=$(($i+1))
done

echo "logout file number is"$i
echo "done logout, and logout file number is "$i  >> result_log.txt
rm -fr tt.txt

#!/bin/bash

#rm -fr tt.txt
ls  recharge  > tt.txt
#find ./recharge/ -name "*.txt" > tt.txt

i=0
for tarfile in `cat tt.txt`
do
  echo ./"recharge"/$tarfile
  sed -i 's/interface=/\ /g' ./"recharge"/$tarfile
  sed  -i 's/&gname=/, /g'   ./"recharge"/$tarfile
  sed   -i 's/&gid=/, /g'   ./"recharge"/$tarfile
  sed  -i 's/&sid=/, /g'    ./"recharge"/$tarfile
  sed  -i 's/&oldsid=/, /g'  ./"recharge"/$tarfile
  sed   -i 's/&user=/, /g'  ./"recharge"/$tarfile
  sed   -i 's/&roleid=/, /g'  ./"recharge"/$tarfile
  sed   -i 's/&dept=/, /g'  ./"recharge"/$tarfile
  sed  -i 's/&time=/, /g'  ./"recharge"/$tarfile
  sed  -i 's/&amount=/, /g'  ./"recharge"/$tarfile
  sed   -i 's/&money=/, /g'  ./"recharge"/$tarfile
  sed   -i 's/&balance=/, /g' ./"recharge"/$tarfile
  sed  -i 's/&level=/, /g'  ./"recharge"/$tarfile
  sed  -i 's/&order=/, /g'  ./"recharge"/$tarfile
  sed  -i 's/&channel=/, /g' ./"recharge"/$tarfile
  cat ./"recharge"/$tarfile >> recharge_after.txt
  i=$(($i+1))
done

echo "recharge file number is"$i
echo "done recharge , and recharge file number is "$i >> result_log.txt
rm -fr tt.txt
~                  

8,

#cat list_make_base_para.sh

遍历101.× 的目录，  然后查找所有 *.tar.gz 文件  到 service.list 中。

[root@10-10-159-137 ip_101.198]# vim list_make_base_para.sh

#!/bin/bash

#echo $1

find . -name "101.*"  > ip_101.198.txt

sed  -i 's/.\// /g'  ip_101.198.txt

for ip_addr in `cat ./ip_101.198.txt`
do
  find "/data/actlog/"$ip_addr -name "*.gz" > ip_101.198_list.txt
done

9， shell 分隔符

#!/bin/bash
oldIFS=$IFS
echo $IFS

IFS=$':'
for i in `cat /etc/passwd`;
do
echo $i
done;
IFS=$oldIFS

10， 追加文件到 文件末尾

#!/bin/bash

i=0
echo "current time is: "$(date)

for fullfile in `ls yb_expend`
do
        fill=${fullfile:0:18}
        tarfile="bf4c52cbd9cd/"$fill".tar.gz"
        echo $tarfile
        echo $fullfile
        tar -zxvf $tarfile
        rm -fr $tarfile
        logfile=`ls act.log.*`
        echo "here is my: "$logfile
        sed  -i 's/gameinfo interface=yb_expend*//g'  $logfile

        linenumbers=`sed -n '$='  $logfile`
        centerline=`expr $linenumbers / 2`
        echo ./yb_expend/$fullfile


        cat ./yb_expend/$fullfile >> $logfile
        #sed -i '200r ./yb_expend/$fullfile '  $logfile
        tar czf $tarfile $logfile
        echo "current time is: "$(date)
        rm ./act.log.*
done

11： 修改   修改时间

for fullfile in `ls .`
do
        fill=${fullfile:8:4}${fullfile:13:2}${fullfile:16:2}
        fill=$(($fill+1))

        touch -d "$fill 01:03:00"  $fullfile
        echo "$fill 01:03:00"
done