shell脚本采集ftp数据样例

关注

原创最新推荐文章于 2025-12-02 11:36:05 发布 484 阅读 AI 写同款· GEO 优化 ›

shell脚本上传文件到ftp服务器,shell脚本实现FTP自动上传文件 -----多个文件-----#!/bin/bashftp -n<open 172.20.10.242user logftp logftpbinarycd /data/ftp/pcidata/pcilogftp/AppFile/loglcd /data/localaccpromptmget *.gzclosebye!----单个文件-----#!/bin/bashftp -n<open ... 阅读详情

#!/bin/bash
#author zcb
# *************************************
# 功能描述:
# 	检查从ftp 下载数据情况
# 输出文件:
#	remain_${month_id}_no_uploaded.txt	未上传到ftp列表
#	remain_head.txt				未上传到ftp列表-表头
#	remain_${month_id}.txt			未下载或者未下载到本地列表
# *************************************

month_id=$1
if [ -z "$1" ]
then
	month_id=`date -d "-1 month" +%Y%m`
fi
next_month_id=`date -d "${month_id}15 +1 month" +%Y%m`

selfpath=$(cd $(dirname $0); pwd)
cd $selfpath

>ftp_month_ports.list

for filename in `cat ftp_normal_ports.list`
do
  echo $filename "----"
  filename_prefix=${filename%_*}
  filename_head=${filename:0:6} 
  filename_month=${filename:0-9:6}
  filename_prov=${filename:0-3}
  echo $filename_prefix
  echo $filename_head
  echo $filename_month
  echo $filename_prov
  if [ "$filename_prov" = "099" ]; then
      echo $filename_prefix"_"$month_id"099" >> ftp_month_ports.list
  else
 
      for prov_id in 010 011 013 017 018 019 030 031 034 036 038 050 051 059 070 071 074 075 076 079 081 083 084 085 086 087 088 089 090 091 097; do
          if [ "$filename_head" = "C01013" ]; then
             last_day=`cal ${month_id:5} ${month_id:0:4}|xargs|awk '{print $NF}'`
             echo $filename_prefix"_"$month_id$last_day$prov_id >> ftp_month_ports.list
          elif [[ "$filename_head" = "C01125" || "$filename_head" = "C01126" ]]; then
	    echo $filename_prefix"_"$next_month_id$"15"$prov_id >> ftp_month_ports.list
	  else
             echo $filename_prefix"_"$month_id$prov_id >> ftp_month_ports.list
          fi
      done
  fi
done


echo `date +"%F %T"` 连接ftp获取文件列表 start
ftp -niv <<- EOF 
open 192.168.1.18   
user user_name passwd
bin 
cd month
ls C01*${month_id}* ftp_ports_${month_id}.list.a
ls C01*${next_month_id}* ftp_ports_${month_id}.list.b
bye   
EOF

echo `date +"%F %T"` 连接ftp获取文件列表 end

cat ftp_ports_${month_id}.list.a | grep -v C01125 | grep -v C01126 > ftp_ports_${month_id}.list
cat ftp_ports_${month_id}.list.b | grep -e C01125 -e C01126 >> ftp_ports_${month_id}.list

rm -f ftp_ports_${month_id}.list.a ftp_ports_${month_id}.list.b

> file_checkresults_${month_id}.txt

ls -l /mnt/sd03/data/C01*${month_id}*|grep -v MD5 > file_collect_${month_id}.txt
ls -l /mnt/sd02/data/data_month/C01*${month_id}*|grep -v MD5 > file_collect_${month_id}.txt
ls -l /mnt/sd07/data/C01*${month_id}*|grep -v MD5 >> file_collect_${month_id}.txt
ls -l /mnt/sd04/data/C01*${month_id}*|grep -v MD5 >> file_collect_${month_id}.txt
ls -l /mnt/sd05/data/C01*${month_id}*|grep -v MD5 | grep -v C01125 | grep -v C01126 >> file_collect_${month_id}.txt
ls -l /mnt/sd05/data/C01*${next_month_id}*|grep -v MD5 | grep -e C01125 -e C01126 >> file_collect_${month_id}.txt
file=`cat ftp_month_ports.list`

echo `date +"%F %T"` 开始对比大小
for filename in $file
do
	get_date=`grep ${filename} file_collect_${month_id}.txt|awk '{print $6,$7,$8}'`
	file_size=`grep ${filename} file_collect_${month_id}.txt|awk '{print $5}'`
	up_date=`grep ${filename} ftp_ports_${month_id}.list|grep -v MD5|awk '{print $6,$7,$8}'`
	up_size=`grep ${filename} ftp_ports_${month_id}.list|grep -v MD5|awk '{print $5}'`
	if [ ! -n "${up_size}" ]; then  
	  up_size=0
	fi
	if [ ! -n "${file_size}" ]; then
	  file_size=0
	fi
	chaa=$((${up_size} - ${file_size}))
	echo ${filename}"--"${file_size}" ------ "${up_size}" ------- "$chaa
	cha=$[${up_size} - ${file_size}]
	echo ${filename}"|"${file_size}"|"${up_size}"|"${cha} >> file_checkresults_${month_id}.txt
done

cat  ${selfpath}/file_checkresults_${month_id}.txt |awk -F '|' '{if($3==0){print $1"|"$2"|"$3"|"$4} }' > ${selfpath}/remain_${month_id}_no_uploaded.txt
cat  ${selfpath}/file_checkresults_${month_id}.txt |awk -F '|' '{if($3==0){print $1} }' | \
gawk -F "_" '{print $1}' | sort | uniq > ${selfpath}/remain_head.txt
cat  ${selfpath}/file_checkresults_${month_id}.txt |awk -F '|' '{if($3!=0 && $4 > 0){print $1"|"$2"|"$3"|"$4} }' > ${selfpath}/remain_${month_id}.txt

echo `date +%F\ %T` 未下载文件 头 ========================
cat ${selfpath}/remain_${month_id}.txt
echo `date +%F\ %T` 未下载文件 尾 ========================
echo `date +"%F %T"` 结束

#########################################################################################
# 开始下载数据

echo `date +"%F %T"` 开始从ftp下载数据到 /mnt/sd02/data/data_month
stime=`date +%s`

for file in $(cat remain_${month_id}.txt |grep -v "MD5" | grep -v "C0105[2-9]"| gawk -F '|' '/C010[0-1]/{print $1}')
do

        fileName=$(grep ${file} remain_${month_id}.txt|grep -v "MD5"|awk -F '|' '{print $1}')
        fileSize=$(grep ${file} remain_${month_id}.txt|grep -v "MD5"|awk -F '|' '{print $3}')

        echo "`date +%Y-%m-%d\ %H:%M:%S`--采集文件信息："
        echo "`date +%Y-%m-%d\ %H:%M:%S`--文件名：$fileName"
        echo "`date +%Y-%m-%d\ %H:%M:%S`--文件大小：$fileSize"

ftp -niv <<- EOF
open 192.168.1.18
user user_name passwd
lcd /mnt/sd02/data/data/data_month
cd month
passive
bin
mget ${fileName}*   
bye   
EOF

done

echo `date +"%F %T"` 结束从ftp下载数据到 /mnt/sd02/data

echo `date +"%F %T"` 开始从ftp下载数据到 /mnt/sd03/data
stime=`date +%s`

for file in $(cat remain_${month_id}.txt |grep -v "MD5" | grep -v "C0105[2-9]"| gawk -F '|' '/C010[2-5]/{print $1}')
do

        fileName=$(grep ${file} remain_${month_id}.txt|grep -v "MD5"|awk -F '|' '{print $1}')
        fileSize=$(grep ${file} remain_${month_id}.txt|grep -v "MD5"|awk -F '|' '{print $3}')

        echo "`date +%Y-%m-%d\ %H:%M:%S`--采集文件信息："
        echo "`date +%Y-%m-%d\ %H:%M:%S`--文件名：$fileName"
        echo "`date +%Y-%m-%d\ %H:%M:%S`--文件大小：$fileSize"

ftp -niv <<- EOF
open 192.168.1.18
user user_name passwd
lcd /mnt/sd03/data
cd month
passive
bin
mget ${fileName}*   
bye   
EOF

done

echo `date +"%F %T"` 结束从ftp下载数据到 /mnt/sd03/data

echo `date +"%F %T"` 开始从ftp下载数据到 /mnt/sd04/data

for file in $(cat remain_${month_id}.txt |grep -v "MD5" | grep -v "C0105[0-1]"| gawk -F '|' '/C010[5-9]/{print $1}')
do

        fileName=$(grep ${file} remain_${month_id}.txt|grep -v "MD5"|awk -F '|' '{print $1}')
        fileSize=$(grep ${file} remain_${month_id}.txt|grep -v "MD5"|awk -F '|' '{print $3}')

        echo "`date +%Y-%m-%d\ %H:%M:%S`--采集文件信息："
        echo "`date +%Y-%m-%d\ %H:%M:%S`--文件名：$fileName"
        echo "`date +%Y-%m-%d\ %H:%M:%S`--文件大小：$fileSize"

ftp -niv <<- EOF
open 192.168.1.18
user user_name passwd
lcd /mnt/sd04/data
cd month
passive
bin
mget ${fileName}*   
bye   
EOF

done

echo `date +"%F %T"` 结束从ftp下载数据到 /mnt/sd04/data

echo `date +"%F %T"` 开始从ftp下载数据到 /mnt/sd05/data

for file in $(cat remain_${month_id}.txt | gawk -F '|' '/C011[0-2]/{print $1}')
do
        fileName=$(grep ${file} remain_${month_id}.txt | awk -F '|' '{print $1}')
        fileSize=$(grep ${file} remain_${month_id}.txt | awk -F '|' '{print $3}')

        echo "`date +%Y-%m-%d\ %H:%M:%S`--采集文件信息："
        echo "`date +%Y-%m-%d\ %H:%M:%S`--文件名：$fileName"
        echo "`date +%Y-%m-%d\ %H:%M:%S`--文件大小：$fileSize"

ftp -niv <<- EOF
open 192.168.1.18
user user_name passwd
lcd /mnt/sd05/data
cd month
passive
bin
mget ${fileName}*
bye
EOF

done

etime=`date +%s`
s=`echo "scale=0; ($etime - $stime)%60" | bc`
m=`echo "scale=0; ($etime - $stime)/60%60" | bc`
h=`echo "scale=0; ($etime - $stime)/60/60" | bc`

echo `date +"%F %T"` 结束从ftp下载数据到 /mnt/sd05/data
echo `date +"%F %T"` end 耗时 $h 小时 $m 分钟 $s 秒

收录于