代元培

统计Shell脚本执行时间

Shell脚本执行SQL语句，获得结果发邮件

脚本获取线上所有机器日志集中到本地单个文件

用于从sftp集群中下载数据文件到本地

DataX脚本任务开发实例记录

#!/bin/sh#监控云上pay-data-load脚本执行进度current_path=$(cd "$(dirname $0)"; pwd)data_path="$current_path/time/"file_input=$current_path/file_inputhtml_file=$current_path/pay_data_load.htmlif [ -f "$html_file" ];then rm $html_filefi#if [ -f "$file_...

# 统计第4列为0112并且第5列为00或者03的行数awk -F '|' '{if(($4=="0112")&&($5=="00"||$5=="03")) print $1}' epcc0303_20210423.txt|wc -l# 去重展示第一列的时间awk -F '|' '{print $1}' epcc0303_20210422.txt|sort|uniq

#!/bin/bash. ~/.bashrcdateEnd=`date +"%Y%m%d"`dateBegin=`date +%Y%m%d -d"-30 day $dateEnd"`if [ $# == 1 ]; then if [ ! -n "$(echo $1| sed -n "/^[0-9]\+$/p")" ]; then echo "FATAL: The parameter is not a number." exit -1 fi i.

20000000000069|950615951|2019-04|3.7020000000000069|950615951|2019-05|1.3020000000000069|950615951|2019-06|1.3020000000000069|950615951|2019-08|1.5020000000000069|950615951|2019-09|1.4020000000000069|950615951|2019-10|1.6020000000000069|950615951|20.

#!/bin/bash. ~/.bashrcif [ $# == 1 ]; then dateBegin=$1 dateEnd=$dateBeginelif [ $# == 2 ]; then dateBegin=$1 dateEnd=$2else echo "dateBegin or dateEnd is empty." exit -1fischema="fn"tbname="t_bank_return"dateBegin=`date +.

# 当前脚本路径currentPath=$(cd "$(dirname $0)";pwd)cd $currentPathcd ..# 数据类型dataType=$1# 获取当前时间记录文件getDate=`head time/${dataType}.time`# 当前时间+1天getDate=`date -d"+1 day $getDate" +%Y%m%d`# 更新时间写入文件echo $getDate > time/${dataType}.time# 删除3个.

# test.sh# 在当前目录下创建文本文件temp，如果文件存在则清空文件$(> temp)# for 循环将参数追加到当前目录的temp文件，逗号分隔，echo -n 不换行for i in $*;do ((n++)) # 从第四个开始拼接 if [[ n -gt 3 ]];then echo -n ${i}, >> temp fidone# str取temp文本里的字符串str=$(cat temp)# 将字符串最后的一个逗号去掉str=${str%.

#!/bin/bashCUR_DIR=$(cd "$(dirname "${BASH_SOURCE-$0}")";pwd)exec 1>>$CUR_DIR"/log/sh.log"exec 2>>$CUR_DIR"/log/sh.log.wf" . ~/.bashrcfilename=$1readfile=$CUR_DIR/var/${filename}echo $datafilecurrentdate=`date +'%Y%m%d'`current=`date .

#!/bin/bashCUR_DIR=$(cd "$(dirname "${BASH_SOURCE-$0}")";pwd)exec 1>>$CUR_DIR"/log/spIdToSpUid.log"exec 2>>$CUR_DIR"/log/spIdToSpUid.log.wf" . ~/.bashrcfileName=$1dateStr=`date +'%Y%m%d'`current=`date "+%Y-%m-%d %H:%M:%S"`timeStamp=`d.

#! /bin/bash# 基于 odp3.0的nginx，适用于odp3.0以下和odp3.0版本的nginx ,增加强制kill container里面的nginx的逻辑# 20160129 升级chkconfig 增加创建log/webserver 目录部分#set -eset -Etrap 'echo "Fail unexpectedly on ${BASH_SOURCE[0]}:$LINENO!" >&2' ERRODP_ROOT=$(readlink -f `.

app_name='pay-data-platform'app_path=$(cd $(dirname $0); pwd)command=$1cmd_nginx="${app_path}/webserver/loadnginx.sh"cmd_fpm="${app_path}/php/sbin/php-fpm"if [ "$command" == 'start' ]; then $cmd_fpm start $cmd_nginx startelif [ "$command" == 'res.

wget -O output2.tar.gz http://irep.build.daiyp.com/product/v3/download/release/dxm/pay-core/payserver-conf/latest/output.tgzif [ ! -s output2.tar.gz ];then echo '404';fi

#!/bin/bash# load_dypserver.shname='dypserver'SV="./supervise.$name"unalias pidof &>/dev/nullexec 0</dev/null#ulimit -c unlimitedcd `dirname $0`#LIBRDKAFKA_DIR=`pwd`/lib#export LD_LIBRARY_PATH=$LIBRDKAFKA_DIR:$LD_LIBRARY_PATHif [ ! -.

if [ `grep -c "*.*.*.*" $CURRENT_DIR/conf/ubclient.conf` -eq '0' ];thenecho -e "\n" >> $CURRENT_DIR/conf/ubclient.confcat>>$CURRENT_DIR/conf/ubclient.conf<<EOF[.UbClient][..@Service]Name : testConnectAll : 0DefaultConnectTimeOut.

#!/bin/bash. ~/.bashrctime=$(date "+%Y-%m-%d %H:%M:%S")echo "${time}"for ((i=0;i<100;i++));do printf "%02d " $i; db_no=`printf "%02d\n" $i` sed 's/_62/_'"$db_no"'/g' translog.txt > translog_${db_no}.txt exit nohup mysql -h1.1.1.

#!/bin/sh# 监控报表PHP_EXE=${PHP_EXE:=/home/work/scripts/php/bin/php}PHP_INI=${PHP_INI:=/home/work/scripts/php/etc/php.ini}ROOT=$(cd "$(dirname $0)"; pwd)OUT="$ROOT/offline_transfer.out"MAIN="$ROOT/imp/OfflineTransfer.php"current_date=`date +%Y%m%d`.

#!/bin/bash. ~/.bashrcfor((i=0;i<1000;i+=4));doecho \{\"jdbcUrl\":[\"jdbc:mysql://1.1.1.1:5500/dyp_db\"],\"table\":[\"t_did_2_uid__tablet_${i}\"]\}, >> make-conf.txtdoneecho "make-conf.sh done"

文件表达式-e filename 如果 filename存在，则为真-d filename 如果 filename为目录，则为真-f filename 如果 filename为常规文件，则为真-L filename 如果 filename为符号链接，则为真-r filename 如果 filename可读，则为真-w filename 如果 filename可写，则为真-x filename 如果 filename可执行，则为真-s filename 如果文件长度不为0，则为真-h ...

path=$(pwd)"/odp_auto"code=0need_code=0need_conf=0odp=0start=0stop=0proxy=0 while [ -n "$1" ]; do # -n str,字符串不为null,长度大于零 case "$1" in -path) path=$2 ;; -code) code=$2 need_code=1 ;; -conf).

#!/bin/bash. ~/.bashrctime=$(date "+%Y-%m-%d %H:%M:%S")echo "${time}"f_day=`date +%Y%m01`echo ${f_day}sed "s/F_DAY/${f_day}/g" t_bi_cost_sql > t_bi_cost.sqlcat t_bi_cost.sql | gp_rd_h > PAY_TO_HY_DEPT_FEE.CSVsed -i '$d' PAY_TO_HY_DEPT_FEE.CS.

# 当前日期current_date=`date +%Y%m%d`time=$(date "+%Y-%m-%d %H:%M:%S")echo "${time}"# 当月第一天f_day=`date +%Y%m01`echo ${f_day}# 昨天日期targetDay=`date -d "-1 days" +"%Y%m%d"`yesterday_date=`date -d 'yesterday' +%Y-%m-%d`# 前天日期qtday=`date -d '2 day ago' .

# 删除文档的第一行sed -i '1d' <file># 删除文档的最后一行sed -i '$d' <file>

#!/bin/bash# run in v2basepath=$(cd `dirname $0`; pwd)DATAPATH="/home/work/datax/ext"SERVICEPORT=9902LOGFILE="/home/work/gpfdist/gp-client/log/gpfdist.log"TIMEOUT=300if [ -f ${LOGFILE} ]then datenow=` date +'%Y%m%d'` mv ${LOGFILE} ${LO.

#!/bin/bash. ~/.bashrcexport LANG="zh_CN.UTF-8"export LC_ALL="zh_CN.UTF-8"current_date=`date +%Y%m%d`echo ${current_date}basepath=$(cd `dirname $0`; pwd)cd ${basepath}tempfile="${basepath}/temp.log"export PGPASSWORD=******ONE_SQL="\\copy (select.

# 将密码写在客户端服务器的用户家目录下，创建一个.pgpass文件，并将权限设置为0600，就可以实现了# .pgpass文件格式如下，每行一个hostname:port:database:username:password# shell执行命令psql -h 地址 -p 端口 -U 用户 -c "CREATE DATABASE test"alias gp_rd='psql -h x.x.x.x -p 5432 -d gp_pay -U gp_rd...

seq 99seq -f %02.0f 99seq -f %02.0f 100 120

#!/bin/bash. ~/.bashrcif [ $# -lt 1 ]then echo "./loaddata db_no" exitfidb_no=$1time=$(date "+%Y-%m-%d %H:%M:%S")echo "${time}"example="/home/work/datax/job/core/example"tempdir="/home/work/datax/job/core"basepath=$(cd `dirname $0`; pwd.

str='0099'new_str=$(echo -e $str | sed -r 's/0*([0-9])/\1/') # 99

#!/bin/sh. ~/.bashrcre="[[:digit:]]+"fields="F_sp_id, F_sp_user_id, F_sp_company, F_sp_display_name, F_create_time"if [[ $1 =~ $re ]]; then fndb -e "SELECT $fields FROM db_name.t_scenario_sp_map WHERE F_sp_id = '$1'\G" exit 0elif [ "$1" =.

awk -F' ' '{print $1}' uids | xargs | tr ' ' ',' >> uids_row

cat a.txt | awk '{sum+=$1} END {print "Sum = ", sum}'

#!/bin/shROOT=$(cd "`dirname $0`/.."; pwd)PHP_EXE=${PHP_EXE:=/home/work/pay-data-scripts/php/bin/php}PHP_INI=${PHP_INI:=/home/work/pay-data-scripts/php/etc/php.ini}channelCode=if [ $# == 1 ];...

#!/bin/bash. ~/.bashrccurrent_date=`date +%Y-%m-%d`echo ${current_date}basepath=$(cd `dirname $0`; pwd)cd $basepathMONITOR_BEGIN=/home/work/daiyuanpei/monitor-beginif [ "`ls -A ${MONITOR_BEGIN...

#!/bin/bash. ~/.bashrccurrent_date=`date +%Y-%m-%d`echo ${current_date}basepath=$(cd `dirname $0`; pwd)cd $basepathFAIL_PATH=/home/work/daiyuanpei/monitor-failif [ "`ls -A ${FAIL_PATH}`" = ""...

#!/bin/bash. ~/.bashrcecho $#if [ $# -lt 1 ]then echo "need job config file" exitfijobname="$1"taskname=`echo $(basename $jobname .xml)`echo "$jobname,taskname=$taskname"DATAX_HOME=...