sqoop 根据id append 动态计算id 脚本

#!/bin/bash

########################################################
#                                                      #
#  @author hunter@doitedu                              #
#  @date   ${DT_INCR}                                  #
#  @desc   oms_order_item增量抽取任务 启动脚本         #
#                                                      #
########################################################

export SQOOP_HOME=/opt/apps/sqoop-1.4.7/

# 加载上次import的最大id
if [ -f preid ]
then
START_ID=`cat preid`
else
START_ID=0
fi

res=`mysql -h 192.168.77.2 -uroot -p123456 <<EOF
use realtimedw;
select max(id) from oms_order_item;
EOF`

CUR_MAX_ID=`echo ${res} | awk '{print $2}'`
echo ${CUR_MAX_ID} > preid


DT_EXTRACT=`date -d'-1 day' +%Y-%m-%d`

if [ $1 && $2 ]
then
ID=$1
DT_EXTRACT=$2
fi


echo "上次导入的最大ID为：${START_ID}"
echo "本次导入所能达到的最大ID为： ${CUR_MAX_ID}"


${SQOOP_HOME}/bin/sqoop import \
--connect jdbc:mysql://192.168.77.2:3306/realtimedw \
--username root \
--password 123456 \
--table oms_order_item \
--target-dir "/incr_test/oms_order_item/${DT_EXTRACT}"  \
--incremental append \
--check-column id \
--null-string '\\N'  \
--null-non-string '\\N' \
--last-value "${START_ID}"  \
--fields-terminated-by ','  \
--split-by id  \
-m 1   \

##


if [ $? -eq 0 ]
then 
echo "congratulations! hive表分区加载成功！ 邮件已发送至admin@51doit.com"
else
echo "节哀顺变! 表分区加载失败! 邮件已发送至admin@51doit.com"
fi