Linux下Shell编程实现基于Hadoop的ETL(Sqoop封装篇)

conf/sqoop.xml

<?xml version="1.0" encoding="UTF-8"?>
<root>
	<sqoop-shell type="import">
		<param key="connect">jdbc:mysql://master:3306/db_cda</param><!-- 数据库连接 必填-->
		<param key="username">root</param><!-- 数据库用户名 必填-->
		<param key="password">123</param><!-- 数据库密码 必填-->
		<param key="table"></param><!--导入的表 必填-->
		<!--<param key="columns"></param>--><!--导入的表的列-->
		<param key="query"></param><!--数据库查询语句，过滤出要导入的数据 -->
		<param key="where"></param><!--过滤数据-->
		<param key="target-dir"></param><!-- HDFS数据仓库存放位置 -->
		<param key="warehouse-dir"></param><!--导入数据到数据仓库的目录-->
		<param key="hive-table"></param>
		<param key="hive-partition-key"></param>
		<param key="hive-partition-value"></param>
		<param key="m">1</param><!--mapper的数量-->
		<param key="hive-overwrite"></param>
		<param key="hive-delims-replacement"></param>
		<param key="compress"></param><!--true or false 是否启用压缩输出-->
		<param key="compression-codec"></param><!--指定压缩算法-->
		<param key="null-string">'\\\\N'</param>
		<param key="null-non-string">'\\\\N'</param>
		<param key="split-by"></param>
		<param key="map-column-hive"></param>
		<param key="append"></param>
		<param key="delete-target-dir">true</param>
		<param key="as-avrodatafile"></param><!--true or false-->
		<param key="as-sequencefile"></param><!--true or false-->
		<param key="as-textfile">true</param><!--true or false-->
		<param key="hive-import">true</param><!--必须为true-->
	</sqoop-shell>
	
	<sqoop-shell type="export">
		<param key="connect">jdbc:mysql://master:3306/db_cda</param><!-- 数据库连接 -->
		<param key="username">root</param><!-- 数据库用户名 -->
		<param key="password">123</param><!-- 数据库密码 -->
		<param key="table"></param>
		<param key="export-dir"></param><!-- HDFS数据仓库存放位置 -->
		<param key="num-mappers">1</param>
		<!--<param key="input-null-string"></param>-->
		<!--<param key="input-null-non-string"></param>-->
		<!--<param key="input-enclosed-by"></param>-->
		<!--<param key="input-escaped-by"></param>-->
		<param key="input-fields-terminated-by">'\\t'</param>
		<!--<param key="input-lines-terminated-by"></param>-->
		<!--<param key="input-optionally-enclosed-by"></param>-->
		<!--<param key="enclosed-by"></param>-->
		<!--<param key="escaped-by"></param>-->
		<!--<param key="fields-terminated-by"></param>-->
		<param key="lines-terminated-by">'\\n'</param>
		<param key="mysql-delimiters"></param>
		<!--<param key="optionally-enclosed-by"></param>-->
	</sqoop-shell>
</root>

bin/sqoop.sh

#!/bin/bash
#get the environment
if [ -f ~/.bashrc ];#进入Bash Shell环境，用于下面命令执行
then
 . ~/.bashrc

#获取当前命令的路径
bin=`dirname $0`
bin=`cd "$bin";pwd`
filename="$bin"/../conf/sqoop.xml
#if file not exsist,will exit;
fi
if [ -z $filename ];then
	echo 'USAGE:COMMAND FILENAME'
	exit 0
fi

#各种预定义变量及数组
declare -i i=0
declare -i j=0
declare -a importKeys
declare -a importValues
declare -a exportKeys
declare -a exportValues

flag=0
#处理掉配置文件中的空格、空行、以及注释等
content=`sed -e 's/\s*\(.*\)\s*$/\1/g' -e 's/\s*\(=\)\s*/\1/g' -e '/^\(\s\)*$/d' -e '/^$/d' -e 's/<!--.*-->//' $filename`
#然后循环遍历XML文件中的每一行
while read line
do \
	if echo ${line}|grep -qE "^<sqoop-shell type=\"import\">" ; then
		flag=1
		continue
	fi

	if echo ${line}|grep -qE "^<sqoop-shell type=\"export\">" ; then
		flag=2
		continue
	fi

	if echo ${line}|grep -qE "<\/sqoop-shell>" ; then
		flag=0
		continue
	fi

	if [ ${flag} -ne 0 ] ; then
		#处理获得到key值
		key=`echo ${line}|sed -n -e 's/<param key=\"//' -e 's/\">.*<\/param>//p'|tr -d '\r'`
		#处理获得到Value值
		value=`echo ${line}|sed -n -e 's/<param key=\".*\">//' -e 's/<\/param>//p'|tr -d '\r'`
		#如果key值为空，则跳过
		if [ ${#key} -eq 0 ] ; then
			continue
		fi
		#否则根据标记，把key和value加入到导入或者导出数组中
		if [ ${flag} -eq 1 ] ; then
			importKeys[$i]=$key
			importValues[$i]=$value
			let i++
		elif [ ${flag} -eq 2 ] ; then
			exportKeys[$j]=$key
			exportValues[$j]=$value
			let j++
		fi
	fi
done \
<<EOF
$content
EOF

#获取第一个命令
command=$1
shift

#循环处理命令中的值，覆盖掉默认值
while [ $# -ne 0 ]
do
	if [ "$command" = "import" ] ; then
		for((k=0;k<${#importKeys[*]};k++))
		do
			#如果存在默认值，则覆盖掉默认值
			if [ "${1:2}" = "${importKeys[$k]}" ] ; then
				importValues[$k]=${2}
				shift
				shift
			fi				
		done
	elif [ "$command" = "export" ] ; then
		for((k=0;k<${#exportKeys[*]};k++))
		do
			if [ "${1:2}" = "${exportKeys[$k]}" ] ; then
				exportValues[$k]=${2}
				shift
				shift
			fi
		done
	fi
done
#循环拼接命令字符串
sqoop="sqoop"
if [ "$command" = "import" ] ; then
	sqoop="${sqoop} import"
	for((k=0;k<${#importKeys[*]};k++))
	do
		if [ ${#importValues[$k]} -eq 0 ] ; then
			continue
		fi
		#如果命令的值为true,则只进行命令的拼接
		if [ "${importValues[$k]}" = "true" ] ; then
			sqoop="${sqoop} --${importKeys[$k]}"
			continue
		fi
		sqoop="${sqoop} --${importKeys[$k]} ${importValues[$k]}"
	done
elif [ "$command" = "export" ] ;then
	sqoop="${sqoop} export"
	for((k=0;k<${#exportKeys[*]};k++))
	do
		if [ ${#exportValues[$k]} -eq 0 ] ; then
			sqoop="${sqoop} --${exportKeys[$k]}"
			continue
		fi
		sqoop="${sqoop} --${exportKeys[$k]} ${exportValues[$k]}"
	done
fi
#执行拼接后的字符串
eval $sqoop

具体Shell代码下载:http://download.youkuaiyun.com/detail/luo849278597/9490920

加群：397706991，共同学习