1.15-1.16 sqoop action

最新推荐文章于 2021-05-23 16:22:26 发布

weixin_30784141

最新推荐文章于 2021-05-23 16:22:26 发布

阅读量331

点赞数

CC 4.0 BY-SA版权

文章标签：数据库

原文链接：http://www.cnblogs.com/weiyiming007/p/10862248.html

本文详细介绍了如何使用Oozie的Sqoop Action进行数据导入操作，包括配置属性、工作流定义、命令执行及错误处理。通过两个实例演示了直接指定参数和使用选项文件的不同方式。

一、sqoop action1

http://oozie.apache.org/docs/4.0.0/DG_SqoopActionExtension.html

1、准备文件

##
[root@hadoop-senior oozie-apps]# mkdir sqoop-import-user


##job.properties
nameNode=hdfs://hadoop-senior.ibeifeng.com:8020
jobTracker=hadoop-senior.ibeifeng.com:8032
queueName=default
oozieAppsRoot=user/root/oozie-apps
oozieDataRoot=user/root/oozie/datas

oozie.use.system.libpath=true

oozie.wf.application.path=${nameNode}/${oozieAppsRoot}/sqoop-import-user

outputDir=sqoop-import-user/output


##
<workflow-app xmlns="uri:oozie:workflow:0.5" name="sqoop-wf">
    <start to="sqoop-node"/>

    <action name="sqoop-node">
        <sqoop xmlns="uri:oozie:sqoop-action:0.3">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${nameNode}/${oozieDataRoot}/${outputDir}"/>
            </prepare>
            <configuration>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
            <command>import --connect jdbc:mysql://hadoop-senior.ibeifeng.com:3306/test --username root --password 123456 --table my_user --target-dir /user/root/oozie/datas/sqoop-import-user/output --fields-terminated-by "$$$" --num-mappers 1</command>
        </sqoop>
        <ok to="end"/>
        <error to="fail"/>
    </action>

    <kill name="fail">
        <message>Sqoop failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>


##
<command></command>        //sqoop要执行的命令

2、运行sqoopaction

##复制mysql驱动
[root@hadoop-senior oozie-apps]# mkdir sqoop-import-user/lib

[root@hadoop-senior oozie-apps]# cp /opt/softwares/mysql-libs/mysql-connector-java-5.1.27/mysql-connector-java-5.1.27-bin.jar sqoop-import-user/lib/


##上传到hdfs
[root@hadoop-senior oozie-apps]# /opt/cdh-5.3.6/hadoop-2.5.0-cdh5.3.6/bin/hdfs dfs -put sqoop-import-user/ /user/root/oozie-apps/



## 运行
export OOZIE_URL=http://hadoop-senior.ibeifeng.com:11000/oozie/
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# bin/oozie job -config oozie-apps/sqoop-import-user/job.properties -run

二、sqoop action2

1、准备文件

##
[root@hadoop-senior oozie-apps]# mkdir sqoop-import-user2


##job.properties
nameNode=hdfs://hadoop-senior.ibeifeng.com:8020
jobTracker=hadoop-senior.ibeifeng.com:8032
queueName=default
oozieAppsRoot=user/root/oozie-apps
oozieDataRoot=user/root/oozie/datas

oozie.use.system.libpath=true

oozie.wf.application.path=${nameNode}/${oozieAppsRoot}/sqoop-import-user

outputDir=sqoop-import-user/output


##workflow.xml
<workflow-app xmlns="uri:oozie:workflow:0.5" name="sqoop-wf">
    <start to="sqoop-node"/>

    <action name="sqoop-node">
        <sqoop xmlns="uri:oozie:sqoop-action:0.3">
            <job-tracker>${jobTracker}</job-tracker>
            <name-node>${nameNode}</name-node>
            <prepare>
                <delete path="${nameNode}/${oozieDataRoot}/${outputDir}"/>
            </prepare>
            <configuration>
                <property>
                    <name>mapred.job.queue.name</name>
                    <value>${queueName}</value>
                </property>
            </configuration>
            <command>import --options-file imp-user.sql</command>
        </sqoop>
        <ok to="end"/>
        <error to="fail"/>
    </action>

    <kill name="fail">
        <message>Sqoop failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
    </kill>
    <end name="end"/>
</workflow-app>


##imp-user.sql
--connect
jdbc:mysql://hadoop-senior.ibeifeng.com:3306/test
--username
root
--password
123456
--table
my_user
--target-dir
/user/root/oozie/datas/sqoop-import-user/output
--fields-terminated-by
"\t"
--num-mappers
1

2、运行

##
[root@hadoop-senior oozie-apps]# cp /opt/softwares/mysql-libs/mysql-connector-java-5.1.27/mysql-connector-java-5.1.27-bin.jar sqoop-import-user2/lib/

##
[root@hadoop-senior oozie-apps]# ls sqoop-import-user2/
imp-user.sql  job.properties  lib  workflow.xml

[root@hadoop-senior oozie-apps]# 
[root@hadoop-senior oozie-apps]# /opt/cdh-5.3.6/hadoop-2.5.0-cdh5.3.6/bin/hdfs dfs -put sqoop-import-user2/ /user/root/oozie-apps


##运行
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# export OOZIE_URL=http://hadoop-senior.ibeifeng.com:11000/oozie/
[root@hadoop-senior oozie-4.0.0-cdh5.3.6]# bin/oozie job -config oozie-apps/sqoop-import-user2/job.properties -run

转载于:https://www.cnblogs.com/weiyiming007/p/10862248.html