1. 安装和配置
brew install sqoop
sqoop version
cd /usr/local/Cellar/sqoop/1.4.6/libexec/conf
cp sqoop-env-template.sh sqoop-env.sh
vim sqoop-env.sh
export HADOOP_HOME="/usr/local/Cellar/hadoop/3.0.0/libexec"
export HBASE_HOME="/usr/local/Cellar/hbase/1.2.6"
export HIVE_HOME="/usr/local/Cellar/hive/2.3.1"
export ZOOCFGDIR="/usr/local/Cellar/zookeeper/3.4.10/"
curl -L 'http://www.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.42.tar.gz/from/http://mysql.he.net/' | tar xz
cp mysql-connector-java-5.1.42/mysql-connector-java-5.1.42-bin.jar /usr/local/Cellar/sqoop/1.4.6/libexec/lib/
2. 测试
- -- -- --
-- -- -- -- .
3. 遇到的问题
- 问题1
- java.lang.Exception: java.lang.RuntimeException:java.lang.ClassNotFoundException: Class example not found
- 原因和解决方法:hadoop 单机模式下无法找到执行目录,切换到伪分布模式可以解决
- 问题2
- No primary key could be found for table test_table. Please specify one with –split-by or perform a sequential import with ‘-m 1’.
- 原因和解决方法:sqoop导入时依赖于MySQL原表的主键索引进行切片,没有添加主键索引会报错,给原表加上 primary key 或者 在 命令行末尾 加上 -m 1
4. 导入实例
# 创建mysql表
CREATE TABLE `example` (
`id` int(11) NOT NULL,
`name` varchar(20) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
# 插入数据
mysql> select * from example;
+
| id | name |
+
| 1 | hello |
| 2 | world |
+
2 rows in set (0.00 sec)
# 创建 hive 表
create table example(id int, name string) row format delimited fields terminated by '\t' lines terminated by '\n';
# 将mysql表导入hadoop
sqoop import -connect jdbc:mysql://localhost/test
# 将生成的文件导入hive
load data inpath 'hdfs://localhost:9000/user/max/example/part-m-00000' into table example
# 以上两步可以合并为一步完成
sqoop import --connect jdbc:mysql://localhost/test --username root --password mima123456 --table example --hive-import --hive-overwrite --hive-table example --fields-terminated-by '\t'