sqoop迁移mysql数据到hive中

最新推荐文章于 2024-02-04 12:25:32 发布

原创最新推荐文章于 2024-02-04 12:25:32 发布 · 462 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#sqoop #迁移 #python

sqoop 专栏收录该内容

1 篇文章

订阅专栏

本文介绍了一种使用SQOOP进行数据增量导入的方法，并提供了详细的Python脚本示例。该脚本能够针对不同的表创建并执行SQOOP任务，支持按create_time或date字段进行增量加载。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

一、先创建sqoop job，使用增量导入，python脚本如下：

#!/usr/bin/python

import os
import sys
import glob
import time
import datetime

##import all row fields --check-column create_time
def job_group0(name):
cmd = 'sqoop job --create %s -- import -m 1 --connect "jdbc:mysql://192.168.76.12:3306/smart_hardware?useSSL=false&user=phidoop&password=phidoop" --table %s --where "create_time < current_date()" --hive-import --hive-database phi_health --hive-table %s --incremental append --check-column create_time --last-value \'1900-01-01\' ' % (name,name,name)
run = os.system(cmd)

##import all row fields --check-column create_time
def job_group1(num1,num2,name):
for i in range(num1,num2):
cmd = 'sqoop job --create %s_%s -- import -m 1 --connect "jdbc:mysql://192.168.76.12:3306/smart_hardware?useSSL=false&user=phidoop&password=phidoop" --table %s_%s --where "create_time < current_date()" --hive-import --hive-database phi_health --hive-table %s --incremental append --check-column create_time --last-value \'1900-01-01\' ' % (name,i,name,i,name)
print cmd
run = os.system(cmd)

##import all row fields --check-column date
def job_group2(name):
cmd = 'sqoop job --create %s -- import -m 1 --connect "jdbc:mysql://192.168.76.12:3306/smart_hardware?useSSL=false&user=phidoop&password=phidoop" --table %s --where "date < current_date()" --hive-import --hive-database phi_health --hive-table %s --incremental append --check-column date --last-value \'1900-01-01\' ' % (name,name,name)
run = os.system(cmd)

##import all row fields --check-column date
def job_group3(num1,num2,name):
for i in range(num1,num2):
cmd = 'sqoop job --create %s_%s -- import -m 1 --connect "jdbc:mysql://192.168.76.12:3306/smart_hardware?useSSL=false&user=phidoop&password=phidoop" --table %s_%s --where "date < current_date()" --hive-import --hive-database phi_health --hive-table %s --incremental append --check-column date --last-value \'1900-01-01\' ' % (name,i,name,i,name)
print cmd
run = os.system(cmd)

if __name__=="__main__":

job_group0("balance_mac_manger_info")
job_group0("balance_measure_info")
job_group1(0,5,"balance_mac_measure_info")
job_group1(0,20,"blood_pressure_measure_info")
job_group1(0,50,"balance_measure_info")
job_group2("user_body_info")
job_group3(0,10,"user_body_info")

二、执行sqoop job的脚本job_exec.py如下：

#!/usr/bin/python

import os
import sys
import glob
import time
import datetime

def job_exec_group0(name):
cmd = 'sqoop job --exec %s' % (name)

##import all row fields
def job_exec_group1(num1,num2,name):
for i in range(num1,num2):
cmd = 'sqoop job --exec %s_%s' % (name,i)
run = os.system(cmd)

if __name__=="__main__":
job_exec_group0("balance_mac_manger_info")
job_exec_group0("balance_measure_info")
job_exec_group0("user_body_info")
job_exec_group1(0,5,"balance_mac_measure_info")
job_exec_group1(0,20,"blood_pressure_measure_info")
job_exec_group1(0,10,"user_body_info")
job_exec_group1(0,50,"balance_measure_info")