DataX读取Oracle写入Hive之orc存储格式

原创已于 2022-04-22 07:13:32 修改 · 2.7k 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#datax

于 2022-04-21 14:07:47 首次发布

hive 同时被 3 个专栏收录

28 篇文章

订阅专栏

datax

19 篇文章

订阅专栏

Oracle

6 篇文章

订阅专栏

1、脚本

{
	"core": {
		"transport": {
			"channel": {
				"speed": {
					"byte": 10485760,
					"record": 100000
				}
			}
		}
	},
	"job": {
		"content": [{
			"reader": {
				"name": "oraclereader",
				"parameter": {
					"column": [
						"BJBH",
						"SXBM",
						"SXMC",
						"BJLX",
						"SQR",
						"SQZJLX",
						"SQZJHM",
						"LXR",
						"LXRZJLX",
						"LXRZJHM",
						"LXDH",
						"LXYZBM",
						"LXTXDZ",
						"FDDBR",
						"SQLY",
						"SQSJ",
						"XMBH",
						"BJZT",
						"ID",
						"OID",
						"QHDM",
						"JGDM",
						"CJSJ",
						"GXSJ",
						"FLAG",
						"EXCHANGE",
						"DATA_UP_UUID",
						"DATA_UP_TIME",
						"DATA_UP_STATUS",
						"PBSNUM",
						"SCKQZSJ"
					],
					"connection": [{
						"jdbcUrl": ["jdbc:oracle:thin:@//ip:1521/orcl"],
						"table": ["BUSINESS_INDEX"]
					}],
					"splitPk": "",
					"password": "root",
					"username": "123456"
				}
			},
			"writer": {
				"name": "hdfswriter",
				"parameter": {
					"column": [{
							"name": "bjbh",
							"type": "STRING"
						},
						{
							"name": "sxbm",
							"type": "STRING"
						},
						{
							"name": "sxmc",
							"type": "STRING"
						},
						{
							"name": "bjlx",
							"type": "STRING"
						},
						{
							"name": "sqr",
							"type": "STRING"
						},
						{
							"name": "sqzjlx",
							"type": "STRING"
						},
						{
							"name": "sqzjhm",
							"type": "STRING"
						},
						{
							"name": "lxr",
							"type": "STRING"
						},
						{
							"name": "lxrzjlx",
							"type": "STRING"
						},
						{
							"name": "lxrzjhm",
							"type": "STRING"
						},
						{
							"name": "lxdh",
							"type": "STRING"
						},
						{
							"name": "lxyzbm",
							"type": "STRING"
						},
						{
							"name": "lxtxdz",
							"type": "STRING"
						},
						{
							"name": "fddbr",
							"type": "STRING"
						},
						{
							"name": "sqly",
							"type": "STRING"
						},
						{
							"name": "sqsj",
							"type": "DATE"
						},
						{
							"name": "xmbh",
							"type": "STRING"
						},
						{
							"name": "bjzt",
							"type": "Double"
						},
						{
							"name": "id",
							"type": "STRING"
						},
						{
							"name": "oid",
							"type": "STRING"
						},
						{
							"name": "qhdm",
							"type": "STRING"
						},
						{
							"name": "jgdm",
							"type": "STRING"
						},
						{
							"name": "cjsj",
							"type": "DATE"
						},
						{
							"name": "gxsj",
							"type": "DATE"
						},
						{
							"name": "flag",
							"type": "STRING"
						},
						{
							"name": "exchange",
							"type": "STRING"
						},
						{
							"name": "data_up_uuid",
							"type": "STRING"
						},
						{
							"name": "data_up_time",
							"type": "DATE"
						},
						{
							"name": "data_up_status",
							"type": "STRING"
						},
						{
							"name": "pbsnum",
							"type": "STRING"
						},
						{
							"name": "sckqzsj",
							"type": "DATE"
						}
					],
					"hadoopConfig": {
						"fs.hdfs.impl.disable.cache": "true"
					},
					"compress": "NONE",
					"defaultFS": "hdfs://node01:8020",
					"fieldDelimiter": "|",
					"fileName": "business_index",
					"fileType": "orc",
					"path": "/user/hive/warehouse/mask_data.db/business_index",
					"writeMode": "append"
				}
			}
		}],
		"setting": {
			"errorLimit": {
				"record": 0,
				"percentage": 0
			},
			"speed": {
				"byte": 20971520,
				"channel": 3,
				"record": 200000
			}
		}
	}
}