datax数据传输实践

gwd777

已于 2023-05-05 15:15:26 修改

阅读量299

点赞数

分类专栏： python Java链接文章标签： sqlserver 数据库 sql

于 2023-03-07 14:39:29 首次发布

本文链接：https://blog.youkuaiyun.com/gwd777/article/details/129382035

版权

python 同时被 2 个专栏收录

5 篇文章

订阅专栏

Java链接

3 篇文章

订阅专栏

1，参考文档

https://zhuanlan.zhihu.com/p/515541286

https://github.com/alibaba/DataX/blob/master/introduction.md

https://github.com/alibaba/DataX

2，sqlserver_2_clickhouse datax脚本

（*）注意这里：sqlserverreader,clickhousewriter 是固定写法

{
  "job": {
    "setting": {
      "speed": {
        "channel":1
      }
    },
    "content": [
      {
        "reader": {
          "name": "sqlserverreader",
          "parameter": {
            "username": "root",
            "password": "123123",
            "connection": [
              {
                "querySql": [
                  "select id,user_name,money from xiaoming.dbo.zy_user;"
                ],
                "jdbcUrl": [
                  "jdbc:sqlserver://xxx.rds.aliyuncs.com:3433"
                ]
              }
            ]
          }
        },
        "writer": {
          "name": "clickhousewriter",
          "parameter": {
            "writeMode": "insert",
            "username": "default",
            "password": "123123",
            "column": [
              "id",
              "user_name",
              "money"
            ],
            "session": [
              "set session sql_mode='ANSI'"
            ],
            "preSql": [
              "truncate table bigdata.ods_zy_user;"
            ],
            "connection": [
              {
                "jdbcUrl":"jdbc:clickhouse://localhost:8123/bigdata?useUnicode=true&characterEncoding=utf-8",
                "table": [
                  "ods_zy_user"
                ]
              }
            ]
          }
        }
      }
    ]
  }
}

{
"job": {
    "setting": {
      "speed": {
        "channel": 1
      }
    },
    "content": [
      {
        "reader": {
          "name": "sqlserverreader",
          "parameter": {
            "username": "your_db_name",
            "password": "your_pwd",
            "column": [
              "[FCUSTID]",
              "[FNAME]",
              "[FDocumentStatus]",
              "[FForbidStatus]",
              "[FCreateOrgId]",
              "[FUseOrgId]",
              "[FCreateDate]",
              "[FModifyDate]",
              "[F_QFES_Text8]",
              "[F_QFES_Text10]",
              "[FDescription]",
              "[C1FDATAVALUE]",
              "[C2FDATAVALUE]",
              "[C3FDATAVALUE]",
              "[FNUMBER]"
            ],
            "splitPk": "sale_id",
            "connection": [
              {
                "table": [
                  "your_tabname"
                ],
                "jdbcUrl": [
                  "jdbc:sqlserver://localhost:3433;DatabaseName=xxdbname"
                ]
              }
            ]
          }
        },
        "writer": {
          "name": "clickhousewriter",
          "parameter": {
            "username": "root",
            "password": "123123",
            "column": [
              "fcustid",
              "fname",
              "fdocumentstatus",
              "forbidstatus",
              "fcreate_orgid",
              "fuser_orgid",
              "fcreate_date",
              "fmodify_date",
              "f_qfes_text8",
              "f_qfes_text10",
              "fdescription",
              "c1_fdata_value",
              "c2_fdata_value",
              "c3_fdata_value",
              "fnumber"
            ],
            "connection": [
              {
                "table": [
                  "your_table_name"
                ],
                "jdbcUrl": "jdbc:clickhouse://localhost:8123/table_name"
              }
            ]
          }
        }
      }
    ]
  }
}

3，datax json格式脚本运行

# Linux运行指令
cd /opt/datax/bin
python datax.py -r streamreader -w streamwriter
python datax/bin/datax.py ./sqlserver_2_ch.json