DataX使用

最新推荐文章于 2025-02-21 17:03:42 发布

原创最新推荐文章于 2025-02-21 17:03:42 发布 · 2.2k 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#datax

DataX 专栏收录该内容

3 篇文章

订阅专栏

官方地址:https://github.com/alibaba/DataX
安装部署:https://blog.youkuaiyun.com/paicmis/article/details/53563824

datax split目前仅支持单一主键，且主键类型是int或者varchar类型
splitPk有值时，channel才会生效，否则都是单通道执行

{
    "job": {
        "content": [
            {
                "reader": {
                    "name": "mysqlreader", 
                    "parameter": {
                        "column": ["id","created_time","updated_time","tenant_id","payload"], 
                        "splitPk": "id",
                        "connection": [
                            {
                                "jdbcUrl": ["jdbc:mysql://ip:3306/apollo_prod?autoReconnect=true"], 
                                "table": ["risk_data"]
                            }
                        ], 
                        "password": "******", 
                        "username": "****", 
                        "where": ""
                    }
                }, 
                "writer": {
                    "name": "postgresqlwriter", 
                    "parameter": {
                        "column": ["id","created_time","updated_time","tenant_id","payload"], 
                        "connection": [
                            {
                                "jdbcUrl": "jdbc:postgresql://ip:3433/appExtend", 
                                "table": ["risk_data"]
                            }
                        ], 
                        "password": "*******", 
                        "postSql": [], 
                        "preSql": ["delete from risk_data"], 
                        "username": "******"
                    }
                }
            }
        ], 
        "setting": {
            "speed": {
                "channel": 10
            }
        }
    }
}

//自定义sql
{
    "job": {
        "content": [
            {
                "reader": {
                    "name": "mysqlreader", 
                    "parameter": {
                        "connection": [
                            {
                                "querySql": ["select id,created_time,updated_time,tenant_id,payload from risk_data limit 1000;"],
                                "jdbcUrl": ["jdbc:mysql://ip:3306/apollo_prod"]
                            }
                        ], 
                        "splitPk": "id",
                        "password": "*******", 
                        "username": "******"
                    }
                }, 
                "writer": {
                    "name": "postgresqlwriter", 
                    "parameter": {
                        "column": ["id","created_time","updated_time","tenant_id","payload"], 
                        "connection": [
                            {
                                "jdbcUrl": "jdbc:postgresql://ip:3433/appExtend", 
                                "table": ["risk_data"]
                            }
                        ], 
                        "password": "********", 
                        "postSql": [], 
                        "preSql": ["delete from risk_data"], 
                        "username": "********"
                    }
                }
            }
        ], 
        "setting": {
            "speed": {
                "channel": 10
            }
        }
    }
}