chunjun-1.12 在两个CDH（kerberos认证）之间同步数配置

本文链接：https://blog.youkuaiyun.com/beExcellentOne/article/details/53185870

本文介绍使用ChunJun同步两个CDH数据文件时的Kerberos认证配置细节。通过具体配置示例，展示了如何设置HDFS路径、用户名、密钥文件等关键参数，确保跨集群数据同步的安全性和稳定性。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

chunjun是以前flinkx项目，现在改名chunjun，具体见 chunjun官网

最近在项目中遇到同步两个CDH数据文件的需求，调研了一圈，发现datax不支持两个cdh同时进行kerberos认证，因此切换到chunjun，chunjun在kerberos认证的配置这块，官网也没有详细的介绍案例，在查看源代码后，确定了配置项及各配置项的值，供大家参考

{
  "job": {
    "content": [
      {
        "reader" : {
          "parameter" : {
            "path" : "hdfs://cdh-test-namenode05:8020/user/hive/warehouse/testuser_db.db/testuser_user/t4.txt",
            "hadoopConfig" : {
              "properties.hadoop.user.name": "testuser",
              "properties.dfs.ha.namenodes.nameservice1": "namenode27,namenode29",
              "properties.fs.defaultFS": "hdfs://nameservice1",
              "properties.dfs.namenode.rpc-address.nameservice1.namenode27": "cdh-test-namenode04:8020",
              "properties.dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
              "properties.dfs.namenode.rpc-address.nameservice1.namenode29": "cdh-test-namenode05:8020",
              "properties.dfs.nameservices": "nameservice1",
              "properties.fs.hdfs.impl.disable.cache": "true",
              "properties.dfs.client.use.datanode.hostname":"true",
              "dfs.client.use.datanode.hostname":"true",
              "properties.fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
              "hadoop.security.authentication": "Kerberos",
              "dfs.namenode.kerberos.principal": "hdfs/_HOST@CDHDEV.COM",
              "dfs.datanode.kerberos.principal": "hdfs/_HOST@CDHDEV.COM",
              "yarn.resourcemanager.principal": "rm/_HOST@CDHDEV.COM",
              "dfs.namenode.kerberos.internal.spnego.principal": "HTTP/_HOST@CDHDEV.COM",
              "hadoop.security.authorization": "true",
              "dfs.namenode.keytab.file": "/app/data/CDH6.3.2/kerberos/testuser.keytab",
              "java.security.krb5.conf": "/app/data/CDH6.3.2/krb5/krb5.conf",
              "useLocalFile": "true",
              "principalFile": "/app/data/CDH6.3.2/kerberos/testuser.keytab",
              "principal": "testuser/cdh-test-namenode05"
            },

            "column": [
              {
                "name": "id",
                "type": "int"
              },
              {
                "name": "name",
                "type": "string"
              }
            ],
            "fieldDelimiter" : ",",
            "encoding" : "utf-8",
            "defaultFS": "hdfs://cdh-test-namenode05:8020",
            "fileType" : "text"
          },
          "name" : "hdfsreader"
        },
        "writer": {
          "name": "hdfswriter",
          "parameter": {
            "path": "/user/hive/warehouse/test_system_user.db/testuser",
            "defaultFS": "hdfs://test01:8020",
            "column": [
              {
                "name": "id",
                "type": "int"
              },
              {
                "name": "name",
                "type": "string"
              }
            ],
            "fileType": "text",
            "fieldDelimiter": ",",
            "compress":"gzip",
            "encoding": "utf-8",
            "fileName": "20220705",
            "writeMode": "overwrite",
            "hadoopConfig": {
              "properties.hadoop.user.name": "test_system_user",
              "properties.dfs.ha.namenodes.nameservice1": "namenode37,namenode51",
              "properties.fs.defaultFS": "hdfs://nameservice1",
              "properties.dfs.namenode.rpc-address.nameservice1.namenode51": "test02:8020",
              "properties.dfs.client.failover.proxy.provider.nameservice1": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
              "properties.dfs.namenode.rpc-address.nameservice1.namenode37": "test01:8020",
              "properties.dfs.nameservices": "nameservice1",
              "properties.dfs.client.use.datanode.hostname":"true",
              "dfs.client.use.datanode.hostname":"true",
              "properties.fs.hdfs.impl.disable.cache": "true",
              "properties.fs.hdfs.impl": "org.apache.hadoop.hdfs.DistributedFileSystem",
              "hadoop.security.authentication": "Kerberos",
              "dfs.namenode.kerberos.principal": "hdfs/_HOST@SINOSIG.COM",
              "dfs.datanode.kerberos.principal": "hdfs/_HOST@SINOSIG.COM",
              "yarn.resourcemanager.principal": "rm/_HOST@SINOSIG.COM",
              "dfs.namenode.kerberos.internal.spnego.principal": "HTTP/_HOST@SINOSIG.COM",
              "hadoop.security.authorization": "true",
              "dfs.namenode.keytab.file": "/app/data/cloudera/CDH-6.3.2/kerberos/test_system_user.keytab",
              "java.security.krb5.conf": "/app/data/cloudera/CDH-6.3.2/krb5/krb5.conf",
              "useLocalFile": "true",
              "principalFile": "/app/data/cloudera/CDH-6.3.2/kerberos/test_system_user.keytab",
              "principal": "test_system_user/test02@SINOSIG.COM"
            }
          }
        }
      }
    ],
    "setting": {
      "speed": {
        "channel": 4,
        "bytes": 0
      }
    }
  }
}