连接kafka通过krb

Spingboot通过Krb、Jaas，Acl认证连接kafka服务器进行操作

1.kafka测试连接

需要krb5.conf文件，kafka_client_jaas.conf，kafka.service.keytab

这边要注意jaasconfig配置文件中的keytab路径是指当前客户端keytab的路径，并非服务器路径，principal为当前keytab文件认证的用户

   @Override
    public TestConnectionDTO testConnection(ConnectionParams params) {
        TestConnectionDTO testConnectionDTO = new TestConnectionDTO();
        //krb认证，设置属性
        UserGroupInformation.reset();
        try {
            synchronized (KdcManagement.lock) {
                KdcManagement.certification(params.getKrb5Conf(),
                        params.getPrincipal(), params.getKeytab(), params.getJaasConfAddress());
            }
        } catch (Exception e) {
            log.error("认证出错");
        }
        Properties properties = new Properties();
        // 配置Kafka服务的访问地址及端口号
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, params.getBootstrapServers());
        properties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
        properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        properties.put("sasl.mechanism", "GSSAPI");
        properties.put("sasl.kerberos.service.name", "kafka");
        // jaas文件需要需要手动编辑内部的keytab地址和principal，keytab地址为容器内地址
        properties.put("sasl.jaas.config", "com.sun.security.auth.module.Krb5LoginModule required " +
         "useKeyTab=true " +
         "storeKey=true " +
         "keyTab=\"" + params.getKeytab() + "\" " +
         "principal=\"" + params.getPrincipal() + "\";");
        AdminClient adminClient = null;
        try {
            adminClient = AdminClient.create(properties);
            ListTopicsOptions options = new ListTopicsOptions();
            // 是否列出内部使用的Topic
            options.listInternal(true);
            ListTopicsResult result = adminClient.listTopics(options);
            testConnectionDTO.setSuccessed(true);
            testConnectionDTO.setMsg("测试连接成功");
        } catch (Exception e) {
            e.printStackTrace();
            log.error("创建失败");
        } finally {
            if (adminClient != null) {
                adminClient.close();
            }
        }
        return testConnectionDTO;
    }

将krb5文件设置到环境变量

public class KdcManagement {
    public static final Object lock = new Object();
    public static void certification(String krb5, String principal, String keytab, String jaasConf) {
        synchronized (lock) {
            System.setProperty("java.security.krb5.conf", krb5);
            try {
                // necessary to resolve Can't get Kerberos realm
                UserGroupInformation.reset();
                Class<?> configClass = Class.forName("sun.security.krb5.Config");
                Field singletonField = configClass.getDeclaredField("singleton");
                singletonField.setAccessible(true);
                singletonField.set(null, null);
                //UserGroupInformation.getCurrentUser 可以查看当前认证情况，是否开启krb认证
                UserGroupInformation.loginUserFromKeytab(principal, keytab);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }

        }
    }
}

2.获取主题前几条数据

    @Override
    public List<TableStructureBo> collectTableData(ConnectionParams params, List<TableStructureBo> tableList) {
        String jaasConfig = getJaasConfig(params.getJaasConfAddress());
        List<String> tableName = tableList.stream().map(TableStructureBo::getTableName).collect(Collectors.toList());
        HashMap<String, List<String>> topicMap = new HashMap<>();
        for (String name : tableName) {
            topicMap.put(name, null);
        }
        UserGroupInformation.reset();
        try {
            synchronized (KdcManagement.lock) {
                KdcManagement.certification(params.getKrb5Conf(),
                        params.getPrincipal(), params.getKeytab(), params.getJaasConfAddress());
            }
        } catch (Exception e) {
            log.error("认证出错");
        }
        Properties properties = new Properties();
        // 配置Kafka服务的访问地址及端口号
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, params.getBootstrapServers());
        properties.put(AdminClientConfig.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        properties.put("fetch.min.bytes", "1024"); // 1KB   拉取最小字节
        properties.put("max.poll.records", "20");  // 一次最大拉取批次
        properties.put("auto.offset.reset", "earliest"); // 从最早的偏移量开始消费
        properties.put(ConsumerConfig.GROUP_ID_CONFIG, "group-" + UUID.randomUUID().toString());  //随机消费者组
        properties.put("sasl.mechanism", "GSSAPI");
        properties.put("sasl.jaas.config", jaasConfig); //jassConfig文件
        // 创建Kafka消费者
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(properties);
        for (String name : tableName) {
            boolean t = false;
            consumer.subscribe(Collections.singletonList(name));  //订阅主题
            // // 触发分区分配并获取分区分配信息
            consumer.poll(Duration.ofMillis(0));
            // // 重置分区偏移量到最开始
            consumer.seekToBeginning(consumer.assignment());
            // consumer.poll拉取可能因为某些情况导致主题有数据但是拉取不到，所以循环3次进行拉取
            for (int i = 0; i < 3; i++) {   
                //循环3次进行拉取，间隔1000毫秒拉取一次
                ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
                for (ConsumerRecord<String, String> record : records) {
                    t = true;
                    String topicName = record.topic();
                    log.info("record-{}", record.value());
                    List<String> topicValue = topicMap.get(topicName); 
                    if (topicValue != null && !topicValue.isEmpty()) {
                        topicValue.add(record.value());
                        topicMap.put(topicName, topicValue);
                    } else {
                        ArrayList<String> newTopicValue = new ArrayList<String>();
                        newTopicValue.add(record.value());
                        topicMap.put(topicName, newTopicValue);
                    }
                }
                if (t) {
                    break;
                }
            }
        }
        consumer.close(); //关闭消费者
        for (TableStructureBo bo : tableList) {
            String topicValueString = JSON.toJSONString(topicValue, JSONWriter.Feature.WriteNullListAsEmpty);
            bo.setTableData(topicValueString);
        }
        return tableList;
    }