Kafka消息队列的搭建和基本使用

最新推荐文章于 2025-06-04 19:09:28 发布

原创最新推荐文章于 2025-06-04 19:09:28 发布 · 1.4k 阅读

1 ·

CC 4.0 BY-SA版权

Java开发语言同时被 2 个专栏收录

40 篇文章

订阅专栏

Kafka消息队列

1 篇文章

订阅专栏

本文详细介绍如何在Linux环境下安装JDK、Zookeeper和Kafka，包括环境变量配置、服务启动及基本命令使用。同时，提供了Java程序调用Kafka的示例代码，包括生产者和消费者程序。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

安装Jdk

sudo tar -vxzf jdk-8u152-linux-x64.tar.gz
sudo mv jdk1.8.0_152 /opt/java

配置Java环境变量：

vim /etc/profile
#在profile文件的最后面添加
export JAVA_HOME=/opt/jdk1.8.0_152
export PATH=$JAVA_HOME/bin:$PATH
#添加成功后通过java -version来查看jdk是否安装成功
java -version
openjdk version "1.8.0_131"
OpenJDK Runtime Environment (build 1.8.0_131-b12)
OpenJDK 64-Bit Server VM (build 25.131-b12, mixed mode)

安装Zookeeper

下载zookeeper安装包，并将文件移动到/opt/zookeeper文件夹下面：

# 解压zookeeper
sudo tar -zxvf zookeeper-3.3.6.tar.gz
# 将zookeeper移动到/opt/zookeeper目录下
sudo mv zookeeper-3.3.6 /opt/zookeeper

编辑zookeeper的配置

# 复制一份zoo_sample.cfg文件并改名为zoo.cfg
sudo cp /opt/zookeeper/zoo_sample.cfg zoo.cfg
# 编辑zoo.cfg 文件
sudo vim /opt/zookeeper/zoo.cfg
#主要修改dataDir和server.1=127.0.0.1:2888:3888这2处
# the directory where the snapshot is stored.
dataDir=/opt/zookeeper/data
# the port at which the clients will connect
clientPort=2181
server.1=127.0.0.1:2888:3888

启动zookeeper

./zkServer.sh start ../conf/zoo.cfg

安装Kafka

下载kafka

sudo wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/1.0.0/kafka_2.12-1.0.0.tgz

解压并将文件移动到/opt/kafka目录下

# 解压
sudo tar -zxvf kafka_2.12-1.0.0.tgz
# 移动
sudo mv kafka_2.12-1 /opt/kafka

编辑kafka的配置

#创建日志存放目录
cd /opt/kafka
mkdir -p log/kafka
#修改配置文件/opt/kafka/config/server.properties
sudo vim /opt/kafka/config/server.properties
#主要修改下面几项内容如下：
broker.id=181
delete.topic.enable=true
listeners = PLAINTEXT://127.0.0.1:9092
log.dirs=/opt/kafka/log/kafka
zookeeper.connect=127.0.0.1:2181

配置kafka的环境变量

sudo vim /etc/profile
#添加如下内容：
KAFKA_HOME=/opt/kafka
PATH=$PATH:$KAFKA_HOME/bin
export PATH KAFKA_HOME

启动kafka

kafka-server-start.sh /opt/kafka/config/server.properties

基本命令的使用

创建一个topic

kafka-topics.sh --create --zookeeper 127.0.0.1:2181 --replication-factor 1 --partitions 1 --topic test

producer向topic发送消息

kafka-console-producer.sh --broker-list 127.0.0.1:9092 --topic test

消费者消费消息

kafka-console-consumer.sh --zookeeper 127.0.0.1:2181 --topic test --from-beginning

Java程序调用

创建Maven项目，并添加依赖

<dependencies>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>0.10.2.0</version>
        </dependency>
</dependencies>

生产者程序

package test;


import org.apache.kafka.clients.producer.*;

import java.util.Properties;

/**
 * @author jiaqing.xu@hand-china.com
 * @version 1.0
 * @name
 * @description
 * @date 2018/9/29
 */
public class ProducerTest {
    private void execMsgSend() throws Exception {
        Properties props = new Properties();
        props.put("bootstrap.servers", "192.168.12.146:9092");
        props.put("acks", "1");
        props.put("retries", 0);
        props.put("batch.size", 16384);
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        //生产者
        Producer<String, String> producer = new KafkaProducer<String, String>(props);
        // 主题
        String topic = "test";
        for (int i = 1; i <= 5; i++) {
            //String value = " this is another message_" + i;
            String value = "{\n" +
                    "\t\"sourceSystem\": \"HPAY\",\n" +
                    "\t\"interfaceName\": \"001\",\n" +
                    "\t\"lineCount\": \"2\",\n" +
                    "\t\"batchNum\": \"20180905171252\",\n" +
                    "\t\"data\": [{\n" +
                    "\t\t\t\"requestNumber\": \"20180905171252\",\n" +
                    "\t\t\t\"orderNumber\": \"7565666\",\n" +
                    "\t\t\t\"paymentNumber\": \"645453353\",\n" +
                    "\t\t\t\"relevanceNumber\": \"54323456\",\n" +
                    "\t\t\t\"paymentTypeCode\": \"订单\",\n" +
                    "\t\t\t\"payDate\": \"2018-08-29 15:12:00.0\",\n" +
                    "\t\t\t\"amount\": \"300.00\",\n" +
                    "\t\t\t\"currency\": \"USD\",\n" +
                    "\t\t\t\"payMethod\": \"WECHAT\",\n" +
                    "\t\t\t\"payChannel\": \"万通\",\n" +
                    "\t\t\t\"sourceCode\": \"PORTAL\",\n" +
                    "\t\t\t\"orderFlowingNumber\": \"545345435\",\n" +
                    "\t\t\t\"paymentAccount\": \"2166785676\"\n" +
                    "\t\t},\n" +
                    "\t\t{\n" +
                    "\t\t\t\"requestNumber\": \"20180905171252\",\n" +
                    "\t\t\t\"orderNumber\": \"7565666\",\n" +
                    "\t\t\t\"paymentNumber\": \"645453353\",\n" +
                    "\t\t\t\"relevanceNumber\": \"54323456\",\n" +
                    "\t\t\t\"paymentTypeCode\": \"订单\",\n" +
                    "\t\t\t\"payDate\": \"2018-08-29 15:12:00.0\",\n" +
                    "\t\t\t\"amount\": \"300.00\",\n" +
                    "\t\t\t\"currency\": \"USD\",\n" +
                    "\t\t\t\"payMethod\": \"WECHAT\",\n" +
                    "\t\t\t\"payChannel\": \"万通\",\n" +
                    "\t\t\t\"sourceCode\": \"PORTAL\",\n" +
                    "\t\t\t\"orderFlowingNumber\": \"545345435\",\n" +
                    "\t\t\t\"paymentAccount\": \"2166785676\"\n" +
                    "\t\t}\n" +
                    "\t]\n" +
                    "}";
            ProducerRecord<String, String> record = new ProducerRecord<>(topic, i + "", value);
            // 发送数据
            producer.send(record, new Callback() {
                // 回调函数
                @Override
                public void onCompletion(RecordMetadata metadata, Exception exception) {
                    System.out.println("message send to partition " + metadata.partition() + ", offset: " + metadata.offset());
                }
            });
            System.out.println(i + " send success");
            // 线程休息1s
            Thread.sleep(1000);
        }
        System.out.println("send message over.");
        producer.close();
    }

    public static void main(String[] args) throws Exception {
        ProducerTest test1 = new ProducerTest();
        test1.execMsgSend();
    }

}

消费者程序

package test;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;

import java.util.Arrays;
import java.util.Properties;

/**
 * @author jiaqing.xu@hand-china.com
 * @version 1.0
 * @name
 * @description
 * @date 2018/9/29
 */
public class Consumer {
    public static void main(String[] s){
        Properties props = new Properties();
        props.put("bootstrap.servers", "192.168.12.146:9092");
        props.put("group.id", "1");
        props.put("enable.auto.commit", "true");
        props.put("auto.commit.interval.ms", "1000");
        props.put("session.timeout.ms", "30000");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        //订阅的主题
        consumer.subscribe(Arrays.asList("test"));
        System.out.println("poll start...");
        while (true) {
            // 拉取记录
            ConsumerRecords<String, String> records = consumer.poll(100);
            for (ConsumerRecord<String, String> record : records) {
                System.out.printf("offset = %d, key = %s, value = %s", record.offset(), record.key(), record.value());
                System.out.println("\n");
            }
        }
    }
}