Spark Streaming 03 分布式消息队列 kafka

1 概述

Kafka® is used for building real-time data pipelines and streaming apps. It is horizontally scalable, fault-tolerant, wicked fast, and runs in production in thousands of companies.

● Building real-time streaming data pipelines that reliably get data between systems or applications
● Building real-time streaming applications that transform or react to the streams of data

1）kafka架构

• producer：生产者
• consumer：消费者
• topics：The Kafka cluster stores streams of records in categories called topics.
• broker：存放信息的地方

2 kafka部署及使用

2.1 单节点单Broker部署及使用

1）下载安装zookeeper，并配置环境变量
2）修改配置文件 zookeeper/conf/zoo.cfg (zoo_sample.cfg --> zoo.cfg)

dataDir=xxxx

3）下载安装kafka，配置环境变量
4）修改配置文件 kafka/config/server.properties

broker.id=0
listeners=PLAINTEXT://:9092
host.name=localhost
log.dirs=/Users/Mac/app/kafka_2.11-0.9.0.0/tmp/kafka-logs
zookeeper.connect=localhost:2181

5）启动 zookeeper

zkServer.sh  start

6）启动 kafka

kafka-server-start.sh $KAFKA_HOME/config/server.properties

7）创建 topic

kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test_topic

8）启动一个producer发送信息

kafka-console-producer.sh --broker-list localhost:9092 --topic test_topic

9）启动一个consumer接收信息

kafka-console-consumer.sh --zookeeper localhost:2181 --topic test_topic 

==> 添加 --from-beginning：表示从头开始消费，不加则只会消费新消息

10）查看创建的topic的信息

kafka-topics.sh --describe --zookeeper localhost:2181 

==> 添加 --topic test_topic，查看指定topic的信息

11）查看所有已经创建的topic

kafka-topics.sh --list --zookeeper localhost:2181

2.2 单节点多Broker部署及使用

1）修改配置文件 复制 server.properties

server-1.properties

broker.id=1
listeners=PLAINTEXT://:9093
log.dirs=/Users/Mac/app/kafka_2.11-0.9.0.0/tmp/kafka-logs-1

server-2.properties

broker.id=2
listeners=PLAINTEXT://:9094
log.dirs=/Users/Mac/app/kafka_2.11-0.9.0.0/tmp/kafka-logs-2

server-3.properties

broker.id=3
listeners=PLAINTEXT://:9095
log.dirs=/Users/Mac/app/kafka_2.11-0.9.0.0/tmp/kafka-logs-3

2）启动 zookeeper
3）启动 kafka

kafka-server-start.sh -daemon $KAFKA_HOME/config/server-1.properties &
kafka-server-start.sh -daemon $KAFKA_HOME/config/server-2.properties &
kafka-server-start.sh -daemon $KAFKA_HOME/config/server-3.properties &

4）创建 topic

kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic my-replicated-topic

5）启动一个producer发送信息

kafka-console-producer.sh --broker-list localhost:9093,localhost:9094,localhost:9095 --topic my-replicated-topic

6）启动一个consumer接收信息

kafka-console-consumer.sh --zookeeper localhost:2181 --topic my-replicated-topic [--from-beginning]

2.3 多节点多Broker部署及使用

3 Kafka Java API 编程

代码地址

1）修改pom.xml，添加依赖

<properties>
        <scala.version>2.11.8</scala.version>
        <kafka.version>0.9.0.0</kafka.version>
</properties>
<!-- kafka 依赖-->
<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka_2.11</artifactId>
    <version>${kafka.version}</version>
</dependency>

2）KafkaProperties.java

package com.lihaogn.spark.kafka;

/**
 * kafka 常用配置
 */
public class KafkaProperties {

    public static final String ZK = "localhost:2181";
    public static final String TOPIC = "test_topic";
    public static final String BROKER_LIST = "localhost:9092";
    public static final String GROUP_ID = "test_group1";
}

3）KafkaProducer.java

package com.lihaogn.spark.kafka;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

import java.util.Properties;

/**
 * kafka 生产者
 */
public class KafkaProducer extends Thread {

    private String topic;

    private Producer<Integer, String> producer;

    public KafkaProducer(String topic) {
        this.topic = topic;

        Properties properties = new Properties();

        properties.put("metadata.broker.list", KafkaProperties.BROKER_LIST);
        properties.put("serializer.class", "kafka.serializer.StringEncoder");
        properties.put("request.required.acks", "1");

        producer = new Producer<Integer, String>(new ProducerConfig(properties));
    }

    @Override
    public void run() {
        int messageId = 1;

        while (true) {
            String message = "message_" + messageId;
            producer.send(new KeyedMessage<Integer, String>(topic, message));
            System.out.println("sent: " + message);

            messageId++;

            try {
                Thread.sleep(2000);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

4）KafkaConsumer.java

package com.lihaogn.spark.kafka;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

/**
 * kafka消费者
 */
public class KafkaConsumer extends Thread {
    private String topic;

    public KafkaConsumer(String topic) {
        this.topic = topic;
    }

    private ConsumerConnector createConnector() {
        Properties properties = new Properties();
        properties.put("zookeeper.connect", KafkaProperties.ZK);
        properties.put("group.id", KafkaProperties.GROUP_ID);
        return Consumer.createJavaConsumerConnector(new ConsumerConfig(properties));
    }

    @Override
    public void run() {
        ConsumerConnector consumer = createConnector();

        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();

        topicCountMap.put(topic, 1);

        Map<String, List<KafkaStream<byte[], byte[]>>> messageStream = consumer.createMessageStreams(topicCountMap);

        // 获取每次收到的数据
        KafkaStream<byte[], byte[]> stream = messageStream.get(topic).get(0);

        ConsumerIterator<byte[], byte[]> iterator = stream.iterator();

        while (iterator.hasNext()) {
            String message = new String(iterator.next().message());
            System.out.println("rec:_" + message);
        }
    }
}

5）KafkaClientApp.java

package com.lihaogn.spark.kafka;

/**
 * kafka 测试
 */
public class KafkaClientApp {
    public static void main(String[] args) {

        new KafkaProducer(KafkaProperties.TOPIC).start();

        new KafkaConsumer(KafkaProperties.TOPIC).start();


    }
}

6）结果