1 概述
Kafka® is used for building real-time data pipelines and streaming apps. It is horizontally scalable, fault-tolerant, wicked fast, and runs in production in thousands of companies.
● Building real-time streaming data pipelines that reliably get data between systems or applications
● Building real-time streaming applications that transform or react to the streams of data
1)kafka架构

- producer:生产者
- consumer:消费者
- topics:The Kafka cluster stores streams of records in categories called topics.
- broker:存放信息的地方
2 kafka部署及使用
2.1 单节点单Broker部署及使用
1)下载安装zookeeper,并配置环境变量
2)修改配置文件 zookeeper/conf/zoo.cfg (zoo_sample.cfg --> zoo.cfg)
dataDir=xxxx
3)下载安装kafka,配置环境变量
4)修改配置文件 kafka/config/server.properties
broker.id=0
listeners=PLAINTEXT://:9092
host.name=localhost
log.dirs=/Users/Mac/app/kafka_2.11-0.9.0.0/tmp/kafka-logs
zookeeper.connect=localhost:2181
5)启动 zookeeper
zkServer.sh start
6)启动 kafka
kafka-server-start.sh $KAFKA_HOME/config/server.properties
7)创建 topic
kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test_topic
8)启动一个producer发送信息
kafka-console-producer.sh --broker-list localhost:9092 --topic test_topic
9)启动一个consumer接收信息
kafka-console-consumer.sh --zookeeper localhost:2181 --topic test_topic
==> 添加 --from-beginning:表示从头开始消费,不加则只会消费新消息
10)查看创建的topic的信息
kafka-topics.sh --describe --zookeeper localhost:2181
==> 添加 --topic test_topic,查看指定topic的信息
11)查看所有已经创建的topic
kafka-topics.sh --list --zookeeper localhost:2181
2.2 单节点多Broker部署及使用
1)修改配置文件 复制 server.properties
server-1.properties
broker.id=1
listeners=PLAINTEXT://:9093
log.dirs=/Users/Mac/app/kafka_2.11-0.9.0.0/tmp/kafka-logs-1
server-2.properties
broker.id=2
listeners=PLAINTEXT://:9094
log.dirs=/Users/Mac/app/kafka_2.11-0.9.0.0/tmp/kafka-logs-2
server-3.properties
broker.id=3
listeners=PLAINTEXT://:9095
log.dirs=/Users/Mac/app/kafka_2.11-0.9.0.0/tmp/kafka-logs-3
2)启动 zookeeper
3)启动 kafka
kafka-server-start.sh -daemon $KAFKA_HOME/config/server-1.properties &
kafka-server-start.sh -daemon $KAFKA_HOME/config/server-2.properties &
kafka-server-start.sh -daemon $KAFKA_HOME/config/server-3.properties &
4)创建 topic
kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 3 --partitions 1 --topic my-replicated-topic
5)启动一个producer发送信息
kafka-console-producer.sh --broker-list localhost:9093,localhost:9094,localhost:9095 --topic my-replicated-topic
6)启动一个consumer接收信息
kafka-console-consumer.sh --zookeeper localhost:2181 --topic my-replicated-topic [--from-beginning]
2.3 多节点多Broker部署及使用
3 Kafka Java API 编程
1)修改pom.xml,添加依赖
<properties>
<scala.version>2.11.8</scala.version>
<kafka.version>0.9.0.0</kafka.version>
</properties>
<!-- kafka 依赖-->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.11</artifactId>
<version>${kafka.version}</version>
</dependency>
2)KafkaProperties.java
package com.lihaogn.spark.kafka;
/**
* kafka 常用配置
*/
public class KafkaProperties {
public static final String ZK = "localhost:2181";
public static final String TOPIC = "test_topic";
public static final String BROKER_LIST = "localhost:9092";
public static final String GROUP_ID = "test_group1";
}
3)KafkaProducer.java
package com.lihaogn.spark.kafka;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
import java.util.Properties;
/**
* kafka 生产者
*/
public class KafkaProducer extends Thread {
private String topic;
private Producer<Integer, String> producer;
public KafkaProducer(String topic) {
this.topic = topic;
Properties properties = new Properties();
properties.put("metadata.broker.list", KafkaProperties.BROKER_LIST);
properties.put("serializer.class", "kafka.serializer.StringEncoder");
properties.put("request.required.acks", "1");
producer = new Producer<Integer, String>(new ProducerConfig(properties));
}
@Override
public void run() {
int messageId = 1;
while (true) {
String message = "message_" + messageId;
producer.send(new KeyedMessage<Integer, String>(topic, message));
System.out.println("sent: " + message);
messageId++;
try {
Thread.sleep(2000);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
4)KafkaConsumer.java
package com.lihaogn.spark.kafka;
import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
/**
* kafka消费者
*/
public class KafkaConsumer extends Thread {
private String topic;
public KafkaConsumer(String topic) {
this.topic = topic;
}
private ConsumerConnector createConnector() {
Properties properties = new Properties();
properties.put("zookeeper.connect", KafkaProperties.ZK);
properties.put("group.id", KafkaProperties.GROUP_ID);
return Consumer.createJavaConsumerConnector(new ConsumerConfig(properties));
}
@Override
public void run() {
ConsumerConnector consumer = createConnector();
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
topicCountMap.put(topic, 1);
Map<String, List<KafkaStream<byte[], byte[]>>> messageStream = consumer.createMessageStreams(topicCountMap);
// 获取每次收到的数据
KafkaStream<byte[], byte[]> stream = messageStream.get(topic).get(0);
ConsumerIterator<byte[], byte[]> iterator = stream.iterator();
while (iterator.hasNext()) {
String message = new String(iterator.next().message());
System.out.println("rec:_" + message);
}
}
}
5)KafkaClientApp.java
package com.lihaogn.spark.kafka;
/**
* kafka 测试
*/
public class KafkaClientApp {
public static void main(String[] args) {
new KafkaProducer(KafkaProperties.TOPIC).start();
new KafkaConsumer(KafkaProperties.TOPIC).start();
}
}
6)结果