kafka的安装和使用

最新推荐文章于 2025-09-12 20:46:57 发布

转载最新推荐文章于 2025-09-12 20:46:57 发布 · 68 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：https://my.oschina.net/xiaozhou18/blog/806260

文章标签：

#大数据 #python #scala

2019独角兽企业重金招聘Python工程师标准>>>

一、安装

1、安装scala 解压 tar -zxvf scala-2.10.4.tgz

2、安装kafka 解压 tar -zxvf kafka_2.11-0.9.0.1.tgz

二、修改kafka/config下的server.properties配置文件如下内容

broker.id=1 不能重复

host.name=node11（机器的ip）

advertised.host.name=node11(机器的ip)

log.dirs=/usr/local/java/kafkalogs （kafka的日志路径）

zookeeper.connect=node22:2181,node33:2181,node44:2181 （zookeeper的集群地址）

三、把配置好的kafka copy到其他三台机器上要修改broker.id host.name advertised.host.name

四、启动zookeeper集群

zookeeper集群环境搭建参考 https://my.oschina.net/xiaozhou18/blog/787132

五、启动kafka集群

/bin/kafka-server-start.sh /usr/local/java/kafka/config/server.properties &

六、测试kafka集群

在node11创建topic bin/kafka-topics.sh --create --zookeeper node22:2181,node33:2181,node44:2181 --replication-factor 1 --partitions 1 --topic test

查看topic bin/kafka-topics.sh --list --zookeeper node22:2181,node33:2181,node44:2181

在node22上发送消息至kafka， bin/kafka-console-producer.sh --broker-list node11:9092，node33:9092 --sync --topic test

在node33查看消bin/kafka-console-consumer.sh --zookeeper node22:2181,node33:2181,node44:2181 --topic test-topic --from-beginning

七、用java 简单操作kafak

1、生产者

package com.xiaozhou.kafka;
import java.util.Properties;
import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;
public class Produce extends Thread{
   //定义一个主题
   private static String topic="kafka-topic";
   public Produce(String topic) {
       this.topic = topic;
   }
   @Override
   public void run() {
       Producer producer=createProducer();
       for(int i=0;i<100;i++){
           //发消息
           producer.send(new KeyedMessage(topic, "hello"+i));
       }
   }
   public Producer createProducer(){
       Properties prop=new Properties();
       //往哪几个broker上发消息
       prop.put("metadata.broker.list", "node22:9092,node33:9092");
       prop.put("serializer.class", "kafka.serializer.StringEncoder");
       ProducerConfig producerConfig = new ProducerConfig(prop);
       Producer producer = new Producer(producerConfig);
       System.out.println(producer);
       return producer;
   }
   public static void main(String[] args) {
       // TODO Auto-generated method stub
       Produce p=new Produce(topic);
       p.start();
   }

}

2、消费者

package com.xiaozhou.kafka;

import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.ConsumerIterator;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.MessageAndMetadata;

public class Consum {
   private static String topic="kafka-topic";
   public static ConsumerConnector createConsumer(){
       Properties props = new Properties();
       props.put("zookeeper.connect", "node22:2181,node33:2181,node44:2181");
       props.put("group.id", "kafkademo");
       props.put("zookeeper.session.timeout.ms", "40000");
       props.put("zookeeper.sync.time.ms", "200");
       props.put("auto.commit.interval.ms", "1000");
       return Consumer.createJavaConsumerConnector( new ConsumerConfig(props));

   }
   public static void main(String[] args) {
       ConsumerConnector consumer = createConsumer();
       Map<String,Integer> map=new HashMap<String, Integer>();
       map.put(topic, 1);
       Map<String, List<KafkaStream<byte[], byte[]>>> messageStreams = consumer.createMessageStreams(map);
       KafkaStream<byte[], byte[]> stream = messageStreams.get(topic).get(0);
       ConsumerIterator<byte[],byte[]> iterator = stream.iterator();
       while(iterator.hasNext())
       {
           byte[] next = iterator.next().message();
           System.out.println("接受到的数据是"+new String(next));
       }
   }