kafka storm 整合_storm1.2.2 kafka1.0.0-优快云博客

本文链接：https://blog.youkuaiyun.com/gengjianchun/article/details/83537562

本文详细介绍如何使用Apache Storm和Apache Kafka进行集成，包括配置Kafka Spout、创建Topology、处理Kafka消息并将其发送到Storm Bolt的过程。通过具体代码示例，读者可以了解如何在实际项目中实现Kafka与Storm的高效结合。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

kafka版本：kafka_2.11-1.1.1

storm版本：apache-storm-1.2.2

pom:

 <dependencies>
        <dependency>
            <groupId>org.apache.storm</groupId>
            <artifactId>storm-kafka-client</artifactId>
            <version>1.2.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>1.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.storm</groupId>
            <artifactId>storm-core</artifactId>
            <version>1.2.2</version>
             <!--  本地测试注释集群运行打开 -->
              <scope>provided</scope>
        </dependency>
    </dependencies>
     <build>
         <plugins>
             <!-- 资源文件拷贝插件 -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-resources-plugin</artifactId>
                 <version>2.7</version>
                 <configuration>
                     <encoding>UTF-8</encoding>
                 </configuration>
             </plugin>
             <!-- java编译插件 -->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>3.2</version>
                 <configuration>
                     <source>1.8</source>
                     <target>1.8</target>
                     <encoding>UTF-8</encoding>
                 </configuration>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
                 <version>2.4</version>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>
                 <version>2.4</version>
                 <configuration>
                     <descriptorRefs>
                         <descriptorRef>jar-with-dependencies</descriptorRef>
                     </descriptorRefs>
                     <archive>
                         <manifest>
                             <mainClass>cn.itcast.kafka.Kafka2Storm</mainClass>
                         </manifest>
                     </archive>
                 </configuration>
                 <executions>
                     <execution>
                         <id>make-assembly</id>
                         <phase>package</phase>
                         <goals>
                             <goal>single</goal>
                       </goals>
                     </execution>
                 </executions>
             </plugin>
         </plugins>
     </build>

package test.stormKafka;

import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.generated.AlreadyAliveException;
import org.apache.storm.generated.AuthorizationException;
import org.apache.storm.generated.InvalidTopologyException;
import org.apache.storm.kafka.spout.KafkaSpout;
import org.apache.storm.kafka.spout.KafkaSpoutConfig;
import org.apache.storm.topology.TopologyBuilder;

public class TopologyClass {

	public static void main(String[] args)
			throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {

		TopologyBuilder topologyBuilder = new TopologyBuilder();

		KafkaSpoutConfig.Builder<String, String> kafkaBuilder = KafkaSpoutConfig.builder("geng2:9092,geng3:9092",
				"mytest");
		// 设置kafka属于哪个组
		kafkaBuilder.setProp("group.id", "mytest");
		// 创建kafkaspoutConfig
		KafkaSpoutConfig<String, String> build = kafkaBuilder.build();
		// 通过kafkaspoutConfig获得kafkaspout
		KafkaSpout<String, String> kafkaSpout = new KafkaSpout<String, String>(build);

		topologyBuilder.setSpout("kafkaSpout", kafkaSpout);
		
		topologyBuilder.setBolt("valueBolt", new KafkaBolt(), 1).shuffleGrouping("kafkaSpout");
	//topologyBuilder.setBolt("mybolt2", new MyBolt(), 1).shuffleGrouping("kafkaSpout");
		// 启动topology的配置信息
		Config config = new Config();
		// 定义集群分配多少个工作进程来执行这个topology
		config.setNumWorkers(3);
		config.setNumAckers(1);

		LocalCluster localCluster = new LocalCluster();
		localCluster.submitTopology("kafkaStomrTopology", config, topologyBuilder.createTopology());
		// 集群模式提交topology
		// StormSubmitter.submitTopologyWithProgressBar("mytest", config,
		// topologyBuilder.createTopology());
	}
}

package test.stormKafka;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Map;
import java.util.Random;

import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichBolt;
import org.apache.storm.tuple.Tuple;

public class KafkaBolt extends BaseRichBolt {

	private OutputCollector collector;

	public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {

		this.collector = collector;
	}

	public void execute(Tuple input) {
		System.out.println(input.toString());
		/*
		 * int nextInt = new Random().nextInt(100); File f = new
		 * File("/home/geng/mytest-"+nextInt); OutputStream out; try { out = new
		 * FileOutputStream(f); out.write(input.getString(4).getBytes());
		 * out.flush(); out.close(); } catch (FileNotFoundException e) { // TODO
		 * Auto-generated catch block e.printStackTrace(); } catch (IOException
		 * e) { // TODO Auto-generated catch block e.printStackTrace(); }
		 */

		collector.ack(input);

	}

	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		// TODO Auto-generated method stub

	}

}