读写kafka 0.7简单示例

Kafka0.8和kafka0.7变化非常大，使用方式和接口class都变掉了。这里简单介绍一下0.7的kafka如何写入数据。

因为刚好工作中用的是0.7的kafka，所以必须要使用0.7的接口往kafka写数据，这是一个老版本的程序，所以意义不大，这是作为参考。

我使用的是maven管理java项目。

首先maven的dependency是这样的：

 

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.x</groupId>
	<artifactId>KafkaWriter</artifactId>
	<version>0.0.1-SNAPSHOT</version>

	<properties>
		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
		<hadoop.version>0.20.2-cdh3u6</hadoop.version>
	</properties>

	<dependencies>

		<!-- Apache Kafka -->
		 <dependency>
		    <groupId>org.scala-lang</groupId>
		    <artifactId>scala-compiler</artifactId>
		    <version>2.8.0</version>
		</dependency>

		 <dependency>
		    <groupId>org.scala-lang</groupId>
		    <artifactId>scala-library</artifactId>
		    <version>2.8.0</version>
		</dependency>

		<dependency>
			<groupId>org.apache.kafka</groupId>
			<artifactId>kafka</artifactId>
			<version>0.7.2</version>


			<exclusions>
				<exclusion>
					<artifactId>jmxri</artifactId>
					<groupId>com.sun.jmx</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jms</artifactId>
					<groupId>javax.jms</groupId>
				</exclusion>
				<exclusion>
					<artifactId>jmxtools</artifactId>
					<groupId>com.sun.jdmk</groupId>
				</exclusion>
			</exclusions>
		</dependency>

		<dependency>
			<groupId>com.101tec</groupId>
			<artifactId>zkclient</artifactId>
			<version>0.4</version>
		</dependency>

		<dependency>
			<groupId>org.apache.commons</groupId>
			<artifactId>commons-lang3</artifactId>
			<version>3.3</version>
		</dependency>
		<dependency>
			<groupId>org.slf4j</groupId>
			<artifactId>slf4j-api</artifactId>
			<version>1.7.9</version>
		</dependency>
		<dependency>
			<groupId>commons-cli</groupId>
			<artifactId>commons-cli</artifactId>
			<version>1.2</version>
		</dependency>


		<!-- Unit Test -->
		<dependency>
			<groupId>org.testng</groupId>
			<artifactId>testng</artifactId>
			<version>6.1.1</version>
			<scope>test</scope>
		</dependency>
	</dependencies>

	<build>
		<finalName>${project.artifactId}</finalName>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-surefire-plugin</artifactId>
				<version>2.8.1</version>
				<configuration>
					<skipTests>true</skipTests>
				</configuration>
			</plugin>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.0</version>
				<configuration>
					<source>1.6</source>
					<target>1.6</target>
					<encoding>UTF-8</encoding>
					<compilerArguments>
						<extdirs>src${file.separator}main${file.separator}lib</extdirs>
					</compilerArguments>
				</configuration>
			</plugin>
			<plugin>
				<artifactId>maven-assembly-plugin</artifactId>
				<configuration>
					<descriptorRefs>
						<descriptorRef>jar-with-dependencies</descriptorRef>
					</descriptorRefs>
				</configuration>
				<executions>
					<execution>
						<id>config</id>
						<phase>package</phase>
						<goals>
							<goal>single</goal>
						</goals>
					</execution>
				</executions>
			</plugin>
		</plugins>
	</build>
</project>

写kafka只有一个简单的java class，通过命令行往kafka写数据，支持两种模式，一种是文件读取，一种是标准输入输出模式。可以根据这个程序，写出符合自己需求的程序。

 

package com.x;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;

import kafka.javaapi.producer.ProducerData;
import kafka.producer.ProducerConfig;
import kafka.javaapi.producer.Producer;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.Properties; 

import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;


/**
 * Flume sink that propogates data to Kafka cluster
 * 
 */
public class KafkaSimpleWriter {

    private static final Logger Logger = LoggerFactory.getLogger(KafkaSimpleWriter.class);

//    private static final String KAFKA_TOPIC = "kafka.topic";
//    private static final String KAFKA_BROKERS = "kafka.brokers";
//    private static final String KAFKA_BUNDLE_SIZE = "kafka.bundle.size";
//    private static final String KAFKA_BUNDLE_INTERVAL_MS = "kafka.bundle.interval.ms";
//    private static final String KAFKA_MESSAGE_SIZE = "kafka.message.size";
//    private static final String KAFKA_HEARTBEAT_MS = "kafka.heartbeat.ms";

    private String topic;
    private String zkString;
    private Producer<String, String> producer;
    private ProducerConfig config;

    private long totalCnt = 0;
    
    
    public KafkaSimpleWriter(String file_path, String topic, String zkString) throws IOException {
    	this.topic = topic;
    	this.zkString = zkString;
    	Properties properties = new Properties();
    	if(file_path != null && file_path.isEmpty()== false ){
    		properties.load(Thread.currentThread().getContextClassLoader().getResourceAsStream(file_path));
    	}
    	
    	SetDefaultProperty(properties, "serializer.class", "kafka.serializer.StringEncoder");
    	SetDefaultProperty(properties, "zk.connect", this.zkString);
    	SetDefaultProperty(properties, "zk.connectiontimeout.ms", "65000");
    	SetDefaultProperty(properties, "producer.type", "async");
    	SetDefaultProperty(properties, "queue.time", "10000");
    	SetDefaultProperty(properties, "queue.size", "10000");
        
        this.config = new ProducerConfig(properties);

        this.producer = new Producer<String, String>(this.config);
    }

	private void SetDefaultProperty(Properties prop, String key, String defaultValue){
		if(prop.getProperty(key) == null){
			prop.setProperty(key, defaultValue);
		}
	}
	
	public void sendMessage(String message){
		
		this.producer.send(new ProducerData<String, String>(topic, message));
		totalCnt++;
		if(totalCnt%1000000 == 0){
			Logger.info("Total send messages:{} to topic:{}", totalCnt, this.topic);
		}
	}
	
	public long getCount(){ return this.totalCnt; }
	
	
	static public void main(String[] args) throws IOException, ParseException{
		String topic = null;
		String zkString = null;
		String config_path = null;
		String data_path = null;
		
		Options opts = new Options();
        opts.addOption("h", false, "help message");
        opts.addOption("data", true, "Data file path");
        opts.addOption("config", true, "config file path");
        opts.addOption("topic", true, "kafka topic");
        opts.addOption("zookeeper", true, "zookeeper list");
        
        BasicParser parser = new BasicParser();
        CommandLine cl = parser.parse(opts, args);
        
        if (cl.hasOption('h') || cl.hasOption("topic") == false 
        		|| cl.hasOption("zookeeper") == false ) {
            HelpFormatter hf = new HelpFormatter();
            hf.printHelp("OptionsTip", opts);
            return ;
        } else {
        	topic = cl.getOptionValue("topic");
            zkString = cl.getOptionValue("zookeeper");
            config_path = cl.getOptionValue("config", null);
            data_path = cl.getOptionValue("data", null);
        }
        Logger.info("topic={}, zookeeper={}, config={}, data={}", topic, zkString, config_path, data_path);
        
        
		KafkaSimpleWriter writer = new KafkaSimpleWriter(config_path, topic, zkString);
		
		if(data_path == null || data_path.isEmpty()){
			BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
	        String s = null;
	        try {
	            while((s = in.readLine()) != null){
	                writer.sendMessage(s);
	            }
	            
	        } catch(IOException e) {
	            e.printStackTrace();
	        } finally {
	        	in.close();
	        }
		}else{
			BufferedReader in = new BufferedReader(new FileReader(data_path));
			try{
					String s = null;
					while((s = in.readLine()) != null){
						writer.sendMessage(s);
					}
			}catch(IOException e){
				e.printStackTrace();
			}finally{
				in.close();
			}
		}

		System.out.println("Successfully write records:" + writer.getCount());
	}
   
}

编译很简单：

 mvn clean package

在target目录下，会得到一个jar包。

运行也很简单：

 java -cp $your_jar_file com.x.KafkaSimpleWriter -zookeeper localhost:2181 -topic test -data data.txt

读kafka也是一个简单的java class，通过命令行从kafka读数据，打印到标准输出模式。可以根据这个程序，写出符合自己需求的程序。

 

package com.x;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import kafka.consumer.Consumer;
import kafka.consumer.ConsumerConfig;
import kafka.consumer.KafkaStream;
import kafka.javaapi.consumer.ConsumerConnector;
import kafka.message.Message;
import kafka.message.MessageAndMetadata;

public class KafkaSimpleReader {
	private static final Logger Logger = LoggerFactory
			.getLogger(KafkaSimpleWriter.class);

	// private static final String KAFKA_TOPIC = "kafka.topic";
	// private static final String KAFKA_BROKERS = "kafka.brokers";
	// private static final String KAFKA_BUNDLE_SIZE = "kafka.bundle.size";
	// private static final String KAFKA_BUNDLE_INTERVAL_MS =
	// "kafka.bundle.interval.ms";
	// private static final String KAFKA_MESSAGE_SIZE = "kafka.message.size";
	// private static final String KAFKA_HEARTBEAT_MS = "kafka.heartbeat.ms";

	private String topic;
	private String zkString;

	private ConsumerConfig consumerConfig;
	private ConsumerConnector consumerConnector;
	KafkaStream<Message> stream;

	private long totalCnt = 0;

	public KafkaSimpleReader(String file_path, String topic, String zkString)
			throws IOException {
		this.topic = topic;
		this.zkString = zkString;

		Properties properties = new Properties();
		if (file_path != null && file_path.isEmpty() == false) {
			properties.load(Thread.currentThread().getContextClassLoader()
					.getResourceAsStream(file_path));
		}

		SetDefaultProperty(properties, "zk.connect", this.zkString);
		SetDefaultProperty(properties, "zk.connectiontimeout.ms", "30000");
		SetDefaultProperty(properties, "groupid", "commonreader");

		System.out.println(properties.toString());

		this.consumerConfig = new ConsumerConfig(properties);

		this.consumerConnector = Consumer
				.createJavaConsumerConnector(consumerConfig);

		HashMap<String, Integer> map = new HashMap<String, Integer>();
		map.put(topic, 1);
		Map<String, List<KafkaStream<Message>>> topicMessageStreams = consumerConnector
				.createMessageStreams(map);
		stream = topicMessageStreams.get(topic).get(0);
	}

	private void SetDefaultProperty(Properties prop, String key,
			String defaultValue) {
		if (prop.getProperty(key) == null) {
			prop.setProperty(key, defaultValue);
		}
	}

	public void readMessage() {

		for (MessageAndMetadata<Message> msgAndMetadata : stream) {
			System.out.println("topic: " + msgAndMetadata.topic());
			Message message = (Message) msgAndMetadata.message();
			ByteBuffer buffer = message.payload();
			byte[] bytes = buffer.array();
			String tmp = new String(bytes);
			System.out.println("message content: " + tmp);
			totalCnt++;
			if (totalCnt % 1000000 == 0) {
				Logger.info("Total send messages:{} to topic:{}", totalCnt,
						this.topic);
			}
		}
	}

	public void close() {
	}

	public long getCount() {
		return this.totalCnt;
	}

	public static void main(String[] args) throws IOException, ParseException{
		String topic = null;
		String zkString = null;
		String config_path = null;
		
		Options opts = new Options();
        opts.addOption("h", false, "help message");
        opts.addOption("config", true, "config file path");
        opts.addOption("topic", true, "kafka topic");
        opts.addOption("zookeeper", true, "zookeeper list");
        
        BasicParser parser = new BasicParser();
        CommandLine cl = parser.parse(opts, args);
        
        if (cl.hasOption('h') || cl.hasOption("topic") == false 
        		|| cl.hasOption("zookeeper") == false ) {
            HelpFormatter hf = new HelpFormatter();
            hf.printHelp("OptionsTip", opts);
            return ;
        } else {
        	topic = cl.getOptionValue("topic");
            zkString = cl.getOptionValue("zookeeper");
            config_path = cl.getOptionValue("config", null);
        }
        Logger.info("topic={}, zookeeper={}, config={}, data={}", topic, zkString, config_path);
        
		KafkaSimpleReader reader = new KafkaSimpleReader(config_path, topic, zkString);
		
		try{
			reader.readMessage();
		}catch (Exception e){
			e.printStackTrace();
		}

		System.out.println("Successfully read records:" + reader.getCount());
	}
}

编译很简单：

 mvn clean package

在target目录下，会得到一个jar包。

运行也很简单：

 java -cp $your_jar_file com.x.KafkaSimpleReader -zookeeper localhost:2181 -topic test