java kafka producer_057 Java中kafka的Producer程序实现

1 packagecom.jun.it;2 importkafka.javaapi.producer.Producer;3 importkafka.producer.KeyedMessage;4 importkafka.producer.ProducerConfig;5 importjava.util.Properties;6 importjava.util.Random;7 importjava.util.concurrent.atomic.AtomicBoolean;8 public classJavaKafkaProducer {9 public static final char[] chars = "qazwsxedcrfvtgbyhnujmikolp0123456789".toCharArray();10 public static final int charsLength =chars.length;11 public static final Random random = newRandom(System.currentTimeMillis());12 private Producer producer = null;13

14 private String topicName = null;15 private String brokerList = null;16 private boolean isSync = true; //默认为同步

17 private String partitionerClass = null; //数据分区器class类

18

19 /**

20 * 构造函数21 *22 *@paramtopicName23 *@parambrokerList24 */

25 publicJavaKafkaProducer(String topicName, String brokerList) {26 this(topicName, brokerList, true, null);27 }28

29 /**

30 * 构造函数31 *32 *@paramtopicName33 *@parambrokerList34 *@parampartitionerClass35 */

36 publicJavaKafkaProducer(String topicName, String brokerList, String partitionerClass) {37 this(topicName, brokerList, true, partitionerClass);38 }39

40 /**

41 * 构造函数,主要是产生producer42 *43 *@paramtopicName44 *@parambrokerList45 *@paramisSync46 */

47 public JavaKafkaProducer(String topicName, String brokerList, booleanisSync, String partitionerClass) {48 //赋值

49 this.topicName =topicName;50 this.brokerList =brokerList;51 this.isSync =isSync;52 this.partitionerClass =partitionerClass;53

54 //1. 给定配置信息：参考http://kafka.apache.org/082/documentation.html#producerconfigs

55 Properties props = newProperties();56 //kafka集群的连接信息

57 props.put("metadata.broker.list", this.brokerList);58 //kafka发送数据方式

59 if (this.isSync) {60 //同步发送数据

61 props.put("producer.type", "sync");62 } else{63 //异步发送数据

64 props.put("producer.type", "async");65 /**

66 * 0: 不等待broker的返回67 * 1: 表示至少等待1个broker返回结果68 * -1：表示等待所有broker返回数据接收成功的结果69 */

70 props.put("request.required.acks", "0");71 }72 //key/value数据序列化的类

73 /**

74 * 默认是：DefaultEncoder, 指发送的数据类型是byte类型75 * 如果发送数据是string类型，必须更改StringEncoder76 */

77 props.put("serializer.class", "kafka.serializer.StringEncoder");78

79 //给定分区器的class参数

80 if (this.partitionerClass != null && !this.partitionerClass.trim().isEmpty()) {81 //默认是：DefaultPartiioner，基于key的hashCode进行hash后进行分区

82 props.put("partitioner.class", this.partitionerClass.trim());83 }84

85 //2. 构建Kafka的Producer Configuration上下文

86 ProducerConfig config = newProducerConfig(props);87

88 //3. 构建Kafka的生产者：Producerr

89 this.producer = new Producer(config);90 }91

92 /**

93 * 关闭producer连接94 */

95 public voidcloseProducer() {96 producer.close();97 }98

99 /**

100 * 提供给外部应用调用的直接运行发送数据代码的方法101 *102 *@paramthreadNumbers103 *@paramisRunning104 */

105 public void run(int threadNumbers, finalAtomicBoolean isRunning) {106 for (int i = 0; i < threadNumbers; i++) {107 new Thread(newRunnable() {108 public voidrun() {109 int count = 0;110 while(isRunning.get()) {111 //只有在运行状态的情况下，才发送数据

112 KeyedMessage message =generateMessage();113 //发送数据

114 producer.send(message);115 count++;116 //打印一下

117 if (count % 100 == 0) {118 System.out.println("Count = " + count + "; message:" +message);119 }120

121 //假设需要休息一下

122 try{123 Thread.sleep(random.nextInt(100) + 10);124 } catch(InterruptedException e) {125 //nothings

126 }127 }128 System.out.println("Thread:" + Thread.currentThread().getName() + " send message count is:" +count);129 }130 }).start();131 }132 }133

134 /**

135 * 产生一个随机的Kafka的KeyedMessage对象136 *137 *@return

138 */

139 public KeyedMessagegenerateMessage() {140 String key = generateString(3) + "_" + random.nextInt(10);141 StringBuilder sb = newStringBuilder();142 int numWords = random.nextInt(5) + 1; //[1,5]单词

143 for (int i = 0; i < numWords; i++) {144 String word = generateString(random.nextInt(5) + 1); //单词中字符最少1个最多5个

145 sb.append(word).append(" ");146 }147 String message =sb.toString().trim();148 return new KeyedMessage(this.topicName, key, message);149 }150

151 /**

152 * 随机生产一个给定长度的字符串153 *154 *@paramnumItems155 *@return

156 */

157 public static String generateString(intnumItems) {158 StringBuilder sb = newStringBuilder();159 for (int i = 0; i < numItems; i++) {160 sb.append(chars[random.nextInt(charsLength)]);161 }162 returnsb.toString();163 }164 }