kafka+sprkStreaming+hbase

1，kafka的安装，参照kafka的安装文档

第一步：建立一个自己的topic:

bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic testspark

启动自己的topic

bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test 

 

第二步：基于maven来构建kafka环境

（1）pom文件：见单独文件

（2）已经安装hbase(本例的hbase表格为：test1)

（3）源码：只是很简单的将数据接过来并写入hbase

import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.hbase.client.{Get, Put, HTable}
import org.apache.hadoop.hbase.util.Bytes
import org.apache.spark.streaming._
import org.apache.spark.streaming.kafka._
import org.apache.spark.SparkConf
object testkafka {
  def main(args: Array[String]) {
    //    if (args.length < 4) {
    //      System.err.println("Usage: KafkaWordCount <zkQuorum>     <group> <topics> <numThreads>")
    //      System.exit(1)
    //    }
    //    StreamingExamples.setStreamingLogLevels()
    //val Array(zkQuorum, group, topics, numThreads) = args
    val zkQuorum = "10.248.27.3:2181"
    val group = "1"
    val topics = "testspark"
    val numThreads = 2
    val sparkConf = new SparkConf().setAppName("testkafka").setMaster("local[2]")
    val ssc =  new StreamingContext(sparkConf, Seconds(2))

    //hbase的配置
    val tablename = "test1"
    val conf = HBaseConfiguration.create
    conf.set("hbase.zookeeper.quorum","10.248.27.1,10.248.27.2,10.248.27.3,10.248.27.4,10.248.27.5")
    conf.set("hbase.zookeeper.property.clientPort","2181")
    conf.set("hbase.master","10.248.27.1:60000")
    val table = new HTable(conf, tablename)
    val databytes = Bytes.toBytes("id")
   // ssc.checkpoint("checkpoint")
    val topicpMap = topics.split(",").map((_,numThreads)).toMap
    val lines = KafkaUtils.createStream(ssc, zkQuorum, group, topicpMap).map(_._2)
    val words = lines.flatMap(_.split(" "))
//    words.collect().foreach(x=>println(x))
    words.print()
   //val pairs = words.map(word => (word, 1))
   // val wordCounts = pairs.reduceByKey(_ + _)
    words.foreachRDD(rdd=>{
      val data=rdd.collect().foreach(x=>{
        val row = Bytes.toBytes("row" + x.toString)
        val p1 = new Put(row)
        p1.add(databytes, Bytes.toBytes("test"+x.toString), Bytes.toBytes("value" + x.toString))
        table.put(p1)
      })
      println("************************************************"+data)
    })
       ssc.start()
    ssc.awaitTermination()
  }

 

 （4）:在idea中本地启动程序，在打开的topic中输入数据，再去hbase表格中查询数据。

 

Pom.xml

 

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>KafkaSparkStreamingHbase</groupId>
    <artifactId>KafkaSparkStreamingHbase</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>1.5</maven.compiler.source>
        <maven.complier.target>1.5</maven.complier.target>
        <encoding>UTF-8</encoding>
        <scala.version>2.10.2</scala.version>
    </properties>
    <dependencies>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>2.10.2</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-actors</artifactId>
            <version>2.10.2</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-compiler</artifactId>
            <version>2.10.2</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-reflect</artifactId>
            <version>2.10.2</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.module</groupId>
            <artifactId>jackson-module-scala_2.10</artifactId>
            <version>2.4.4</version>
        </dependency>

        <dependency>
            <groupId>com.typesafe.akka</groupId>
            <artifactId>akka-actor_2.10</artifactId>
            <version>2.3.6</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.10</artifactId>
            <version>1.3.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-assembly_2.10</artifactId>
            <version>1.1.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_2.10</artifactId>
            <version>1.3.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming-kafka_2.10</artifactId>
            <version>1.3.1</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>3.8.1</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.34</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase</artifactId>
            <version>0.98.1-hadoop2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>0.98.1-hadoop2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-common</artifactId>
            <version>0.98.1-hadoop2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-server</artifactId>
            <version>0.98.1-hadoop2</version>
        </dependency>

    </dependencies>
    <build>
        <sourceDirectory>src/main/spark/</sourceDirectory>
        <testSourceDirectory>src/test/scala/</testSourceDirectory>
        <plugins>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <version>2.15.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                        <configuration>
                            <args>
                                <arg>-dependencyfile</arg>
                                <arg>${project.build.directory}/.scala_dependencies</arg>
                            </args>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>