spark streaming案例整理

scala程序实现，接收一个端口发来的信息：端口信息使用模拟程序来实现一个端口发来的信息。
代码如下：

程序生成A-G的随机字母

 

importjava.io.PrintWriter

importjava.net.ServerSocket

classLoggerSimulation {

 

}

object LoggerSimulation{

  /**

    * 生成一个字母

    * @param 字母的下标

    * @return  生成的字母

    */

  def gennerateContent(index:Int):String = {

    importscala.collection.mutable.ListBuffer

    val charList = ListBuffer[Char]();

    for(i<-65to90){

      charList += i.toChar

    }

    val charArray = charList.toArray

    charArray(index).toString();

  }

 

  /**

    * 生成随机下标

    * @return 返回一个下标

    */

  def index = {

    importjava.util.Random

    val rdm = newRandom()

    rdm.nextInt(7)

  }

 

  

  def main(args: Array[String]): Unit = {

  

 

    val listener = newServerSocket(9090) //输入端口号

    println("已经做好连接的准备-------")

    while(true){

      val socket = listener.accept()

      newThread(){

        override def run(): Unit = {

          println("Got client connected from:"+socket.getInetAddress)

          val out = newPrintWriter(socket.getOutputStream,true)

          while(true){

            Thread.sleep(2000) //间隔时间

            val content = gennerateContent(index)

            println(content)

            out.write(content+"\n")

            out.flush()

          }

          socket.close()

        }

      }.start()

    }

  }

}

spark程序

importorg.apache.spark.storage.StorageLevel

importorg.apache.spark.streaming.{Seconds, StreamingContext}

importorg.apache.spark.{SparkConf, SparkContext}

classNetWorkWordCount {

 

}

object NetWorkWordCount{

  def main(args: Array[String]): Unit = {

 

 

    val config = newSparkConf().setAppName("NetWorkWordCount").setMaster("local");

    val ssc = newStreamingContext(config,Seconds(5)); //多长时间为间隔

    val lines = ssc.socketTextStream("10.1.31.12",8899,StorageLevel.MEMORY_ONLY_SER) //主机和端口地址

    val words = lines.flatMap(line => line.split(" "));

    val wordCount = words.map(x=>(x,1)).reduceByKey(_+_);

    wordCount.print()

    ssc.start();

    ssc.awaitTermination();

  }

}

另一种方式为

打开一个终端，打开一个终端，输入 命令 nc -lk 9999   然后输入内容即可

http://www.jianshu.com/p/59733597d448 挺详细的