SparkStreaming练习（包括：从Linux系统接收数据，flatmap切分，maptopair配对，reducebykey聚合，print打印输出，期间接受5S的监控，启动，等待输入）【J】

本文链接：https://blog.youkuaiyun.com/wyqwilliam/article/details/81273810

本文展示了如何使用SparkStreaming从Linux系统接收数据，通过flatmap进行切分，使用mapToPair进行配对，并利用reduceByKey进行聚合操作。程序每5秒接收一次数据，最终结果进行打印输出。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

package com.bjsxt;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;

import scala.Tuple2;
import scala.actors.threadpool.Arrays;

public class SparkStreamingTest {
   public static void main(String[] args) {
       SparkConf conf=new SparkConf().setAppName("test").setMaster("local");
       JavaSparkContext sc=new JavaSparkContext(conf);
       JavaStreamingContext jsc=new JavaStreamingContext(sc,Durations.seconds(5));
       JavaReceiverInputDStream<String> lines = jsc.socketTextStream("node01", 9999);
       JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() {

           @Override
           public Iterable<String> call(String line) throws Exception {

               return Arrays.asList(line.split(" "));
           }
       });
       JavaPairDStream<String, Integer> pairwords = words.mapToPair(new PairFunction<String, String, Integer>() {

           @Override
           public Tuple2<String, Integer> call(String word) throws Exception {

               return new Tuple2<String,Integer>(word,1);
           }
       });
       JavaPairDStream<String, Integer> result = pairwords.reduceByKey(new Function2<Integer, Integer, Integer>() {

           @Override
           public Integer call(Integer v1, Integer v2) throws Exception {

               return v1+v2;
           }
       });
       result.print();
       jsc.start();
       jsc.awaitTermination();
       jsc.stop();
   }
}