Spark Streaming和Flink的Word Count对比

最新推荐文章于 2024-07-03 17:19:32 发布

拖鞋皮鞋

最新推荐文章于 2024-07-03 17:19:32 发布

阅读量306

点赞数

文章标签： spark

博客介绍了通过netcat构造流输入，创建Spark DStream和Flink DataSream进行处理。对比了Spark和Flink，二者都运行在Hadoop YARN上，性能上Flink最优，迭代次数越多越明显。在流式计算、SQL支持等方面，二者各有优劣，Spark在sql方面有优势，Flink在流式和迭代计算支持力度将增强。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

准备：

nccat for windows/linux 都可以通过 TCP 套接字连接，从流数据中创建了一个 Spark DStream/ Flink DataSream, 然后进行处理, 时间窗口大小为10s
因为示例需要, 所以需要下载一个netcat, 来构造流的输入。

代码：

spark streaming


package cn.kee.spark;
public final class JavaNetworkWordCount {  
   private static final Pattern SPACE = Pattern.compile(” “);  
 
  public static void main(String[] args) throws Exception {  
      if (args.length < 2) {  
           System.err.println(“Usage: JavaNetworkWordCount <hostname> <port>”);  
          System.exit(1);  
       }  
     StreamingExamples.setStreamingLogLevels();  
        SparkConf sparkConf = new SparkConf().setAppName(“JavaNetworkWordCount”);  
      JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, Durations.seconds(1));  
      JavaReceiverInputDStream<String> lines = ssc.socketTextStream(  
              args[0], Integer.parseInt(args[1]), StorageLevels.MEMORY_AND_DISK_SER);  
     JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() {  
           @Override  
          public Iterator<String> call(String x) {  
              return Arrays.asList(SPACE.split(x)).iterator();  
            }  
     });  
       JavaPairDStream<String, Integer> wordCounts = words.mapToPair(  
              new PairFunction<String, String, Integer>() {  
                 @Override  
                  public Tuple2<String, Integer> call(String s) {  
                       return new Tuple2<>(s, 1);  
                 }  
             }).reduceByKey(new Function2<Integer, Integer, Integer>() {  
                   @Override  
                  public Integer call(Integer i1, Integer i2) {  
                       return i1 + i2;  
                 }  
             });  
       wordCounts.print();  
       ssc.start();  
      ssc.awaitTermination();  
   }  
}

Flink DataSream


package cn.kee.flink;
 
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;
/**
 * Example :SocketWindowWordCount
 * @author keehang
 *
 */
public class SocketWindowWordCount {
 
 public static void main(String[] args) throws Exception {
 
      // the port to connect to
     final int port = 9999;
     /*try {
          final ParameterTool params = ParameterTool.fromArgs(args);
            port = params.getInt("port");
     } catch (Exception e) {
           System.err.println("No port specified. Please run 'SocketWindowWordCount --port <port>'");
          return;
       }*/
  
        // get the execution environment
      final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
 
     // get input data by connecting to the socket
     DataStream<String> text = env.socketTextStream("localhost", port, "\n");
 
       // parse the data, group it, window it, and aggregate the counts
      DataStream<WordWithCount> windowCounts = text
             .flatMap(new FlatMapFunction<String, WordWithCount>() {
                 @Override
                    public void flatMap(String value, Collector<WordWithCount> out) {
                     for (String word : value.split("\\s")) {
                         out.collect(new WordWithCount(word, 1L));
                        }
                   }
               })
              .keyBy("word")
             .timeWindow(Time.seconds(5), Time.seconds(1))
             .reduce(new ReduceFunction<WordWithCount>() {
                   @Override
                    public WordWithCount reduce(WordWithCount a, WordWithCount b) {
                       return new WordWithCount(a.word, a.count + b.count);
                    }
               });
 
        // print the results with a single thread, rather than in parallel
        windowCounts.print().setParallelism(1);
 
       env.execute("Socket Window WordCount");
    }
}