flink2_ source读取数据

最新推荐文章于 2024-11-14 22:23:30 发布

原创最新推荐文章于 2024-11-14 22:23:30 发布 · 394 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#flink #大数据 #kafka

flink 同时被 2 个专栏收录

4 篇文章

订阅专栏

大数据计算组件

4 篇文章

订阅专栏

本文详细介绍了 Apache Flink 中的数据源读取方法，包括从集合、文件及 Kafka 中读取数据的具体实现方式，并提供了自定义数据源的示例代码。

在flink程序流程中属于第二步

从集合或文件中读取

// 从集合中读取
env.fromCollection(Arrays.asList(
                "hello java",
                "hello spark",
                "hello flink",
                "hello scala"));

// 从文件中读取
env.readTextFile("<filePath>");

从kafka中读取

添加第三方包

<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-connector-kafka-0.11_2.11</artifactId>
    <version>1.10.1</version>
 </dependency>

连接代码

// 定义链接kafka的对应配置，告诉程序如何连接kafka,即kafka地址，offset的机制等
Properties properties = new Properties();
properties.setProperty("bootstrap.servers","localhost:9092");
properties.setProperty("group_id","consumer-group");
properties.setProperty("key.deserializer","org.apache.kafka.common.serialization.StrinDeserializer");       
properties.setProperty("value.deserializer","org.apache.kafka.common.serialization.StrinDeserializer");
properties.setProperty("auto.offset.reset","latest");

DataStreamSource<String> = 
          env.addSource(new FlinkKafkaConsumer010<String>(
                // kafka的topic
                "test",
                new SimpleStringSchema(),
                properties));

自定义source 数据源

package com.flinkTest.source;

import com.flinkTest.POJO.SensorReading;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.source.SourceFunction;

import java.util.HashMap;
import java.util.Random;

public class ModifyMySource {
    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        DataStreamSource inputStream = env.addSource(new MySource()); // 从自定义数据源读取数据

        inputStream.print(); // 没有逻辑处理，直接打印

        env.execute();

    }
    
    /** 自定义数据源， 数据是 生产1组 10个温度传感器的对象
     *  
     * */
    public static class  MySource<i> implements SourceFunction<SensorReading> {
        private boolean running = true;
        
        @Override
        public void run(SourceContext sourceContext) throws Exception {

            Random random = new Random();
            //生成10个初始sensor温度
            // 设置初始温度值(10个)
            HashMap<String,Double> tempMap = new HashMap<>();

            for (int i =0; i<10 ; i++){
                tempMap.put("sensor_" + (i+1), 60+random.nextGaussian()*20);
            }
            
            while(running){
                for(String id: tempMap.keySet()){
                    Double newTemp = tempMap.get(id) + random.nextGaussian();
                    tempMap.put(id,newTemp);
                    sourceContext.collect(new SensorReading(id,System.currentTimeMillis(),newTemp));
                }

                Thread.sleep(1000);
            }
        }

        @Override
        public void cancel() {
            running = false;
        }
    }
    
    // 温度传感器 内部类
    public  static  class SensorReading {
        private String id;
        private Long timestamp;
        private Double temperature;

    public SensorReading() {
        }

    public SensorReading(String id, Long timestamp, Double temperature) {
            this.id = id;
            this.timestamp = timestamp;
            this.temperature = temperature;
        }

        public String getId() {
            return id;
        }

        public void setId(String id) {
            this.id = id;
        }

        public Long getTimestamp() {
            return timestamp;
        }

        public void setTimestamp(Long timestamp) {
            this.timestamp = timestamp;
        }

        public Double getTemperature() {
            return temperature;
        }

        public void setTemperature(Double temperature) {
            this.temperature = temperature;
        }

        @Override
        public String toString() {
            return "SensorReading{" +
                    "id='" + id + '\'' +
                    ", timestamp=" + timestamp +
                    ", temperature=" + temperature +
                    '}';
        }
    }
}