DataStreamAPI之Sink

最新推荐文章于 2024-09-04 10:25:50 发布

原创

最新推荐文章于 2024-09-04 10:25:50 发布 · 733 阅读

1 ·

CC 4.0 BY-SA版权

文章标签：

#kafka #java #分布式 #flink

1.KafkaSink

使用kafkaSink首先需要添加依赖

<dependency>
    <groupId>org.apache.flink</groupId>
    <artifactId>flink-connector-kafka_2.12</artifactId>
    <version>1.13.1</version>
</dependency>
<dependency>
    <groupId>com.alibaba</groupId>
    <artifactId>fastjson</artifactId>
    <version>1.2.75</version>
</dependency>

最重要的就是创建一个Flink程序用的生产者

FlinkKafkaProducer<String> kafkaProducer = new FlinkKafkaProducer<>("hadoop102:9092", "first", new SimpleStringSchema());

其中String泛型代表你写入到kafka的数据类型

package net.cyan.Sink;
import com.alibaba.fastjson.JSON;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.SimpleTimerService;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;

public class Demo1_kafkaSink {
   
   
    public static void main(String[] args) {
   
   
        //创建执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //创建FlinkKafkaProducer
        FlinkKafkaProducer<String> kafkaProducer = new FlinkKafkaProducer<>("hadoop102:9092", "first", new SimpleStringSchema());

        env.socketTextStream("hadoop103",9999)
               //map处理数据
                .map(new MapFunction<String, String>() {
   
   
                    @Override
                    public String map(String s) throws Exception {
   
   
                        //切割数据集
                        String[] split = s.split(",");
                        //转换
                        return JSON.toJSONString(split[0].concat(split[1]).concat(split[2]));
                    }
                })
                 //添加
                .addSink(kafkaProducer);

        try {
   
   
            //启动执行环境
            env.execute();
        } catch (Exception e) {
   
   
            e.printStackTrace();
        }

    }
}