flink1.18.0版本读取kafka中的数据放入到msyql(tidb)中

最新推荐文章于 2024-11-19 17:33:06 发布

weixin_45003481

最新推荐文章于 2024-11-19 17:33:06 发布

阅读量612

点赞数

文章标签： kafka tidb linq

本文链接：https://blog.youkuaiyun.com/weixin_45003481/article/details/134533505

版权

Stn实体类

package org.flink;

import lombok.Data;

import java.io.Serializable;

@Data
public class Stn implements Serializable {
    // 时间戳
    private long ts;
    // 车牌号
    private String vehicleNo;
    // 线路号
    private String routeNo;
    // 子线路号
    private String subRouteNo;
    // 采集时间
    private String collectTime;
    // GPS状态
    private int gpsState;
    // 经度
    private double latitude;
    // 纬度
    private double longitude;
    // 速度
    private double speed;
    // 方向角
    private int angle;
    // 运行方向
    private int runDirection;
    // 到站状态
    private int arriveState;
    // 站点编码
    private String stationNo;
    // 车载设备运行状态
    private int deviceState;

    // 标签枚举值
    private String location;
    private String category;
    private int port;
    private String biz;

    @Override
    public String toString() {
        return "Stn{" +
                "ts=" + ts +
                ", vehicleNo='" + vehicleNo + '\'' +
                ", routeNo='" + routeNo + '\'' +
                ", subRouteNo='" + subRouteNo + '\'' +
                ", collectTime='" + collectTime + '\'' +
                ", gpsState=" + gpsState +
                ", latitude=" + latitude +
                ", longitude=" + longitude +
                ", speed=" + speed +
                ", angle=" + angle +
                ", runDirection=" + runDirection +
                ", arriveState=" + arriveState +
                ", stationNo='" + stationNo + '\'' +
                ", deviceState=" + deviceState +
                '}';
    }
}

KafkaToMysql

package org.flink;

import com.alibaba.fastjson.JSON;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.connector.jdbc.JdbcConnectionOptions;
import org.apache.flink.connector.jdbc.JdbcExecutionOptions;
import org.apache.flink.connector.jdbc.JdbcSink;
import org.apache.flink.connector.kafka.source.KafkaSource;
import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.ProcessFunction;
import org.apache.flink.util.Collector;

import java.util.concurrent.TimeUnit;

public class KafkaToMysql {
    public static void main(String[] args) throws Exception {

        // 设置执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //重启策略
        //固定延时重启 (最多尝试3次, 每次间隔10s)
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, Time.of(10, TimeUnit.SECONDS))); // 间隔
        //失败率重启 (在10分钟内最多尝试3次, 每次至少间隔1分钟)
        env.setRestartStrategy(RestartStrategies.failureRateRestart(3,Time.minutes(10), Time.minutes(1)));


        // 从预定义的元素中创建数据流
        KafkaSource<String> source = KafkaSource.<String>builder()
                .setBootstrapServers("服务地址")
                .setTopics("topic")
                .setGroupId("flink_cdc_test")
                .setStartingOffsets(OffsetsInitializer.latest())//消费策略
                .setValueOnlyDeserializer(new SimpleStringSchema())
                .build();

        DataStreamSource<String> streamSource = env.fromSource(source, WatermarkStrategy.noWatermarks(),"Kafka Source").setParallelism(3);

        //streamSource.print();


        SingleOutputStreamOperator<Stn> stnStream = streamSource.process(new ProcessFunction<String, Stn>() {
            @Override
            public void processElement(String value, ProcessFunction<String, Stn>.Context ctx, Collector<Stn> out) throws Exception {
                Stn stn = JSON.parseObject(value, Stn.class);
                out.collect(stn);
            }
        });


        //将预警匹配到的数据写入mysql
        //数据写入MySQL策略
        JdbcExecutionOptions executionOptions = JdbcExecutionOptions.builder()
                .withBatchSize(5000) // 每1000条记录批处理一次
                .withBatchIntervalMs(2*60*1000) // 或者每200毫秒批处理一次
                .withMaxRetries(5) // 如果遇到错误，重试最多5次
                .build();


        // 数据写入MySQL
        stnStream.addSink(JdbcSink.sink(
                "INSERT INTO stn_history (vehicle_no,route_no,station_no,collect_time,gps_state," +
                        "latitude,longitude,speed,angle,run_direction,arrive_state,device_state)" +
                        " VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
                (ps, value) -> {
                    // 设置参数
                    ps.setString(1,value.getVehicleNo());
                    ps.setString(2,value.getRouteNo());
                    ps.setString(3,value.getStationNo());
                    ps.setString(4,value.getCollectTime());
                    ps.setInt(5,value.getGpsState());
                    ps.setDouble(6,value.getLatitude());
                    ps.setDouble(7,value.getLongitude());
                    ps.setDouble(8,value.getSpeed());
                    ps.setInt(9,value.getAngle());
                    ps.setInt(10,value.getRunDirection());
                    ps.setInt(11,value.getArriveState());
                    ps.setInt(12,value.getDeviceState());
                },
                executionOptions,
                new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
                        .withUrl("jdbc:mysql://172.10.100.206:4000/qd_tenant_azs64g855ung?allowPublicKeyRetrieval=true&useSSL=false&useUnicode=true&characterEncoding=UTF8&serverTimezone=GMT")
                        .withDriverName("com.mysql.jdbc.Driver")
                        .withUsername("用户名")
                        .withPassword("tidb密码")
                        .build()
        )).setParallelism(1);

        // 启动Flink程序
        env.execute();
    }

}

pom.xml文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>kafka_to_mysql</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <flink.version>1.18.0</flink.version>
    </properties>


    <dependencies>






        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-streaming-java -->
        <!--<dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java</artifactId>
            <version>1.18.0</version>
&lt;!&ndash;            <scope>${project.build.scope}</scope>&ndash;&gt;
        </dependency>-->


        <!--<dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-core</artifactId>
            <version>1.18.0</version>
        </dependency>-->


        <!-- https://mvnrepository.com/artifact/commons-collections/commons-collections -->
        <dependency>
            <groupId>commons-collections</groupId>
            <artifactId>commons-collections</artifactId>
            <version>3.2.2</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-clients -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients</artifactId>
            <version>${flink.version}</version>
            <exclusions>
                <exclusion>
                    <artifactId>lz4-java</artifactId>
                    <groupId>org.lz4</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>snappy-java</artifactId>
                    <groupId>org.xerial.snappy</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>kryo</artifactId>
                    <groupId>com.esotericsoftware.kryo</groupId>
                </exclusion>
            </exclusions>
        </dependency>



        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka</artifactId>
            <version>3.0.1-1.18</version>
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-jdbc -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-jdbc_2.12</artifactId>
            <version>1.9.0</version>
            <scope>provided</scope>
        </dependency>



        <!--        flink jdbc-->
        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-jdbc -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc</artifactId>
            <version>3.1.1-1.17</version>
            <!--            <scope>provided</scope>-->
        </dependency>

        <!-- MySQL Connector/J -->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>8.0.31</version>
        </dependency>



        <!--        flink-table-api-java-->
        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-table-api-java -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-table-api-java-bridge -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge</artifactId>
            <version>1.18.0</version>
<!--            <scope>provided</scope>-->
        </dependency>


        <!-- https://mvnrepository.com/artifact/org.apache.flink/flink-table-planner -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_2.12</artifactId>
            <version>${flink.version}</version>
            <exclusions>
                <exclusion>
                    <artifactId>scala-library</artifactId>
                    <groupId>org.scala-lang</groupId>
                </exclusion>
            </exclusions>
            <!--            <scope>test</scope>-->
        </dependency>



        <!--fastjson    用于Json文件转bean-->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>2.0.35</version>
        </dependency>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.30</version>
            <scope>compile</scope>
        </dependency>


    </dependencies>






    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.1.1</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <artifactSet>
                                <excludes>
                                    <exclude>com.google.code.findbugs:jsr305</exclude>
                                </excludes>
                            </artifactSet>
                            <filters>
                                <filter>
                                    <!-- Do not copy the signatures in the META-INF folder.
                                    Otherwise, this might cause SecurityExceptions when using the JAR. -->
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                            <transformers>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
                                    <!-- Replace this with the main class of your job -->
                                    <mainClass>org.flink.KafkaToMysql</mainClass>
                                </transformer>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                            </transformers>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>2.3.2</version>
                <configuration>
                    <source>8</source>
                    <target>8</target>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>

提交任务之后,可以正常运行