Flink read data from hdfs

最新推荐文章于 2022-04-21 18:41:53 发布

原创最新推荐文章于 2022-04-21 18:41:53 发布 · 1.1k 阅读

3 ·

CC 4.0 BY-SA版权

文章标签：

#hdfs

hdfs 专栏收录该内容

2 篇文章

订阅专栏

本文介绍如何使用Apache Flink从HDFS中读取数据，包括DataStreamSource实时读取和DataSet批处理两种方式。通过具体代码示例，展示了如何配置参数实现递归读取文件，并将数据写入本地文件系统。

DataStreamSource 读取hdfs

package com.umetrip.umeflink.connector.hdfs;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import java.util.Arrays;
import java.util.List;


public class UmeHdfsSource {
 
    public static void main(String[] args) throws Exception {

        StreamExecutionEnvironment env1 = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> stringDataStreamSource = env1.readTextFile("hdfs://10.5.xxx.xxx:8020/test/seamless/2019-06-25/part-0-0.txt");
        // stringDataStreamSource.print();
        stringDataStreamSource.writeAsText("/Users/xxxx/testdata/").setParallelism(1);
        env1.execute();
    }
}

DataSet 批处理

package com.umetrip.umeflink.connector.hdfs;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import java.util.Arrays;
import java.util.List;


public class UmeHdfsSource {
 
    public static void main(String[] args) throws Exception {


       ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        // create a configuration object
        Configuration parameters = new Configuration();
        // set the recursive enumeration parameter
        parameters.setBoolean("recursive.file.enumeration", true);
        // read text file from a HDFS running at nnHost:nnPort
        // DataSet<String> hdfsLines = env.readTextFile("hdfs://10.5.xxx.xxx:8020/TestData");
        // 如果本地读取HA HDFS，需要把core-site.xml,hdfs-site.xml放到resources目录下，路径为hdfs://namespace/xx/xx
        //DataSet<String> hdfsLines = env.readTextFile("hdfs://10.5.xxx.xx:8020/test/seamless/2019-06-25");
        DataSet<String> hdfsLines = env.readTextFile("hdfs://10.5.xxx.xxx:8020/test/seamless/2019-06-25").withParameters(parameters).withParameters(parameters);
        hdfsLines.print();

 
}

       <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-hadoop-compatibility_2.11</artifactId>
            <version>1.8.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.8.0</version>
            <!--<scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.12</artifactId>
            <!--<scope>provided</scope>-->
            <version>1.8.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.12</artifactId>
            <version>1.8.0</version>
            <!--<scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-filesystem_2.12</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-hadoop-fs</artifactId>
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-shaded-hadoop2</artifactId>
        </dependency>