Java上传本地文件到HDFS

Java上传本地文件到HDFS的三种方法

原创已于 2022-02-09 17:26:08 修改 · 1.9k 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#hdfs #java #hadoop

于 2022-02-09 16:45:21 首次发布

大数据相关专栏收录该内容

59 篇文章

订阅专栏

本文介绍了使用Java编程将本地文件上传到HDFS的三种方法，包括使用FileSystem API的基本步骤，如配置HDFS地址、检查目标路径、创建目录及执行文件复制操作。

Java上传本地文件到HDFS

方法一

/**
 * @ Author zhangsf
 * @CreateTime 2022/2/9 - 4:33 PM
 */
package SparkWordCount;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class CopyFile {
    public static void main(String[] args) {
        Configuration conf = new Configuration();
        conf.addResource(new Path("/usr/local/Cellar/hadoop/3.3.1/libexec/etc/hadoop/core-site.xml"));
        String localDir = "/usr/local/Cellar/hadoop/3.3.1/sbin/httpfs.sh";
        String hdfsDir = "hdfs://localhost:9000/zhangvalue";
        try{
            Path localPath = new Path(localDir);
            Path hdfsPath = new Path(hdfsDir);
            FileSystem hdfs = FileSystem.get(conf);
            if(!hdfs.exists(hdfsPath)){
                hdfs.mkdirs(hdfsPath);
            }
            hdfs.copyFromLocalFile(localPath, hdfsPath);
        }catch(Exception e){
            e.printStackTrace();
        }
    }
}

在这里插入图片描述

方法二

/**
 * @ Author zhangsf
 * @CreateTime 2022/2/9 - 5:09 PM
 */
package SparkWordCount;

import java.io.IOException;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
//上传本地文件到HDFS
public class UploadToHDFS {
    public static void main(String[] args) throws IOException {
        //获取配置文件信息
        Configuration conf = new Configuration();
        conf.addResource(new Path("/usr/local/Cellar/hadoop/3.3.1/libexec/etc/hadoop/core-site.xml"));
        //获取文件系统
        FileSystem hdfs = FileSystem.get(conf);
        //文件名称
        Path src = new Path("/usr/local/Cellar/hadoop/3.3.1/libexec/etc/hadoop/core-site.xml");
        Path dst = new Path("hdfs://localhost:9000/zhangvalue");
        if(!hdfs.exists(dst)){
            hdfs.mkdirs(dst);
        }
        hdfs.copyFromLocalFile(src, dst);
        System.out.println("Upload to " + conf.get("fs.default.name"));
        FileStatus files[] = hdfs.listStatus(dst);
        for(FileStatus file:files){
            System.out.println(file.getPath());
        }
    }
}

在这里插入图片描述

方法三

/**
 * @ Author zhangsf
 * @CreateTime 2022/2/9 - 5:20 PM
 */
package SparkWordCount;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
public class UploadSingle {
    public static void main(String[] args) throws URISyntaxException, IOException{
        Configuration conf = new Configuration();
        URI uri = new URI("hdfs://localhost:9000");
        FileSystem fs = FileSystem.get(uri,conf);
        Path resP = new Path("/usr/local/Cellar/hadoop/3.3.1/libexec/etc/hadoop/yarn-env.sh");
        Path destP = new Path("/zhangvalue");
        if(!fs.exists(destP)){
            fs.mkdirs(destP);
        }
        fs.copyFromLocalFile(resP, destP);
        fs.close();
    }
}