hadoop上传和下载文件

最新推荐文章于 2025-04-13 12:41:41 发布

原创最新推荐文章于 2025-04-13 12:41:41 发布 · 2.3k 阅读

1 ·

CC 4.0 BY-SA版权

本文介绍了如何在Hadoop系统中进行文件的下载操作，特别是将文件从Hadoop下载到Windows环境的步骤。同时，提供了一个相关干货网址供读者深入学习。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

上传windows文件到hadoop：

package test;

import java.io.BufferedInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapreduce.Job;



public class updata {

	public static void main(String[] args) throws Exception{
		try {
			String INPUT_PATH = args[0];
			String OUT_PATH = args[1];
			//in对应的是本地文件系统的目录
			InputStream in = new BufferedInputStream(new FileInputStream(INPUT_PATH));
			Configuration conf = new Configuration();
			final Job job = new Job(conf,updata.class.getSimpleName());
			//打包运行必须执行的秘密方法;
			job.setJarByClass(updata.class);
			//获得hadoop系统的连接 
			getfile(OUT_PATH, in, conf);
			System.out.println("success");
			} catch (Exception e) {
			System.out.println(e.toString());
			}
	}

	private static void getfile(String dst, InputStream in, Configuration conf) throws IOException {
		FileSystem fs = FileSystem.get(URI.create(dst),conf);
		//out对应的是Hadoop文件系统中的目录
		OutputStream out = fs.create(new Path(dst));
		IOUtils.copyBytes(in, out, 4096,true);//4096是4k字节
	}

}

从hadoop中下载文件到windows：

package test;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URL;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapreduce.Job;

public class download {
	// 静态块，设置hdfs协议
	static  {
        URL. setURLStreamHandlerFactory ( new  FsUrlStreamHandlerFactory());
	}
	public static void main(String[] args) throws Exception{
		
		try {
			//第一个参数传递进来的是Hadoop文件系统中的某个文件的URI
			String INPUT_PATH =  "hdfs://LS-PC:9000/out/3.jpg";
			String OUT_PATH = "C://new/3.jpg";
			Configuration conf = new Configuration();
			//conf.set("Hadoop.job.ugi", "hadoop-user,hadoop-user");
			//打包运行必须执行的秘密方法;
			final Job job = new Job(conf,updata.class.getSimpleName());	
			job.setJarByClass(updata.class);
			
			download(INPUT_PATH, OUT_PATH, conf);
			
			System.out.println("Download success");
			} catch (Exception e) {
			System.out.println(e.toString());
			}
	}
	private static void download(String INPUT_PATH, String OUT_PATH, Configuration conf)
			throws IOException, FileNotFoundException {
		//FileSystem是用户操作HDFS的核心类，它获得URI对应的HDFS文件系统 
		FileSystem fs = FileSystem.get(URI.create(INPUT_PATH), conf);
		 //让FileSystem打开一个uri对应的FSDataInputStream文件输入流，读取这个文件  
		FSDataInputStream in=null;
		in=fs.open(new Path(INPUT_PATH));
		//out对应的是Hadoop文件系统中的目录
		OutputStream out = new BufferedOutputStream(new FileOutputStream(OUT_PATH));			
		IOUtils.copyBytes(in, out, 4096,true);//4096是4k字节
	}


}