java HDFS文件操作（增删改读）

最新推荐文章于 2025-02-25 14:43:31 发布

原创最新推荐文章于 2025-02-25 14:43:31 发布 · 945 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#hdfs #文件 #操作 #增删改读 #java

linux 同时被 3 个专栏收录

3 篇文章

订阅专栏

hadoop

1 篇文章

订阅专栏

hdfs

1 篇文章

订阅专栏

本文介绍了一套针对Hadoop分布式文件系统(HDFS)的操作工具类，包括文件上传、下载、创建、删除及读写等功能，并提供了核心配置及依赖的jar包清单。

<span style="background-color: rgb(255, 255, 255);">1、 代码                                                                                                                                         </span>

package com.ctrip.bi.uss.util;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;

import com.ctrip.bi.uss.mgr.LogMgr;

public class HdfsUtils {
	private static Logger log = LogMgr.getUss_log();

	public static void main(String[] args) throws Exception {
		copyHDFS2Local("hdfs://ns/tmp/data/index/2014-09-05/",
				"/opt/ctrip/web/work/data/ws/");
	}

	// 上传文件到hdfs
	public static void uploadLocalFileHDFS(String s, String d)
			throws IOException {
		Configuration config = new Configuration();
		FileSystem hdfs = FileSystem.get(config);
		Path src = new Path(s);
		Path dst = new Path(d);
		hdfs.copyFromLocalFile(src, dst);
		hdfs.close();
	}

	// 拷贝整个hdfs下的文件夹到本地
	public static boolean copyHDFS2Local(String hdfsFilePath,
			String localFilePath) throws Exception {
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(conf);
		Path localPath = new Path(localFilePath);
		Path hdfsPath = new Path(hdfsFilePath);
		try {
			fs.copyToLocalFile(hdfsPath, localPath);
			return true;
		} catch (Exception e) {
			log.error(e.getMessage() + " " + e.getCause());
			return false;
		}

	}

	// 在hdfs中创建文件
	public static void createNewHDFSFile(String toCreateFilePath, String content)
			throws IOException {
		Configuration config = new Configuration();
		FileSystem hdfs = FileSystem.get(config);
		FSDataOutputStream os = hdfs.create(new Path(toCreateFilePath));
		os.write(content.getBytes("UTF-8"));
		os.close();
		hdfs.close();
	}

	// 删除hdfs中文件
	public static boolean deleteHDFSFile(String dst) throws IOException {
		Configuration config = new Configuration();
		FileSystem hdfs = FileSystem.get(config);
		Path path = new Path(dst);
		boolean isDeleted = hdfs.delete(path);
		hdfs.close();

		return isDeleted;
	}

	// 读取hdfs文件内容
	public static byte[] readHDFSFile(String dst) throws Exception {
<span style="white-space:pre">		</span>Configuration conf = new Configuration();
<span style="white-space:pre">		</span>FileSystem fs = FileSystem.get(conf);


<span style="white-space:pre">		</span>// check if the file exists
<span style="white-space:pre">		</span>Path path = new Path(dst);
<span style="white-space:pre">		</span>if (fs.exists(path)) {
<span style="white-space:pre">			</span>FSDataInputStream is = fs.open(path);
<span style="white-space:pre">			</span>// get the file info to create the buffer
<span style="white-space:pre">			</span>FileStatus stat = fs.getFileStatus(path);
<span style="white-space:pre">			</span>// create the buffer
<span style="white-space:pre">			</span>byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat
<span style="white-space:pre">					</span>.getLen()))];
<span style="white-space:pre">			</span>is.readFully(0, buffer);
<span style="white-space:pre">			</span>is.close();
<span style="white-space:pre">			</span>fs.close();


<span style="white-space:pre">			</span>return buffer;
<span style="white-space:pre">		</span>} else {
<span style="white-space:pre">			</span>throw new Exception("the file is not found .");
<span style="white-space:pre">		</span>}
<span style="white-space:pre">	</span>}

	// 新建目录
	public static void mkdir(String dir) throws IOException {
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(conf);
		fs.mkdirs(new Path(dir));
		fs.close();
	}

	// 删除目录
	public static void deleteDir(String dir) throws IOException {
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(conf);
		fs.delete(new Path(dir));
		fs.close();
	}

	// 列出所有的文件
	public static void listAll(String src, String local) throws IOException {
		Configuration conf = new Configuration();
		FileSystem fs = FileSystem.get(conf);
		FileStatus[] stats = fs.listStatus(new Path(src));
		for (int i = 0; i < stats.length; ++i) {
			if (stats[i].isFile()) {
				System.out.println(stats[i].getPath().toString());
			} else if (stats[i].isDirectory()) {
				System.out.println(stats[i].getPath().toString());
			} else if (stats[i].isSymlink()) {
				System.out.println(stats[i].getPath().toString());
			}
		}
		fs.close();
	}

}

2、配置文件 core-site.xml （此文件需要放在类的根目录下）

<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://ns</value>
	</property>
	<property>
		<name>fs.file.impl</name>
		<value>org.apache.hadoop.fs.LocalFileSystem</value>
		<description>The FileSystem for file: uris.</description>
	</property>

	<property>
		<name>fs.hdfs.impl</name>
		<value>org.apache.hadoop.hdfs.DistributedFileSystem</value>
		<description>The FileSystem for hdfs: uris.</description>
	</property>
	<property>
		<name>dfs.nameservices</name>
		<value>ns</value>
	</property>
	<property>
		<name>dfs.ha.namenodes.ns</name>
		<value>SVR2368HP360,SVR2369HP360</value>
	</property>
	<property>
		<name>dfs.namenode.rpc-address.ns.SVR2368HP360</name>
		<value>SVR2368HP360.hadoop.test.sh.ctriptravel.com:54310</value>
	</property>
	<property>
		<name>dfs.namenode.rpc-address.ns.SVR2369HP360</name>
		<value>SVR2369HP360.hadoop.test.sh.ctriptravel.com:54310</value>
	</property>
	<property>
		<name>dfs.client.failover.proxy.provider.ns</name>
		<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
		</value>
	</property>

</configuration>

以上的配置信息每台机子各异，因此需要相应改动而不能直接copy

3、必须的jar包

commons-cli-1.2.jar

commons-collections-3.1.jar

commons-configuration-1.6.jar

commons-io-2.1.jar

commons-lang-2.5.jar

commons-logging-1.1.1.jar

guava-11.0.2.jar

hadoop-auth-2.0.0-cdh4.2.1.jar

hadoop-common-2.0.0-cdh4.6.0.jar

hadoop-core.jar