大数据学习之HDFS 对应的简单java代码操作

学习大数据操作，Hadoop 旗下对hdfs操作入门级的java 编码：

import java.io.InputStream;
import java.net.URL;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsUrlStreamHandlerFactory;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;
/**
 * hdfs简单的操作
 * @author hardycheers
 *
 */
public class HadoopFirst {
	/**
	 * 读取hdfs上某一文件的内容
	 * @throws Exception
	 */
	@Test
	public void read() throws Exception {
		URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
		URL url = new URL("hdfs://192.168.163.128:9000/user/hadoop/file5.txt");
		InputStream is = url.openStream();
		IOUtils.copyBytes(is, System.out, 1024);
		IOUtils.closeStream(is);
	}
	/**
	 * 展示hdfs目录下的文件列表
	 * @throws Exception
	 */
	@Test
	public void listFiles() throws Exception {
		Configuration config = new Configuration();
		config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
		FileSystem fileSystem = FileSystem.get(config);
		System.out.println(fileSystem.getClass());
		Path path = new Path("/user/hadoop");
		RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(path, true);
		while (listFiles.hasNext()) {
			LocatedFileStatus locatedFileStatus = listFiles.next();
			System.out.println("路径： "+locatedFileStatus.getPath() + " 大小： "+ locatedFileStatus.getLen());
			
		}
	}
	/**
	 * 展示hdfs上的目录结构
	 * @throws Exception
	 */
	@Test
	public void listStatus() throws Exception {
		Configuration config = new Configuration();
		config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
		FileSystem fileSystem = FileSystem.get(config);
		Path path = new Path("/");
		FileStatus[] listStatus = fileSystem.listStatus(path);
		for (FileStatus fileStatus : listStatus) {
			System.out.println("目录： "+fileStatus.getPath());
		}
	}
	/**
	 * 创建目录 : 需要增加执行参数：-DHADOOP_USER_NAME=hadoop
	 * @throws Exception
	 */
	@Test
	public void testMkdir() throws Exception {
		Configuration config = new Configuration();
		config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
		FileSystem fileSystem = FileSystem.get(config);
		Path path = new Path("/bigdata");
		if(!fileSystem.exists(path)){
			boolean status = fileSystem.mkdirs(path);
			System.out.println("创建目录状态： "+status);
		}
	}
	/**
	 * 删除文件夹操作 需要增加执行参数：-DHADOOP_USER_NAME=hadoop
	 * @throws Exception
	 */
	@Test
	public void testDeleteFolder() throws Exception {
		Configuration config = new Configuration();
		config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
		FileSystem fileSystem = FileSystem.get(config);
		Path path = new Path("/bigdata");
		if(fileSystem.exists(path)){
			boolean status = fileSystem.delete(path, true);
			System.out.println("删除目录状态： "+status);
		}
	}
	/**
	 * 上传本地文件到HDFS操作 需要增加执行参数：-DHADOOP_USER_NAME=hadoop
	 * @throws Exception
	 */
	@Test
	public void uploadFile() throws Exception {
		Configuration config = new Configuration();
		config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
		FileSystem fileSystem = FileSystem.get(config);
		Path path = new Path("/user");
		Path source = new Path("D:/test/linuxcmd.txt");
		fileSystem.copyFromLocalFile(source,path);
		System.out.println("上传完成!!!");
	}
	/**
	 * 下载HDFS上的文件到本地操作 需要增加执行参数：-DHADOOP_USER_NAME=hadoop
	 * @throws Exception
	 */
	@Test
	public void downloadFile() throws Exception {
		Configuration config = new Configuration();
		config.set("fs.defaultFS", "hdfs://192.168.163.128:9000");
		FileSystem fileSystem = FileSystem.get(config);
		Path path = new Path("/user/linuxcmd.txt");
		Path dist_path = new Path("D:/test/123.txt");
		fileSystem.copyToLocalFile(path,dist_path);
		System.out.println("下载完成!!!");
	}
}

增加执行参数的位置