package com.cn.demo01; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FsUrlStreamHandlerFactory; import org.junit.Test; import java.io.*; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; public class HdfsOperate { //通过URL来获取,不常用了解 @Test public void downHdfsFiles() throws IOException { //注册一个驱动,访问hdfs文件系统 URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory()); String url = "hdfs://node01:8020/profile"; InputStream inputStream = new URL(url).openStream(); OutputStream outputStream = new FileOutputStream(new File("D:\\hello.txt")); IOUtils.copy(inputStream, outputStream); IOUtils.closeQuietly(inputStream); IOUtils.closeQuietly(outputStream); } //通过FileSystem类来获取,了解源码 不加参数默认本地文件系统的原因 @Test public void getFilesystem() throws Exception { Configuration configuration = new Configuration(); configuration.set("fs.defaultFS","hdfs://node01:8020"); FileSystem fileSystem = FileSystem.get(configuration); System.out.println(fileSystem.toString()); fileSystem.close(); } @Test public void getFilesystem2() throws URISyntaxException, IOException { Configuration configuration = new Configuration(); FileSystem fileSystem = FileSystem.get(new URI("hdfs://node01:8020"),configuration); System.out.println(fileSystem.toString()); fileSystem.close(); } @Test public void getFilesystem3() throws IOException { Configuration configuration = new Configuration(); configuration.set("fs.defaultFS","hdfs://node01:8020"); FileSystem fileSystem = FileSystem.newInstance(configuration); System.out.println(fileSystem.toString()); fileSystem.close(); } @Test public void getFilesystem4() throws IOException, URISyntaxException { Configuration configuration = new Configuration(); FileSystem fileSystem = FileSystem.newInstance(new URI("hdfs://node01:8020"),configuration); System.out.println(fileSystem.toString()); fileSystem.close(); } }