七 HDFS客户端操作

1. HDFS客户端环境准备

(1) 下载hadoop client 包并解压到一个无中文无空格的目录下 :
链接：https://pan.baidu.com/s/1aYt3IHMYTO10XLzURkODlg
提取码：3e4m
(以上包是win10 的其它系统需要自己下载)

(2) 配置HADOOP_HOME环境变量


(3) 创建一个Maven工程HdfsClientDemo
(4) 导入相应的依赖坐标+日志
pom.xml文件添加:

<dependencies>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>RELEASE</version>
		</dependency>
		<dependency>
			<groupId>org.apache.logging.log4j</groupId>
			<artifactId>log4j-core</artifactId>
			<version>2.8.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.7.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.7.2</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>2.7.2</version>
		</dependency>
		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.8</version>
			<scope>system</scope>
			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
		</dependency>
</dependencies>

注意：如果Eclipse/Idea打印不出日志，在控制台上只显示

1.log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).  
2.log4j:WARN Please initialize the log4j system properly.  
3.log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.

需要在项目的src/main/resources目录下，新建一个文件，命名为“log4j.properties”，在文件中填入

log4j.rootLogger=INFO, stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.logfile=org.apache.log4j.FileAppender
log4j.appender.logfile.File=target/spring.log
log4j.appender.logfile.layout=org.apache.log4j.PatternLayout
log4j.appender.logfile.layout.ConversionPattern=%d %p [%c] - %m%n

2. 在idea 上连接和操作集群

在idea 上操作集群一般步骤为
(1) 设置配置参数
(2) 获取文件系统和集群建立连接
(3) 调用API对集群进行操作
(4) 关闭资源

如下:

package atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsClient {
    FileSystem fs;

    //打印文件夹中的内容信息
    @Test
    public void testShowFiles() throws IOException {
        RemoteIterator<LocatedFileStatus> status = fs.listFiles(new Path("/"), true);
        while (status.hasNext()) {
            LocatedFileStatus next = status.next();
            System.out.println("FileName: " + next.getPath().getName());
            System.out.println("长度: " + next.getLen());
            System.out.println("权限: " + next.getPermission());
            System.out.println("分组: " + next.getGroup());
            BlockLocation[] bls = next.getBlockLocations();
            for (BlockLocation bl : bls) {
                String[] hosts = bl.getHosts();
                for (String host : hosts) {
                    System.out.println("host: " + host);
                }
            }

            System.out.println();

        }
    }

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        Configuration configuration = new Configuration(); //设置配置
        //configuration.set("dfs.replication", "1");
        fs = FileSystem.get(new URI("hdfs://xiaoai01:9000"), configuration, "andy");
        //获取连接
        System.out.println("init OK!");
    }

    //下载文件
    @Test
    public void testGet() throws IOException {
        fs.copyToLocalFile(false, new Path("/andy/input/test.txt"), new Path("D:\\"));
    }

    //创建文件夹
    @Test
    public void testMkdirs() throws IOException {
        fs.mkdirs(new Path("/1108/daxian/banzhang"));
    }


    //上传文件
    @Test
    public void testPut() throws IOException {
        fs.copyFromLocalFile(false, new Path("D:\\README.txt"), new Path("/andy/"));
    }


    @After
    public void uninit() throws IOException {

        fs.close();//关闭资源
        System.out.println("uninit OK!");
    }

}