hadoop-API视频配套操作源码

本文章基于尚硅谷Hadoop 3.x视频进行总结，仅作为学习交流使用        视频链接如下：

46_尚硅谷_Hadoop_HDFS_API环境准备_哔哩哔哩_bilibili

目录

1.API创建文件夹

2.上传

3.参数优先级

4.下载

5.删除

关键代码

整体代码

6.文件的移动和更名

关键代码

整体代码

7.获取文件详情信息

关键代码

整体代码

8.判断是文件还是目录

关键代码

---

1.API创建文件夹

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsClient {
    private FileSystem fs;
    
    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://hadoop102:8020");
        Configuration configuration = new Configuration();
        String user = "atguigu";
        fs = FileSystem.get(uri, configuration, user);
    }
    
    @After
    public void close() throws IOException {
        fs.close();
    }
    
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/xiyou/huaguoshan1"));
    }
}

2.上传

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsClient {
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://hadoop102:8020");
        Configuration configuration = new Configuration();
        String user = "atguigu";
        fs = FileSystem.get(uri, configuration, user);
    }

    //关闭资源
    @After
    public void close() throws IOException {
        fs.close();
    }

    //创建目录
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/xiyou/huaguoshan1"));
    }

    //上传
    @Test
    public void testPut() throws IOException {
        //参数一：表示删除原数据 参数二：是否允许覆盖 参数三：原数据路径 参数四：目的地路径
        fs.copyFromLocalFile(true,true,new Path("D:\\hadoop(1)\\sunwukong.txt"),new Path("hdfs://hadoop102/xiyou/huaguoshan"));
    }
}

3.参数优先级

/*
 *参数优先级
 * hdfs-default.xml => hdfs-site.xml =>在项目资源目录下的配置文件 =>代码里面的配置
 */ 

hdfs-site.xml配置文件

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsClient {
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://hadoop102:8020");
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication","2");

        String user = "atguigu";
        fs = FileSystem.get(uri, configuration, user);
    }

    //关闭资源
    @After
    public void close() throws IOException {
        fs.close();
    }

    //创建目录
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/xiyou/huaguoshan1"));
    }

    //上传
    /*
     *参数优先级
     * hdfs-default.xml => hdfs-site.xml =>在项目资源目录下的配置文件 =>代码里面的配置
     */
    @Test
    public void testPut() throws IOException {
        //参数一：表示删除原数据 参数二：是否允许覆盖 参数三：原数据路径 参数四：目的地路径
        fs.copyFromLocalFile(false,true,new Path("D:\\hadoop(1)\\sunwukong.txt"),new Path("hdfs://hadoop102/xiyou/huaguoshan"));
    }
}

4.下载

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsClient {
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://hadoop102:8020");
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication","2");

        String user = "atguigu";
        fs = FileSystem.get(uri, configuration, user);
    }

    //关闭资源
    @After
    public void close() throws IOException {
        fs.close();
    }

    //创建目录
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/xiyou/huaguoshan1"));
    }

    //上传
    /*
     *参数优先级
     * hdfs-default.xml => hdfs-site.xml =>在项目资源目录下的配置文件 =>代码里面的配置
     */
    @Test
    public void testPut() throws IOException {
        //参数一：表示删除原数据 参数二：是否允许覆盖 参数三：原数据路径 参数四：目的地路径
        fs.copyFromLocalFile(false,true,new Path("D:\\hadoop(1)\\sunwukong.txt"),new Path("hdfs://hadoop102/xiyou/huaguoshan"));
    }

    //下载
    @Test
    public void testGet() throws IOException {
        //参数1：源文件是否删除 参数2：原文件路径HDFS 参数三：目标地址路径Windows 参数4：
        fs.copyToLocalFile(true,new Path("hdfs://hadoop102/xiyou/huaguoshan"),new Path("D:\\hadoop(1)"),true);
    }
}

5.删除

关键代码

    //删除
    @Test
    public void testRm() throws IOException {
        //参数:1.要删除的路径 2.是否递归删除
        //删除文件
        //fs.delete(new Path("/jdk-8u341-linux-x64.tar.gz"),false);

        //删除空目录
        //fs.delete(new Path("/xiyou1"),false);

        //删除非空目录
        fs.delete(new Path("/jinguo"),true);
    }

整体代码

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsClient {
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://hadoop102:8020");
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication","2");

        String user = "atguigu";
        fs = FileSystem.get(uri, configuration, user);
    }

    //关闭资源
    @After
    public void close() throws IOException {
        fs.close();
    }

    //创建目录
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/xiyou/huaguoshan1"));
    }

    //上传
    /*
     *参数优先级
     * hdfs-default.xml => hdfs-site.xml =>在项目资源目录下的配置文件 =>代码里面的配置
     */
    @Test
    public void testPut() throws IOException {
        //参数一：表示删除原数据 参数二：是否允许覆盖 参数三：原数据路径 参数四：目的地路径
        fs.copyFromLocalFile(false,true,new Path("D:\\hadoop(1)\\sunwukong.txt"),new Path("hdfs://hadoop102/xiyou/huaguoshan"));
    }

    //下载
    @Test
    public void testGet() throws IOException {
        //参数1：源文件是否删除 参数2：原文件路径HDFS 参数三：目标地址路径Windows 参数4：
        fs.copyToLocalFile(true,new Path("hdfs://hadoop102/xiyou/huaguoshan"),new Path("D:\\hadoop(1)"),true);
    }

    //删除
    @Test
    public void testRm() throws IOException {
        //参数:1.要删除的路径 2.是否递归删除
        //删除文件
        //fs.delete(new Path("/jdk-8u341-linux-x64.tar.gz"),false);

        //删除空目录
        //fs.delete(new Path("/xiyou1"),false);

        //删除非空目录
        fs.delete(new Path("/jinguo"),true);
    }
}

6.文件的移动和更名

关键代码

    //文件的更名和移动
    @Test
    public void testmv() throws IOException {
        //参数:1.原文件路径 2.目标文件路径
        //对文件名称的修改
        //fs.rename(new Path("/wcinput/word.txt"),new Path("/wcinput/ss.txt"));

        //文件的移动和更名
        //fs.rename(new Path("/wcinput/ss.txt"),new Path("/cls.txt"));
        
        //目录的更名
        fs.rename(new Path("/wcinput"),new Path("/wcinput1"));
    }

整体代码

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class HdfsClient {
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://hadoop102:8020");
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication","2");

        String user = "atguigu";
        fs = FileSystem.get(uri, configuration, user);
    }

    //关闭资源
    @After
    public void close() throws IOException {
        fs.close();
    }

    //创建目录
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/xiyou/huaguoshan1"));
    }

    //上传
    /*
     *参数优先级
     * hdfs-default.xml => hdfs-site.xml =>在项目资源目录下的配置文件 =>代码里面的配置
     */
    @Test
    public void testPut() throws IOException {
        //参数一：表示删除原数据 参数二：是否允许覆盖 参数三：原数据路径 参数四：目的地路径
        fs.copyFromLocalFile(false,true,new Path("D:\\hadoop(1)\\sunwukong.txt"),new Path("hdfs://hadoop102/xiyou/huaguoshan"));
    }

    //下载
    @Test
    public void testGet() throws IOException {
        //参数1：源文件是否删除 参数2：原文件路径HDFS 参数三：目标地址路径Windows 参数4：
        fs.copyToLocalFile(true,new Path("hdfs://hadoop102/xiyou/huaguoshan"),new Path("D:\\hadoop(1)"),true);
    }

    //删除
    @Test
    public void testRm() throws IOException {
        //参数:1.要删除的路径 2.是否递归删除
        //删除文件
        //fs.delete(new Path("/jdk-8u341-linux-x64.tar.gz"),false);

        //删除空目录
        //fs.delete(new Path("/xiyou1"),false);

        //删除非空目录
        fs.delete(new Path("/jinguo"),true);
    }

    //文件的更名和移动
    @Test
    public void testmv() throws IOException {
        //参数:1.原文件路径 2.目标文件路径
        //对文件名称的修改
        //fs.rename(new Path("/wcinput/word.txt"),new Path("/wcinput/ss.txt"));

        //文件的移动和更名
        //fs.rename(new Path("/wcinput/ss.txt"),new Path("/cls.txt"));

        //目录的更名
        fs.rename(new Path("/wcinput"),new Path("/wcinput1"));
    }
}

7.获取文件详情信息

关键代码

    //获取文件详情信息
    @Test
    public void fileDetail() throws IOException {
        //获取所有文件信息
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);

        //遍历信息
        while (listFiles.hasNext()) {
            LocatedFileStatus fileStatus = listFiles.next();

            System.out.println("============" + fileStatus.getPath() + "============");
            System.out.println(fileStatus.getPermission());
            System.out.println(fileStatus.getOwner());
            System.out.println(fileStatus.getGroup());
            System.out.println(fileStatus.getLen());
            System.out.println(fileStatus.getModificationTime());
            System.out.println(fileStatus.getReplication());
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPath().getName());

            //获取块信息
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            System.out.println(Arrays.toString(blockLocations));
        }
    }

整体代码

package com.atguigu.hdfs;

import com.sun.xml.internal.fastinfoset.tools.FI_DOM_Or_XML_DOM_SAX_SAXEvent;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;

public class HdfsClient {
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://hadoop102:8020");
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication", "2");

        String user = "atguigu";
        fs = FileSystem.get(uri, configuration, user);
    }

    //关闭资源
    @After
    public void close() throws IOException {
        fs.close();
    }

    //创建目录
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/xiyou/huaguoshan1"));
    }

    //上传
    /*
     *参数优先级
     * hdfs-default.xml => hdfs-site.xml =>在项目资源目录下的配置文件 =>代码里面的配置
     */
    @Test
    public void testPut() throws IOException {
        //参数一：表示删除原数据 参数二：是否允许覆盖 参数三：原数据路径 参数四：目的地路径
        fs.copyFromLocalFile(false, true, new Path("D:\\hadoop(1)\\sunwukong.txt"), new Path("hdfs://hadoop102/xiyou/huaguoshan"));
    }

    //下载
    @Test
    public void testGet() throws IOException {
        //参数1：源文件是否删除 参数2：原文件路径HDFS 参数三：目标地址路径Windows 参数4：
        fs.copyToLocalFile(true, new Path("hdfs://hadoop102/xiyou/huaguoshan"), new Path("D:\\hadoop(1)"), true);
    }

    //删除
    @Test
    public void testRm() throws IOException {
        //参数:1.要删除的路径 2.是否递归删除
        //删除文件
        //fs.delete(new Path("/jdk-8u341-linux-x64.tar.gz"),false);

        //删除空目录
        //fs.delete(new Path("/xiyou1"),false);

        //删除非空目录
        fs.delete(new Path("/jinguo"), true);
    }

    //文件的更名和移动
    @Test
    public void testmv() throws IOException {
        //参数:1.原文件路径 2.目标文件路径
        //对文件名称的修改
        //fs.rename(new Path("/wcinput/word.txt"),new Path("/wcinput/ss.txt"));

        //文件的移动和更名
        //fs.rename(new Path("/wcinput/ss.txt"),new Path("/cls.txt"));

        //目录的更名
        fs.rename(new Path("/wcinput"), new Path("/wcinput1"));
    }

    //获取文件详情信息
    @Test
    public void fileDetail() throws IOException {
        //获取所有文件信息
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);

        //遍历信息
        while (listFiles.hasNext()) {
            LocatedFileStatus fileStatus = listFiles.next();

            System.out.println("============" + fileStatus.getPath() + "============");
            System.out.println(fileStatus.getPermission());
            System.out.println(fileStatus.getOwner());
            System.out.println(fileStatus.getGroup());
            System.out.println(fileStatus.getLen());
            System.out.println(fileStatus.getModificationTime());
            System.out.println(fileStatus.getReplication());
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPath().getName());

            //获取块信息
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            System.out.println(Arrays.toString(blockLocations));
        }
    }
}

8.判断是文件还是目录

关键代码

    //判断是文件夹还是文件
    @Test
    public void testFile() throws IOException {
        FileStatus[] listStatus = fs.listStatus(new Path("/"));

        for (FileStatus status : listStatus) {

            if (status.isFile()) {
                System.out.println("文件:" + status.getPath().getName());
            } else {
                System.out.println("目录:" + status.getPath().getName());
            }
        }
    }

整体代码

package com.atguigu.hdfs;

import com.sun.xml.internal.fastinfoset.tools.FI_DOM_Or_XML_DOM_SAX_SAXEvent;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;

public class HdfsClient {
    private FileSystem fs;

    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        URI uri = new URI("hdfs://hadoop102:8020");
        Configuration configuration = new Configuration();
        configuration.set("dfs.replication", "2");

        String user = "atguigu";
        fs = FileSystem.get(uri, configuration, user);
    }

    //关闭资源
    @After
    public void close() throws IOException {
        fs.close();
    }

    //创建目录
    @Test
    public void testmkdir() throws URISyntaxException, IOException, InterruptedException {
        fs.mkdirs(new Path("/xiyou/huaguoshan1"));
    }

    //上传
    /*
     *参数优先级
     * hdfs-default.xml => hdfs-site.xml =>在项目资源目录下的配置文件 =>代码里面的配置
     */
    @Test
    public void testPut() throws IOException {
        //参数一：表示删除原数据 参数二：是否允许覆盖 参数三：原数据路径 参数四：目的地路径
        fs.copyFromLocalFile(false, true, new Path("D:\\hadoop(1)\\sunwukong.txt"), new Path("hdfs://hadoop102/xiyou/huaguoshan"));
    }

    //下载
    @Test
    public void testGet() throws IOException {
        //参数1：源文件是否删除 参数2：原文件路径HDFS 参数三：目标地址路径Windows 参数4：
        fs.copyToLocalFile(true, new Path("hdfs://hadoop102/xiyou/huaguoshan"), new Path("D:\\hadoop(1)"), true);
    }

    //删除
    @Test
    public void testRm() throws IOException {
        //参数:1.要删除的路径 2.是否递归删除
        //删除文件
        //fs.delete(new Path("/jdk-8u341-linux-x64.tar.gz"),false);

        //删除空目录
        //fs.delete(new Path("/xiyou1"),false);

        //删除非空目录
        fs.delete(new Path("/jinguo"), true);
    }

    //文件的更名和移动
    @Test
    public void testmv() throws IOException {
        //参数:1.原文件路径 2.目标文件路径
        //对文件名称的修改
        //fs.rename(new Path("/wcinput/word.txt"),new Path("/wcinput/ss.txt"));

        //文件的移动和更名
        //fs.rename(new Path("/wcinput/ss.txt"),new Path("/cls.txt"));

        //目录的更名
        fs.rename(new Path("/wcinput"), new Path("/wcinput1"));
    }

    //获取文件详情信息
    @Test
    public void fileDetail() throws IOException {
        //获取所有文件信息
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);

        //遍历信息
        while (listFiles.hasNext()) {
            LocatedFileStatus fileStatus = listFiles.next();

            System.out.println("============" + fileStatus.getPath() + "============");
            System.out.println(fileStatus.getPermission());
            System.out.println(fileStatus.getOwner());
            System.out.println(fileStatus.getGroup());
            System.out.println(fileStatus.getLen());
            System.out.println(fileStatus.getModificationTime());
            System.out.println(fileStatus.getReplication());
            System.out.println(fileStatus.getBlockSize());
            System.out.println(fileStatus.getPath().getName());

            //获取块信息
            BlockLocation[] blockLocations = fileStatus.getBlockLocations();
            System.out.println(Arrays.toString(blockLocations));
        }
    }

    //判断是文件夹还是文件
    @Test
    public void testFile() throws IOException {
        FileStatus[] listStatus = fs.listStatus(new Path("/"));

        for (FileStatus status : listStatus) {

            if (status.isFile()) {
                System.out.println("文件:" + status.getPath().getName());
            } else {
                System.out.println("目录:" + status.getPath().getName());
            }
        }
    }
}