Java API 操作 HDFS

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.net.URI;
import java.util.Date;

/**
 * 实验
 */
public class experiment {

    private FileSystem fileSystem;

    @Before
    public void init() throws Exception {
        URI uri = new URI("hdfs://hadoop102:8020");
        fileSystem = FileSystem.get(uri, new Configuration());
    }

    @After
    public void close() throws IOException {
        fileSystem.close();
    }

    /**
     * 上传文件：
     *      如果文件不存在，就直接上传；
     *      如果文件存在，由用户决定是覆盖原文件还是追加到原文件末尾
     */
    @Test
    public void put() throws IOException {
        Path local = new Path("D:\\data\\a.txt");
        Path dfs = new Path("/input/a.txt");
        // 判断文件是否已经存在
        boolean exists = fileSystem.exists(dfs);
        if (exists) {
            // 已经存在，由用户决定是否覆盖
//            Scanner scanner = new Scanner(System.in);
//            String s = scanner.nextLine();
            String s = "true";
            if ("true".equals(s)) {
                // 覆盖
                fileSystem.copyFromLocalFile(false, true, local, dfs);
            } else {
                FileInputStream fileInputStream = new FileInputStream("D:\\data\\a.txt");
                BufferedReader reader = new BufferedReader(new InputStreamReader(fileInputStream));
                // 追加
                FSDataOutputStream outputStream = fileSystem.append(dfs);
                String line = reader.readLine();
                outputStream.writeUTF(line);
                // close stream
                fileInputStream.close();
                outputStream.close();
            }
        } else {
            // 不存在：直接上传
            fileSystem.copyFromLocalFile(local, dfs);
        }
    }

    /**
     * 下载文件到本地，如果本地已存在，则对文件重命名
     */
    @Test
    public void get() throws IOException {
        // 要下载的文件
        Path dfs = new Path("/input/a.txt");
        // 判断本地是否存在
        File file = new File("D:\\data\\a.txt");
        boolean exists = file.exists();
        if (exists) {
            // 已存在，重命名
            System.out.println("a.txt already exist!");
            Path local = new Path("D:\\data\\a_" + new Date().getTime() + ".txt");
            fileSystem.copyToLocalFile(dfs, local);
        } else {
            // 不存在
            Path local = new Path("D:\\data\\a.txt");
            fileSystem.copyToLocalFile(dfs, local);
        }
    }

    /**
     * 输出hdfs中的文件内容到控制台
     */
    @Test
    public void cat() throws IOException {
        FSDataInputStream inputStream = fileSystem.open(new Path("/in/hello.txt"));
        IOUtils.copyBytes(inputStream, System.out, new Configuration());
    }

    /**
     * 显示指定文件的权限、大小、创建时间、路径等信息
     */
    @Test
    public void fileInfo() throws IOException {
        FileStatus fileStatus = fileSystem.getFileStatus(new Path("/input/a.txt"));
        printInfo(fileStatus);
    }

    // 辅助打印文件信息
    private void printInfo(FileStatus fileStatus) {
        System.out.println("***********************************");
        System.out.println(fileStatus.getPath());
        System.out.println(fileStatus.getOwner());
        System.out.println(fileStatus.getPermission());
        System.out.println(fileStatus.getLen());
        System.out.println(fileStatus.getBlockSize() / 1024 / 1024 + "M");
        System.out.println(fileStatus.getGroup());
        System.out.println(fileStatus.getReplication());
        System.out.println(fileStatus.getAccessTime());
        System.out.println(fileStatus.getModificationTime());
    }

    /**
     * 显示目录下所有文件的信息
     */
    @Test
    public void dirInfo() throws IOException {
        // recursive：是否递归遍历
        RemoteIterator<LocatedFileStatus> iterator = fileSystem.listFiles(new Path("/input"), true);
        while (iterator.hasNext()) {
            LocatedFileStatus fileStatus = iterator.next();
            printInfo(fileStatus);
        }
    }

    /**
     * 创建目录
     */
    @Test
    public void createDir() throws IOException {
        Path path = new Path("/dir");
        boolean exists = fileSystem.exists(path);
        if (exists) {
            System.out.println("dir already exist!");
        } else {
            boolean mkdirs = fileSystem.mkdirs(path);
            if (mkdirs) {
                System.out.println("make success");
            } else {
                System.out.println("make fail");
            }
        }
    }

    /**
     * 删除目录
     */
    @Test
    public void deleteDir() throws IOException {
        Path path = new Path("/dir");
        boolean exists = fileSystem.exists(path);
        if (exists) {
            // recursive：是否递归删除
            fileSystem.delete(path, true);
        } else {
            System.out.println("dir not exist");
        }
    }

    /**
     * 创建文件
     */
    @Test
    public void createFile() throws IOException {
        Path path = new Path("/dir/a.txt");
        boolean exists = fileSystem.exists(path);
        if (exists) {
            System.out.println("file already exist!");
        } else {
            boolean newFile = fileSystem.createNewFile(path);
            if (newFile) {
                System.out.println("success");
            }
        }
    }

    /**
     * 删除文件
     */
    @Test
    public void deleteFile() throws IOException {
        Path path = new Path("/dir/a.txt");
        boolean exists = fileSystem.exists(path);
        if (exists) {
            boolean delete = fileSystem.delete(path, false);
        } else {
            System.out.println("file not exist");
        }
    }

    /**
     * 追加内容到文件的开头或末尾
     */
    @Test
    public void append() throws IOException {
        Path path = new Path("/dir/a.txt");
        boolean tail = false;
        if (tail) {
            // 追加内容到文件的末尾
            FSDataOutputStream outputStream = fileSystem.append(path);
            IOUtils.copyBytes(new ByteArrayInputStream("-tail content".getBytes()), outputStream, new Configuration());
        } else {
            // 追加到文件的开头
            Path tmp = new Path("D:\\data\\tmp.txt");
            // 先把dfs的文件move到local
            fileSystem.moveToLocalFile(path, tmp);
            // 新建该文件
            createFile();
            // 追加用户的输入
            FSDataOutputStream outputStream = fileSystem.append(path);
            IOUtils.copyBytes(new ByteArrayInputStream("-head content\n".getBytes()), outputStream, new Configuration());
            // 追加原先的内容
            FileInputStream fileInputStream = new FileInputStream("D:\\data\\tmp.txt");
            outputStream = fileSystem.append(path);
            IOUtils.copyBytes(fileInputStream, outputStream, new Configuration());
        }
    }

    /**
     * 移动文件
     */
    @Test
    public void move() throws IOException {
        fileSystem.rename(new Path("/dir/a.txt"), new Path("/in/a.txt"));
    }
}