Java操作HDFS文件系统

Java API操作HDFS文件系统

pom.xml

  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.11</version>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-hdfs</artifactId>
      <version>2.7.3</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-common</artifactId>
      <version>2.7.3</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-core -->
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-core</artifactId>
      <version>1.2.1</version>
    </dependency>
  </dependencies>

package com.nike.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.io.IOUtils;

import java.io.*;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Date;

public class HadoopCtrl {
    private static Configuration conf = new Configuration();
    private static FileSystem fs = null;
    private static String url = null;
    private static URI uri = null;

    static {
        conf.addResource("core-site.xml");
        conf.addResource("hdfs-site.xml");
        url = conf.get("fs.defaultFS");
        uri = URI.create(url);
        try {
            fs = FileSystem.get(uri,conf);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 追加到hdfs文件
     * @param src 本地文件
     * @param dest HDFS文件
     */
    public void appendToFile(String src,String dest){
        try {
            OutputStream out = fs.append(new Path(dest));
            InputStream in = new BufferedInputStream(new FileInputStream(src));
            IOUtils.copyBytes(in,out,conf,true);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 查看HDFS系统中文件内容
     * @param url
     */
    public void cat(String url){
        try {
            InputStream input = fs.open(new Path(url));
            IOUtils.copyBytes(input,System.out,conf,true);
//            BufferedInputStream br = new BufferedInputStream(input);
//            byte[] buf = new byte[1024];
//            int len = -1;
//            while ((len = br.read(buf))!=-1){
//                System.out.println(new String(buf,0,len));
//            }
//            br.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 将本地文件上传到HDFS
     * @param src
     * @param dest
     */
    public void copyFromLocal(String src,String dest){
        try {
            fs.copyFromLocalFile(new Path(src),new Path(dest));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 将HDFS中的文件复制到本地
     * @param src
     * @param dest
     */
    public void copyToLocal(String src,String dest){
        try {
            fs.copyToLocalFile(new Path(src),new Path(dest));
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public void count(String url){
        Path path = new Path(url);
        try {
            ContentSummary sum = fs.getContentSummary(path);
//            System.out.println(sum.getFileCount());
//            System.out.println(sum.getDirectoryCount());
//            System.out.println(sum.getLength());
            System.out.println(sum.getDirectoryCount()+"\t"+sum.getFileCount()+"\t"+sum.getLength()+"\t"+url);

        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 创建快照
     * @param url
     * @param snapName
     */
    public void createSnapshot(String url,String snapName){
        Path path = new Path(url);
        try {
            HdfsAdmin admin = new HdfsAdmin(uri,conf);
            admin.allowSnapshot(path);
            Path snapshot = fs.createSnapshot(path, snapName);
            System.out.println(snapshot.toString());
        } catch (IOException e) {
            e.printStackTrace();
        }

    }

    /**
     * 删除快照
     * @param url
     * @param snapName
     */
    public void deleteSnapshot(String url,String snapName){
        Path path = new Path(url);
        try {
            fs.deleteSnapshot(path,snapName);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 将HDFS中的文件下载到本地
     * @param src
     * @param dest
     */
    public void get(String src,String dest){
        try {
            InputStream in = fs.open(new Path(src));
            OutputStream out = new BufferedOutputStream(new FileOutputStream(dest));
            IOUtils.copyBytes(in,out,conf,true);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 上传文件
     * @param src
     * @param dest
     */
    public void put(String src,String dest){
        try {
            OutputStream out = fs.create(new Path(dest));
            InputStream in = new BufferedInputStream(new FileInputStream(src));
            IOUtils.copyBytes(in,out,conf,true);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 创建目录
     * @param url
     */
    public void mkdir(String url){
        Path path = new Path(url);
        try {
            fs.mkdirs(path);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 删除文件或目录
     * @param url
     */
    public void rmr(String url){
        Path path = new Path(url);
        try {
            fs.delete(path,true);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    /**
     * 创建空白文件
     * @param url
     */
    public void touchz(String url){
        Path path = new Path(url);
        try {
            fs.createNewFile(path);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public void ls(String url){
        Path path = new Path(url);
        try {
            RemoteIterator<LocatedFileStatus> itr = fs.listLocatedStatus(path);
            while(itr.hasNext()){
                FileStatus fileStatus = itr.next();
                listInfo(fileStatus);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public void listInfo(FileStatus fileStatus){
        StringBuilder sb = new StringBuilder();
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
        if (fileStatus.isDirectory()){
            sb.append("d"+fileStatus.getPermission()+"\t"+"-");
        }else{
            sb.append("-"+fileStatus.getPermission()+"\t"+fileStatus.getReplication());
        }
        sb.append("\t"+fileStatus.getOwner().toString()+"\t"+fileStatus.getGroup().toString()+
                "\t"+fileStatus.getLen()+"\t"+sdf.format(new Date(fileStatus.getModificationTime()))+"\t");
        sb.append(fileStatus.getPath().toString().substring(url.length()));
        System.out.println(sb.toString());
    }

    public void lsr(String url){
        Path path = new Path(url);
        try {
            RemoteIterator<LocatedFileStatus> itr = fs.listLocatedStatus(path);
            while(itr.hasNext()){
                FileStatus fileStatus = itr.next();
                listInfo(fileStatus);
                if (fileStatus.isDirectory()){
                    lsr(fileStatus.getPath().toString());
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}