HDFS上文件处理、Java文件读写

使用Java api实现文档的上传/下载/删除文件：

1　上传文件到HDFS

有时需要自动将文件上传到HDFS上，在java中可以通过如下函数实现：

public static boolean put2HDFS(String src , String dst , Configuration conf){ 

        Path dstPath = new Path(dst) ; 

        try{ 

            FileSystem hdfs = dstPath.getFileSystem(conf) ; 

            hdfs.copyFromLocalFile(false, new Path(src), dstPath) ; 

        }catch(IOException ie){ 

            ie.printStackTrace() ; 

            return false ; 

        } 

        return true ; 

    }

 

2　 HDFS上文件下载

通过代码实现HDFS上文件的下载：

public static boolean getFromHDFS(String src , String dst , Configuration conf){ 

        Path dstPath = new Path(dst) ; 

        try{ 

            FileSystem dhfs = dstPath.getFileSystem(conf) ; 

            dhfs.copyToLocalFile(false, new Path(src), dstPath) ; 

        }catch(IOException ie){ 

            ie.printStackTrace() ; 

            return false ; 

        } 

        return true ; 

    }

调用该函数时需要加下面这行代码：

conf.addResource(new Path("E:/hadoop/conf/core-site.xml"));

 

3　 HDFS上文件删除

当需要迭代mapreduce时，需要删除HDFS上的中间文件，防止占用太多空间，删除函数如下：

public static boolean checkAndDel(final String path , Configuration conf){ 

        Path dstPath = new Path(path) ; 

        try{ 

            FileSystem dhfs = dstPath.getFileSystem(conf) ; 

            if(dhfs.exists(dstPath)){ 

                dhfs.delete(dstPath, true) ; 

            }else{ 

                return false ; 

            } 

        }catch(IOException ie ){ 

            ie.printStackTrace() ; 

            return false ; 

        } 

        return true ; 

    }

Java文件读写

java中有好几种读写文件的方法，以FileInputStream、FileOutputStream类示例如下： http://yjmyzz.cnblogs.com

package jmyang.file;

 

import java.io.*;

public class FileTest {

    

   

    public static boolean delete(String fileName){

        boolean result = false;

        File f = new File(fileName);

        if (f.exists()){

            try{

                result = f.delete();

            }

            catch(Exception e){

                e.printStackTrace();

            }          

        }

        else{

            result = true;

        }

        return result;

    }

    

   

    public static String read(String fileName) {

        File f = new File(fileName);

        if (!f.exists()) {

            return "File not found!";

        }

        FileInputStream fs;

        String result = null;

        try {

            fs = new FileInputStream(f);

            byte[] b = new byte[fs.available()];

            fs.read(b);

            fs.close();

            result = new String(b);

        } catch (Exception e) {

            e.printStackTrace();

        }

 

        return result;

    }

 

   

    public static boolean write(String fileName, String fileContent) {

        boolean result = false;

        File f = new File(fileName);

        try {

            FileOutputStream fs = new FileOutputStream(f);

            byte[] b = fileContent.getBytes();

            fs.write(b);

            fs.flush();

            fs.close();

            result = true;

        } catch (Exception e) {

            e.printStackTrace();

        }

        return result;

    }

 

   

    public static boolean append(String fileName, String fileContent) {

        boolean result = false;

        File f = new File(fileName);

        try {

            if (f.exists()) {

                FileInputStream fsIn = new FileInputStream(f);

                byte[] bIn = new byte[fsIn.available()];

                fsIn.read(bIn);

                String oldFileContent = new String(bIn);

                //System.out.println("旧内容:" + oldFileContent);

                fsIn.close();

                if (!oldFileContent.equalsIgnoreCase("")) {

                    fileContent = oldFileContent + "\r\n" + fileContent;

                    //System.out.println("新内容:" + fileContent);

                }

            }

 

            FileOutputStream fs = new FileOutputStream(f);

            byte[] b = fileContent.getBytes();

            fs.write(b);

            fs.flush();

            fs.close();

            result = true;

        } catch (Exception e) {

            e.printStackTrace();

        }

        return result;

    }

}