调用java api 访问HDFS

依赖

<dependency>
			<groupId> org.apache.hadoop</groupId>
			<artifactId>hadoop-client </artifactId>
			<version>2.6.0</version>
</dependency>

上传文件

/**
     * 上传文件到hdfs系统
     * 伪分布方式启动下的实现
     * @param s 源文件
     * @param d 目标路径
     * @throws IOException
     * @see [类、类#方法、类#成员]
     */
    public static void uploadLocalFile2HDFS(String s, String d)
        throws IOException
    {
        
        Configuration config = new Configuration();
        Path dstDir = new Path(d); 
        FileSystem hdfs = dstDir.getFileSystem(config);
        
        Path src = new Path(s);
  
        
        hdfs.copyFromLocalFile(src, dstDir);
        
        hdfs.close();
    }

创建文件并写入内容

/**
     * 创建文件并写入内容
     * <功能详细描述>
     * @param toCreateFilePath 文件名【可以指定路径】
     * @param content 文件内容
     * @param hdfsAddress 连接的HDFS地址
     * @throws IOException
     * @see [类、类#方法、类#成员]
     */
    public static void createNewHDFSFile(String toCreateFilePath, String content,String hdfsAddress) throws IOException
    {
        Configuration config = new Configuration();
        Path dstDir = new Path(hdfsAddress); 
        FileSystem hdfs = dstDir.getFileSystem(config);
        
        FSDataOutputStream os = hdfs.create(new Path(toCreateFilePath));

        os.write(content.getBytes("UTF-8"));
        
        os.close();
        
        hdfs.close();
    }

删除文件/目录

/**
     * 删除文件/目录
     * <功能详细描述>
     * @param dst
     * @param hdfsAddress
     * @return
     * @throws IOException
     * @see [类、类#方法、类#成员]
     */
    public static boolean deleteHDFSFile(String dst,String hdfsAddress) throws IOException
    {
        Configuration config = new Configuration();
        Path dstDir = new Path(hdfsAddress); 
        FileSystem hdfs = dstDir.getFileSystem(config);
        
        Path dstPath = new Path(dst); 
        boolean isDeleted = hdfs.deleteOnExit(dstPath);
        
        hdfs.close();
        
        return isDeleted;
    }

读取文件

/**
     * 读取文件
     * <功能详细描述>
     * @param dst
     * @param hdfsAddress
     * @return
     * @throws Exception
     * @see [类、类#方法、类#成员]
     */
    public static byte[] readHDFSFile(String dst,String hdfsAddress) throws Exception
    {
        Configuration config = new Configuration();
        Path dstDir = new Path(hdfsAddress); 
        FileSystem fs = dstDir.getFileSystem(config);
        
        // check if the file exists
        Path path = new Path(dst);
        if ( fs.exists(path) )
        {
            FSDataInputStream is = fs.open(path);
            // get the file info to create the buffer
            FileStatus stat = fs.getFileStatus(path);
            
            // create the buffer
            byte[] buffer = new byte[Integer.parseInt(String.valueOf(stat.getLen()))];
            is.readFully(0, buffer);
            
            is.close();
            fs.close();
            
            return buffer;
        }
        else
        {
            throw new Exception("the file is not found .");
        }
    }

创建目录

/**
     * 创建目录及文件
     * 可递归的创建多层目录
     * @param dir
     * @param hdfsAddress
     * @throws IOException
     * @see [类、类#方法、类#成员]
     */
    public static void mkdir(String dir,String hdfsAddress) throws IOException
    {
        Configuration config = new Configuration();
        Path dstDir = new Path(hdfsAddress); 
        FileSystem fs = dstDir.getFileSystem(config);
        
        fs.mkdirs(new Path(dir));
        
        fs.close();
    }

读取目录下的所有文件

/**
     * 展示目录下的文件和文件夹
     * <功能详细描述>
     * @param dir
     * @param hdfsAddress
     * @throws IOException
     * @see [类、类#方法、类#成员]
     */
    public static void listAll(String dir,String hdfsAddress) throws IOException
    {
        Configuration config = new Configuration();
        Path dstDir = new Path(hdfsAddress); 
        FileSystem fs = dstDir.getFileSystem(config);
        
        FileStatus[] stats = fs.listStatus(new Path(dir));
        
        for(int i = 0; i < stats.length; ++i)
        {
            if (stats[i].isFile())
            {
                // regular file
                System.out.println(stats[i].getPath().toString());
            }
            else if (stats[i].isDirectory())
            {
                // dir
                System.out.println(stats[i].getPath().toString());
            }
            else if(stats[i].isSymlink())
            {
                // is s symlink in linux
                System.out.println(stats[i].getPath().toString());
            }
                 
        }
        fs.close();
    }

将本地文件内容copy到hdfs文件中

/**
     * 将本地文件内容copy到hdfs文件中
     * <功能详细描述>
     * @param localFilePath 本地文件路径
     * @param hdfsPath hdfs文件路径
     * @throws FileNotFoundException 
     * @throws IOException
     * @see [类、类#方法、类#成员]
     */
    public static void FileCopyWhitProcess(String localFilePath, String hdfsPath) throws FileNotFoundException{
        InputStream in = new BufferedInputStream(new FileInputStream(localFilePath));
        Configuration config = new Configuration();
        FileSystem fs;
        FSDataOutputStream out;
        try
        {
           
            fs = FileSystem.get(URI.create(hdfsPath), config);
            out= fs.create(new Path(hdfsPath), new Progressable()
            {
                
                @Override
                public void progress()
                {
                    System.out.println(".");
                    
                }
            });
            IOUtils.copyBytes(in, out, 4096, true);
        }
        catch (IOException e)
        {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            IOUtils.closeStream(in);
        }
        
    }