hadoop hdfs java api 文件操作类

HDFSUtil.java 是操作 hadoop 文件系统的工具类

 

package test.hadoop.util; import java.util.Iterator; import java.util.Map.Entry; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import com.emar.adwiser.common.util.LogUtil; import com.emar.adwiser.common.util.Logs; /*** * HDFS 工具类 * @author zhaidw * */ public class HDFSUtil { private static Logs log = LogUtil.getLog(HDFSUtil.class); public synchronized static FileSystem getFileSystem(String ip, int port) { FileSystem fs = null; String url = "hdfs://" + ip + ":" + String.valueOf(port); Configuration config = new Configuration(); config.set("fs.default.name", url); try { fs = FileSystem.get(config); } catch (Exception e) { log.error("getFileSystem failed :" + ExceptionUtils.getFullStackTrace(e)); } return fs; } public synchronized static void listNode(FileSystem fs) { DistributedFileSystem dfs = (DistributedFileSystem) fs; try { DatanodeInfo[] infos = dfs.getDataNodeStats(); for (DatanodeInfo node : infos) { System.out.println("HostName: " + node.getHostName() + "/n" + node.getDatanodeReport()); System.out.println("--------------------------------"); } } catch (Exception e) { log.error("list node list failed :" + ExceptionUtils.getFullStackTrace(e)); } } /** * 打印系统配置 * * @param fs */ public synchronized static void listConfig(FileSystem fs) { Iterator<Entry<String, String>> entrys = fs.getConf().iterator(); while (entrys.hasNext()) { Entry<String, String> item = entrys.next(); log.info(item.getKey() + ": " + item.getValue()); } } /** * 创建目录和父目录 * * @param fs * @param dirName */ public synchronized static void mkdirs(FileSystem fs, String dirName) { // Path home = fs.getHomeDirectory(); Path workDir = fs.getWorkingDirectory(); String dir = workDir + "/" + dirName; Path src = new Path(dir); // FsPermission p = FsPermission.getDefault(); boolean succ; try { succ = fs.mkdirs(src); if (succ) { log.info("create directory " + dir + " successed. "); } else { log.info("create directory " + dir + " failed. "); } } catch (Exception e) { log.error("create directory " + dir + " failed :" + ExceptionUtils.getFullStackTrace(e)); } } /** * 删除目录和子目录 * * @param fs * @param dirName */ public synchronized static void rmdirs(FileSystem fs, String dirName) { // Path home = fs.getHomeDirectory(); Path workDir = fs.getWorkingDirectory(); String dir = workDir + "/" + dirName; Path src = new Path(dir); boolean succ; try { succ = fs.delete(src, true); if (succ) { log.info("remove directory " + dir + " successed. "); } else { log.info("remove directory " + dir + " failed. "); } } catch (Exception e) { log.error("remove directory " + dir + " failed :" + ExceptionUtils.getFullStackTrace(e)); } } /** * 上传目录或文件 * * @param fs * @param local * @param remote */ public synchronized static void upload(FileSystem fs, String local, String remote) { // Path home = fs.getHomeDirectory(); Path workDir = fs.getWorkingDirectory(); Path dst = new Path(workDir + "/" + remote); Path src = new Path(local); try { fs.copyFromLocalFile(false, true, src, dst); log.info("upload " + local + " to " + remote + " successed. "); } catch (Exception e) { log.error("upload " + local + " to " + remote + " failed :" + ExceptionUtils.getFullStackTrace(e)); } } /** * 下载目录或文件 * * @param fs * @param local * @param remote */ public synchronized static void download(FileSystem fs, String local, String remote) { // Path home = fs.getHomeDirectory(); Path workDir = fs.getWorkingDirectory(); Path dst = new Path(workDir + "/" + remote); Path src = new Path(local); try { fs.copyToLocalFile(false, dst, src); log.info("download from " + remote + " to " + local + " successed. "); } catch (Exception e) { log.error("download from " + remote + " to " + local + " failed :" + ExceptionUtils.getFullStackTrace(e)); } } /** * 字节数转换 * * @param size * @return */ public synchronized static String convertSize(long size) { String result = String.valueOf(size); if (size < 1024 * 1024) { result = String.valueOf(size / 1024) + " KB"; } else if (size >= 1024 * 1024 && size < 1024 * 1024 * 1024) { result = String.valueOf(size / 1024 / 1024) + " MB"; } else if (size >= 1024 * 1024 * 1024) { result = String.valueOf(size / 1024 / 1024 / 1024) + " GB"; } else { result = result + " B"; } return result; } /** * 遍历HDFS上的文件和目录 * * @param fs * @param path */ public synchronized static void listFile(FileSystem fs, String path) { Path workDir = fs.getWorkingDirectory(); Path dst; if (null == path || "".equals(path)) { dst = new Path(workDir + "/" + path); } else { dst = new Path(path); } try { String relativePath = ""; FileStatus[] fList = fs.listStatus(dst); for (FileStatus f : fList) { if (null != f) { relativePath = new StringBuffer() .append(f.getPath().getParent()).append("/") .append(f.getPath().getName()).toString(); if (f.isDir()) { listFile(fs, relativePath); } else { System.out.println(convertSize(f.getLen()) + "/t/t" + relativePath); } } } } catch (Exception e) { log.error("list files of " + path + " failed :" + ExceptionUtils.getFullStackTrace(e)); } finally { } } public synchronized static void write(FileSystem fs, String path, String data) { // Path home = fs.getHomeDirectory(); Path workDir = fs.getWorkingDirectory(); Path dst = new Path(workDir + "/" + path); try { FSDataOutputStream dos = fs.create(dst); dos.writeUTF(data); dos.close(); log.info("write content to " + path + " successed. "); } catch (Exception e) { log.error("write content to " + path + " failed :" + ExceptionUtils.getFullStackTrace(e)); } } public synchronized static void append(FileSystem fs, String path, String data) { // Path home = fs.getHomeDirectory(); Path workDir = fs.getWorkingDirectory(); Path dst = new Path(workDir + "/" + path); try { FSDataOutputStream dos = fs.append(dst); dos.writeUTF(data); dos.close(); log.info("append content to " + path + " successed. "); } catch (Exception e) { log.error("append content to " + path + " failed :" + ExceptionUtils.getFullStackTrace(e)); } } public synchronized static String read(FileSystem fs, String path) { String content = null; // Path home = fs.getHomeDirectory(); Path workDir = fs.getWorkingDirectory(); Path dst = new Path(workDir + "/" + path); try { // reading FSDataInputStream dis = fs.open(dst); content = dis.readUTF(); dis.close(); log.info("read content from " + path + " successed. "); } catch (Exception e) { log.error("read content from " + path + " failed :" + ExceptionUtils.getFullStackTrace(e)); } return content; } }  

 

示例用法：

 

package test.hadoop; import org.apache.hadoop.fs.FileSystem; import test.hadoop.util.HDFSUtil; public class Demo { /** * @param args * @throws Exception */ public static void main(String[] args) throws Exception { FileSystem fs = HDFSUtil.getFileSystem("192.168.1.101", 9000); // FsUtil.listConfig(fs); String dirName = "demo.txt"; // FsUtil.mkdirs(fs, dirName); // FsUtil.rmdirs(fs, dirName); // FsUtil.upload(fs, "D:/help/js", dirName); // FsUtil.download(fs, "D:/help/js2", dirName); HDFSUtil.write(fs, dirName, "test-测试"); HDFSUtil.append(fs, dirName, "/ntest-测试2"); // // String content = FsUtil.read(fs, dirName); // // System.out.println(content); // FsUtil.listFile(fs, ""); // FsUtil.listNode(fs); fs.close(); } }