HDFS的读写

最新推荐文章于 2024-06-11 05:30:00 发布

原创最新推荐文章于 2024-06-11 05:30:00 发布 · 225 阅读

0 ·

CC 4.0 BY-SA版权

本文详细介绍如何使用Java API从HDFS中读取MapReduce输出结果，并将数据写入HDFS的方法。通过示例代码，读者可以学习到配置Hadoop环境、创建文件系统实例、读取和写入数据的具体步骤。

1、读取mr的结果输出目录

public static List<String> getHdfsFileColumns(Stirng path){
   List<String> colimns = new ArrayList<String>();
   InputStreamReader isr = null;
   BufferedReader br = null;
   try{
      Configuration conf = new Configuration();
      conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributeFileSystem");
      FileSystem fs = FileSystem.get(URI.create(path),conf);
      FileStatus [] status = fs.listStatus(new Path(path));
      for (FileStatus file : status){
            if(!file.getPath().getName.startWith("part-r")){
                continue;
          }
            FSDataInputStream fsdata = fs.open(file.getPath());
            isr = new InputStreamReader(fsdata);
            br = new BufferedReader(isr);
            String line = "";
            while((line = br.readLine())!=null){
               columns.add(line);

          }

     }
      isr.close();
      br.close();


   }catch (Exception e){
      e.printStackTrace();
   }



}

2、将数据写到hdfs

public static List<String> getHdfsFileColumns(Stirng path){
   List<String> colimns = new ArrayList<String>();
   InputStreamReader isr = null;
   BufferedReader br = null;
   try{
      Configuration conf = new Configuration();
      conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributeFileSystem");
      FileSystem fs = FileSystem.get(URI.create(path),conf,"root");
      FSDataOutputStream out= fs.create(new Path(path));
      byte [] bb ="abc".getBytes();
      out.write(bb);
      out.close();
     }catch(Exceptionn e){
       e.printStackTrace();
    }
     


   }



}