用java写数据存入HDFS中，采用追加模式

最新推荐文章于 2023-02-13 11:06:17 发布

原创最新推荐文章于 2023-02-13 11:06:17 发布 · 1.4k 阅读

6 ·

CC 4.0 BY-SA版权

文章标签：

#hadoop #hdfs #大数据

bug解答专栏收录该内容

22 篇文章

订阅专栏

本文介绍如何使用Java代码将数据追加写入Hadoop的HDFS文件系统，主要涉及Hadoop相关API的使用，适用于大数据处理场景。

代码如下：：：：：

package com.postpony.bulkprint.BigData;

import com.cloudera.org.codehaus.jackson.map.ObjectMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.IOException;

public class Input_HDFS {
    private static Configuration conf = new Configuration();
    static {
        conf.setBoolean("dfs.support.append", true);
        conf.set("fs.defaultFS","hdfs://node01:8020");
        conf.set("dfs.client.block.write.replace-datanode-on-failure.policy", "NEVER");
        conf.set("fe.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
        conf.setBoolean("dfs.client.block.write.replace-datanode-on-failure.enable", true);
    }
    public static void main(String[] args) {
        String a = "西安人的城墙下是西安人的火车";
        //JSONObject jo = JSONObject.parseObject(new String("123"));
        try {
            write2hdfs(a,"/test02/test005");
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
    /**
     * json 存储
     *
     * @param
     * @param
     * @throws IOException
     */
    public static void write2hdfs(Object listContent, String filePath) throws IOException {
        ObjectMapper objectMapper = new ObjectMapper();
        FileSystem fs = null;
        Path path = new Path(filePath);
        FSDataOutputStream output = null;
        fs = path.getFileSystem(conf);
//如果此文件不存在则创建新文件
        if (!fs.exists(path)) {
            fs.createNewFile(path);
        }
        output = fs.append(new Path(filePath));
        //System.out.println(listContent.toString());
        output.write(objectMapper.writeValueAsString(listContent).getBytes("UTF-8"));
        output.write("\n".getBytes("UTF-8"));//换行
        fs.close();
        output.close();

    }
}

pom文件：

<repositories>
   <repository>
      <id>aliyun</id>
      <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
   </repository>
   <repository>
      <id>cloudera</id>
      <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
   </repository>
   <repository>
      <id>jboss</id>
      <url>http://repository.jboss.com/nexus/content/groups/public</url>
   </repository>
</repositories>
<properties>
   <hadoop.version>2.6.0-cdh5.14.0</hadoop.version>
</properties>
<dependency>
   <groupId> org.apache.cassandra</groupId>
   <artifactId>cassandra-all</artifactId>
   <version>0.8.1</version>

   <exclusions>
      <exclusion>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-log4j12</artifactId>
      </exclusion>
      <exclusion>
         <groupId>log4j</groupId>
         <artifactId>log4j</artifactId>
      </exclusion>
   </exclusions>

</dependency>
<dependency>
   <groupId>org.apache.hadoop</groupId>
   <artifactId>hadoop-client</artifactId>
   <version>${hadoop.version}</version>
</dependency>

<dependency>
   <groupId>postpony</groupId>
   <artifactId>postpony-micro-service-starter</artifactId>
</dependency>