Spark2.1.0 向MongoDB写入json数据

最新推荐文章于 2021-02-22 15:41:10 发布

向阳争渡

最新推荐文章于 2021-02-22 15:41:10 发布

阅读量1.2k

点赞数

CC 4.0 BY-SA版权

分类专栏：大数据/Spark 数据库文章标签： Spark MongoDB

本文链接：https://blog.youkuaiyun.com/yangyang_yangqi/article/details/78998430

大数据/Spark 同时被 2 个专栏收录

14 篇文章

订阅专栏

数据库

3 篇文章

订阅专栏

本文介绍如何使用Spark从外部文件读取JSON数据，并通过DataFrame将数据写入MongoDB的过程。涉及Spark配置、数据读取及转换为特定对象类型，最终实现数据存储。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

需求：

1、从外部文件读取json数据
2、根据需求拆分数据
3、利用DataFrame直接写入MongoDB

Spark-Mongodb官网写入MongoDB实例
采用官网实例的方案实验，不成功，且json数据中部分字段为空，读取报错。

import com.mongodb.spark.MongoSpark;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.SparkSession;
import org.bson.Document;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.json.JSONObject;
import com.mongodb.spark.config.WriteConfig;
import org.apache.spark.api.java.function.Function;
import java.util.HashMap;
import java.util.Map;
import mongo.Book;
public class Mongo {
    public static void main(String[] args) {
        SparkSession spark = SparkSession.builder()
                .appName("mongo")
                .master("local[4]")
                .config("spark.mongodb.input.url","mongodb://127.0.0.1/spark.mongo")
                .config("spark.mongodb.output.uri","mongodb://127.0.0.1/spark.mongoœ")
                .getOrCreate();
        JavaRDD<String> input = spark.sparkContext()
                .textFile("/Users/yangyang/Desktop/json/part-r-00003.txt",1)
                .toJavaRDD().map(x -> x.split("____")[1]);
        Map<String, String> writeOverrides = new HashMap<String, String>();
        writeOverrides.put("collection", "spark");
        writeOverrides.put("writeConcern.w", "majority");
        WriteConfig writeConfig = WriteConfig.create(spark).withOptions(writeOverrides);

        JavaRDD<Book> books = input.map(new Function<String, Book>() {
            public Book call(String s) throws Exception {
                JSONObject jsons = new JSONObject(s);
                Book book = new Book();

                book.setBookId(jsons.get("bookId").toString());
                book.setContent(jsons.get("content").toString());
                book.setContentStartPos(jsons.get("contentStartPos").toString());
                book.setCoord(jsons.get("coord").toString());
                book.setId(jsons.get("id").toString());
                book.setLineColor(jsons.get("lineColor").toString());
                book.setLineType(jsons.get("lineType").toString());
                book.setLineWidth(jsons.get("lineWidth").toString());
                book.setNoteCatalog(jsons.get("noteCatalog").toString());
                book.setNoteLabels(jsons.get("noteLabels").toString());
                book.setNoteOrigin(jsons.get("noteOrigin").toString());
                book.setNotePath(jsons.get("notePath").toString());
                book.setNotePostil(jsons.get("notePostil").toString());
                book.setNoteType(jsons.get("noteType").toString());
                book.setPageAngle(jsons.get("pageAngle").toString());
                book.setPageHeight(jsons.get("pageHeight").toString());
                book.setPageIndex(jsons.get("pageIndex").toString());
                book.setPageWidth(jsons.get("pageWidth").toString());
                book.setPdfId(jsons.get("pdfId").toString());
                book.setUpdateTime(jsons.get("updateTime").toString());
                book.setUserName(jsons.get("userName").toString());
                book.setSourceType(jsons.get("sourceType").toString());
                return book;
            }
        });
        MongoSpark.save(books,Book.class);
        spark.close();
    }
}

研究了下MongoSpark的API，API里就有转化json RDD的方法。

import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import com.mongodb.spark.MongoSpark;

public class DataSetMongo {
    public static void main(String[] args) {
        SparkSession spark = SparkSession.builder()
                .appName("mongo")
                .master("local[4]")
                .config("spark.mongodb.input.uri", "mongodb://127.0.0.1/spark.mongo")
                .config("spark.mongodb.output.uri", "mongodb://127.0.0.1/spark.momgo")
                .getOrCreate();
        JavaRDD<String> input = spark.sparkContext()
                .textFile("/Users/yangyang/Desktop/json/part-r-00003.txt",1)
                .toJavaRDD().map(x -> x.split("____")[1]);
        Dataset<Row> books = MongoSpark.read(spark).json(input);
        books.show();
        MongoSpark.write(books).option("collection", "mongo").mode("overwrite").save();; 
        spark.close();
    }
}