redis读取数据,写入mysql

最新推荐文章于 2025-05-18 11:44:21 发布

念念不忘_

最新推荐文章于 2025-05-18 11:44:21 发布

阅读量3.3k

点赞数

CC 4.0 BY-SA版权

分类专栏： spark redis mysql

本文链接：https://blog.youkuaiyun.com/bb23417274/article/details/87859458

spark 同时被 3 个专栏收录

53 篇文章

订阅专栏

mysql

10 篇文章

订阅专栏

redis

6 篇文章

订阅专栏

本文介绍了如何从Redis中读取数据并将其有效写入MySQL的过程，探讨了数据迁移的策略和可能遇到的问题。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

package com.ws.spark

import java.io.{File, PrintWriter}
import java.util.Properties

import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}
import redis.clients.jedis.Jedis


object RedisToMysql {

  def main(args: Array[String]): Unit = {
    import scala.collection.JavaConversions._
    val jedis: Jedis = new Jedis("192.168.0.21", 6379)
    val list = jedis.lrange("rank:arena_rank", 0, 2500)

    val printWriter: PrintWriter = new PrintWriter(new File("rank.txt"))

    for (l <- list) {
    //写入文本,每次写入都是从新的一行开始
      printWriter.println(l)
    }
    printWriter.close()

    val session: SparkSession = SparkSession.builder().appName("SqlWordCount").master("local[*]").getOrCreate()

    val dataFrame: DataFrame = session.read.json("rank.txt")

    val filterRdd: Dataset[Row] = dataFrame.filter(x => {
      val uid: String = x.getAs[String]("uid")
      uid.length > 20
    })

    filterRdd.createTempView("user_arena")

    val frame: DataFrame = session.sql(" select * from user_arena")

    val p = new Properties()
    p.put("user", "root")
    p.put("password", "root")

    //SaveMode枚举
    frame.write.mode("append").jdbc("jdbc:mysql://192.168.0.21:3306/cok_db1?characterEncoding=utf-8", "user_arena_2", p)

    session.stop()
  }
}

灵感来自