MyUtil:保存数据到Mysql_mytutl-优快云博客

本文链接：https://blog.youkuaiyun.com/drl_blogs/article/details/95722304

本文介绍了一个名为MyUtil的实用工具类，详细阐述了如何利用该工具便捷地进行数据的保存操作，旨在提高将数据存储到MySQL数据库的效率和简便性。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

import java.sql.DriverManager

import org.apache.kafka.common.TopicPartition
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.kafka010.OffsetRange

import scala.collection.mutable

object DataToMysql {
  def getconn ={
    DriverManager.getConnection(
    "jdbc:mysql://localhost:3306/test",
    "root",
    "1234"
  )
  }

  //保存到数据库mysql
def saveToMysql(result1: RDD[(String, Int)])={
  result1.foreachPartition(filter=>{
    val conn=getconn
    //遍历。储存数据
    filter.foreach(tp=>{
      val count = getword(tp._1)+tp._2
      val ps=conn.prepareStatement("replace into wordcount values(?,?)")
      ps.setString(1,tp._1)
      ps.setInt(2,count)
      ps.executeUpdate()
      ps.close()
    })
    //关闭
    conn.close()
  })
}
  //手动维护偏移量
  def KafkaOffset(groupid:String,ranges: Array[OffsetRange])={
   val conn=getconn
    for (o <- ranges){
      val ps=conn.prepareStatement("replace into offsets values(?,?,?,?)")
      ps.setString(1,groupid)
      ps.setString(2,o.topic)
      ps.setInt(3,o.partition)
      ps.setLong(4,o.untilOffset)
      ps.executeUpdate()
      ps.close()
    }
   conn.close()
  }
  //从数据库中获取偏移量信息
  def getOffset(groupid:String,topic:String): mutable.Map[TopicPartition, Long] ={
    import collection.mutable.Map
    //定义一个变量去接收
    val offset=Map[TopicPartition,Long]()
    val conn=getconn
   val ps= conn.prepareStatement("select * from offsets where groupid=? and topic=?")
    ps.setString(1,groupid)
    ps.setString(2,topic)
    //得到一个结果集
    val resultSet = ps.executeQuery()
   while(resultSet.next()){
     //赋值给offset
     offset += new TopicPartition(resultSet.getString("topic"),resultSet.getInt("partition")) -> resultSet.getLong("offset")
   }
    //返回结果
    offset
  }


  def getword(word:String) ={
    val conn=getconn
    val ps= conn.prepareStatement("select * from wordcount where word=?")
    ps.setString(1,word)
    val resultSet = ps.executeQuery()
    if (resultSet.next()){
      resultSet.getInt("count")
    }else{
      0
    }
  }
}