用spark Streamming 将mysql的数据 展示到控制台

第一步

依赖

  <!--对接MysQL-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.38</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>2.4.3</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.9.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-auth</artifactId>
            <version>2.9.2</version>
        </dependency>

drver编写

package com.baizhi.demo02

import com.mysql.jdbc.Driver
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration
import org.apache.hadoop.mapreduce.lib.db.DBInputFormat
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.rdd.RDD

object SparkRDDMySQL {
  def main(args: Array[String]): Unit = {
    //1.创建SparkContext
    val sparkConf = new SparkConf()
      .setMaster("local[6]")
      .setAppName("wordcount")
    val sc=new SparkContext(sparkConf)
    //2.创建并行集合 数据集 抽象成 Spark 并行集合 RDD （理解Scala集合升级版）
    val hadoopConfig = new Configuration()
    DBConfiguration.configureDB(hadoopConfig,
      "com.mysql.jdbc.Driver",
      "jdbc:mysql://localhost:3306/test",
      "root", "root")

    hadoopConfig.set(DBConfiguration.INPUT_QUERY,"select id,name from t_user")
    hadoopConfig.set(DBConfiguration.INPUT_COUNT_QUERY,"select count(*) from t_user")
    hadoopConfig.set(DBConfiguration.INPUT_CLASS_PROPERTY,"com.baizhi.demo02.UserDBWriteable")

    val userRDD:RDD[(LongWritable,UserDBWriteable)] = sc.newAPIHadoopRDD(hadoopConfig,classOf[DBInputFormat[UserDBWriteable]],classOf[LongWritable],classOf[UserDBWriteable])

    userRDD.map(t=>(t._2.id,t._2.name))
        .foreach(tuple=>println(tuple))//action算子
    //4.关闭SparkContext
    sc.stop();
  }
}

package com.baizhi.demo02

import java.sql.{PreparedStatement, ResultSet}

import org.apache.hadoop.mapreduce.lib.db.DBWritable

class UserDBWriteable extends DBWritable{
  var id:Int=_
  var name:String=_
  //无需是实现-使用DBOutputForamt使用
  override def write(preparedStatement: PreparedStatement): Unit = {

  }

  //封装数据
  override def readFields(rs: ResultSet): Unit = {
    id = rs.getInt("id")
    name=rs.getString("name")
  }
}