foreachrdd写入mysql_通过SparkStreaming的foreachRDD把处理后的数据写入外部存储系统中...

1、技术实现解析

2、实现实战

关键部分代码：resultRowRDD.foreachPartition( partitionOfRecords => {

// ConnectionPool is a static, lazily initialized pool of connections

val connection = ConnectionPool.getConnection()

partitionOfRecords.foreach(record => {

val sql = "insert into categorytop3(category, item, click_count) values ('" + record.getAs("category") + "','" +

record.getAs("item") + "'," + record.getAs("click_count") + ")"

val stmt = connection.createStatement()

stmt.executeUpdate(sql)

})

ConnectionPool.returnConnection(connection)

})

ConnectionPool.javapackage com.tom.spark.sparkstreaming;

import java.sql.Connection;

import java.sql.DriverManager;

import java.util.LinkedList;

public class ConnectionPool {

private static LinkedList connectionQueue;

static {

try {

Class.forName("com.mysql.jdbc.Driver");

} catch (ClassNotFoundException e) {

e.printStackTrace();

}

}

public synchronized static Connection getConnection() {

try {

if(connectionQueue == null) {

connectionQueue = new LinkedList();

for(int i = 0; i < 5; i++) {

Connection conn = DriverManager.getConnection(

"jdbc:mysql://Master:3306/sparkstreaming",

"root",

"778899");

connectionQueue.push(conn);

}

}

} catch(Exception e) {

e.printStackTrace();

}

return connectionQueue.poll();

}

public static void returnConnection(Connection conn) {connectionQueue.push(conn);}

}