Flink分析k8s容器日志存入Mysql

需求：  分析k8s容器报错信息及相关日志

解决办法：flume采集k8s主机pod日志到kafka,flink消费kafka数据后入mysql

环境准备：k8s上部署flume ,kafka,mysql前面文章已介绍

工具：idea

编写Flink程序

package org.example

import org.apache.flink.api.common.functions.RichFunction
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.api.common.typeinfo.Types
import org.apache.flink.configuration.Configuration
import org.apache.flink.connector.jdbc.{JdbcConnectionOptions, JdbcExecutionOptions, JdbcSink, JdbcStatementBuilder}
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
import org.apache.flink.streaming.api.functions.sink.{RichSinkFunction, SinkFunction}
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer}
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}

import java.sql
import java.sql.{Connection, DriverManager, PreparedStatement}
import java.util.Properties
import scala.util.Random



/**
 * description: 
 *
 * @author Fc 
 * @date 2025/03/04 09:31:31 
 * @version 1.0.0 
 */
case class containerd(host: String,time: String,level: String,msg: String)

object Test {

  def main(args: Array[String]): Unit = {
    // 定义MySQL连接配置
    val jdbcUrl = "jdbc:mysql://192.168.33.123:30001/fc"
    val username = "root"
    val password = "iopYN!234"
    val driverName = "com.mysql.cj.jdbc.Driver"

    // 定义SQL插入语句
    val insertSql = "INSERT INTO containerd (host,time,level,msg) VALUES (?,?,?,?)"

    val properties = new Properties()
    properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.33.124:31092")
    properties.put(ConsumerConfig.GROUP_ID_CONFIG, "host")
    properties.put("request.timeout.ms", "120000")
    properties.put("metadata.max.age.ms", "30000")
//    properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.33.124:31092")
//    properties.setProperty(ProducerConfig.BATCH_SIZE_CONFIG, "10")
//    properties.setProperty(ProducerConfig.LINGER_MS_CONFIG, "50")
//    properties.setProperty(ProducerConfig.RETRIES_CONFIG, "2")
//    properties.setProperty(ProducerConfig.ACKS_CONFIG, "1")
//    properties.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
//      "org.apache.kafka.common.serialization.IntegerSerializer")
//    properties.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
//      "org.apache.kafka.common.serialization.StringSerializer")

//    val topic = "person_test"
//    val producer: KafkaProducer[Integer, String] = new KafkaProducer(properties);
//    val rand = new Random()
//
//    for (i <- 1 to 2000) {
//      val line: String = s"$i,Origami$i,${rand.nextInt(30) + 18},${if (rand.nextInt(10) >= 8) "Male" else "Female"}"
//      val record: ProducerRecord[Integer, String] =
//        new ProducerRecord[Integer, String](topic, 0, System.currentTimeMillis(), i, line)
//      print(record)
//      producer.send(record)
//      Thread.sleep(50 + rand.nextInt(500))
//    }
//
//    producer.close()
//    producer.flush()
val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment

    var arr1: String = ""

    //增加kafkasource  数据源   kafka的消费者
    val value: SingleOutputStreamOperator[String] = env.addSource(new FlinkKafkaConsumer("mytopic1", new SimpleStringSchema, properties))
      .uid("kafkaSource").name("kafkaSource").setParallelism(3)

    //value.print()
    //Mar  4 12:30:51 k8s-master03-125 containerd: time="2025-03-04T12:30:51+08:00" level=info msg="About to acquire host-wide IPAM lock." source="ipam_plugin.go:357"

      val result: SingleOutputStreamOperator[containerd] = value.map( data => {

      val s: Array[String] = data.split("containerd:")
      val s1: Array[String] = s(0).trim.split(" ")
        //time="2025-03-18T15:00:08+08:00" level=info msg="About to acquire host-wide IPAM lock." source="ipam_plugin.go:357"
      val s2: Array[String] = s(1).trim.split("level=")
      val s3: Array[String] = s2(0).trim.split("\"")
      //s3(1).dropRight(1).trim
        //info msg="About to acquire host-wide IPAM lock." source="ipam_plugin.go:357"
      val s4: Array[String] = s2(1).trim.split("msg=")
        //s4(1).trim
        if (s4(1).length <= 255){
          arr1 = s4(1)
        }
      print(s1(3).trim, s3(1).dropRight(1).trim, s4(0).trim, arr1+"\n")
      //print(s(0),s(1),s(2),s(3),s(4),s(5),s(6),s(7),s(8))
      //containerd(s(0),s(1),s(2),s(3),s(4),s(5),s(6),s(7))

      containerd(s1(3).trim, s3(1).trim, s4(0).trim, arr1)

    })
//      .returns(Types.TUPLE((Types.STRING,Types.STRING,Types.STRING,Types.STRING)))
//
//    result.addSink(JdbcSink.sink(
//      "INSERT INTO containerd (host,time,level,msg) VALUES (?,?,?,?)",
//      new JdbcStatementBuilder[containerd] {
//        override def accept(t: PreparedStatement, u: containerd): Unit = {
//          t.setString(1,u.host)
//          t.setString(1,u.time)
//          t.setString(1,u.level)
//          t.setString(1,u.msg)
//        }
//        },
//      new JdbcConnectionOptions.JdbcConnectionOptionsBuilder()
//        .withUrl("jdbc:mysql://192.168.33.124:30001/monitor?useSSL=false")
//        //.withDriverName("com.jdbc.jdbc.Driver")
//        .withDriverName("com.mysql.jdbc.Driver")
//        .withUsername("root")
//        .withPassword("iopYN!234")
//        .build()
//    ))

    result.addSink(new MyJdbcSinkFunc())


    env.execute()


  }

  /**
   *
   */
  class MyJdbcSinkFunc extends RichSinkFunction[containerd] {

    // 定义sql连接、预编译器
    var conn: Connection = _
    var insertStmt: PreparedStatement = _

    override def open(parameters: Configuration): Unit = {
      super.open(parameters)
       conn = DriverManager.getConnection("jdbc:mysql://192.168.33.124:30001/monitor?useSSL=false", "root", "iopYN!234")
      insertStmt= conn.prepareStatement("INSERT INTO containerd (host,time,level,msg) VALUES (?,?,?,?)")
    }


    override def invoke(value: containerd, context: SinkFunction.Context): Unit = {
      super.invoke(value, context)
      insertStmt.setString(1,value.host)
      insertStmt.setString(2,value.time)
      insertStmt.setString(3,value.level)
      insertStmt.setString(4,value.msg)
      insertStmt.executeUpdate()

    }

    override def close(): Unit = {
      super.close()
      insertStmt.close()
      conn.close()
    }

  }





}

创建Mysql数据库和表

create database monitor;
use monitor;
 
CREATE TABLE `containerd` (
  `host` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '??',
  `time` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '??',
  `level` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'level',
  `msg` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

捋清楚执行顺序：

Flume采集–>Flink执行程序–>向Flume监听的文件加入数据—>Mysql查询表