alertmanager告警存储方案

背景：由于alertmanager没有历史告警存储功能。在工作中我们可能需要 收集历史告警信息进行数据分析，找到系统的瓶颈，从而提高稳定性

实现方法：go语言写一个 针对alertmanager的webhook 程序。收集告警信息进行过滤然后存入es中

具体代码如下

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"github.com/elastic/go-elasticsearch/v8"
	"github.com/elastic/go-elasticsearch/v8/esapi"
	"github.com/gin-gonic/gin"
	"log"
	"net/http"
	"reflect"
	"time"
)

type Alert struct {
	Status      string      `json:"status"`
	Labels      Labels      `json:"labels"`
	Annotations Annotations `json:"annotations"`
	StartsAt    string      `json:"startsAt"`
	EndsAt      string      `json:"endsAt"`
	//GeneratorURL string      `json:"generatorURL"`
	//Fingerprint  string      `json:"fingerprint"`
}

type Labels struct {
	AlertName string `json:"alertname"`
	Group     string `json:"group"`
	Instance  string `json:"instance"`
	Job       string `json:"job"`
	Severity  string `json:"severity"`
	Team      string `json:"team"`
}

type Annotations struct {
	Description string `json:"description"`
	//Summary     string `json:"summary"`
	//Value       string `json:"value"`
}

type AlertDetail struct {
	AlertName   string
	Group       string
	Severity    string
	Team        string
	Description string
	Time        float64
}

func HandleJSON(c *gin.Context) {
	var data struct {
		//Receiver          string      `json:"receiver"`
		Status string  `json:"status"`
		Alerts []Alert `json:"alerts"`
		//GroupLabels       Labels      `json:"groupLabels"`
		//CommonLabels      Labels      `json:"commonLabels"`
		//CommonAnnotations Annotations `json:"commonAnnotations"`
		//TruncatedAlerts   int         `json:"truncatedAlerts"`
	}

	if err := c.ShouldBindJSON(&data); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	//fmt.Println(data)

	if data.Status != "resolved" {
		c.JSON(http.StatusOK, gin.H{"message": "JSON data received and processed"})
		return
	}

	//配置es
	cfg := elasticsearch.Config{
		Addresses: []string{"http://127.0.0.1:9200"}, // 连接地址，根据自己的实际情况修改
	}

	// 创建 Elasticsearch 客户端
	client, err := elasticsearch.NewClient(cfg)
	if err != nil {
		fmt.Println("Error creating the client: ", err)
		return
	}

	for _, a := range data.Alerts {
		startsAt, _ := time.Parse(time.RFC3339, a.StartsAt)
		endsAt, _ := time.Parse(time.RFC3339, a.EndsAt)
		duration := endsAt.Sub(startsAt)
		durationInHours := duration.Hours()
		alertdetail := AlertDetail{
			AlertName:   a.Labels.AlertName,
			Group:       a.Labels.Group,
			Team:        a.Labels.Team,
			Description: a.Annotations.Description,
			Time:        durationInHours,
		}
		fmt.Println(alertdetail)
		// 将文档结构体转换为 JSON 字符串
		jsonStr, err := json.Marshal(alertdetail)
		if err != nil {
			fmt.Println("Error marshaling document: ", err)
			return
		}

		// 创建索引请求
		req := esapi.IndexRequest{
			Index: "alert",
			Body:  bytes.NewReader(jsonStr),
		}

		// 执行索引请求
		res, err := req.Do(context.Background(), client)
		if err != nil {
			fmt.Println("Error executing the request: ", err)
			return
		}
		defer res.Body.Close()

		// 检查响应状态码
		if res.IsError() {
			fmt.Printf("Error indexing document: %s", res.Status())
		} else {
			fmt.Println("Document indexed successfully")
		}

	}

	c.JSON(http.StatusOK, gin.H{"message": "JSON data received and processed"})
}

func SearchES(c *gin.Context) {
	//配置es
	cfg := elasticsearch.Config{
		Addresses: []string{"http://127.0.0.1:9200"}, // 连接地址，根据自己的实际情况修改
	}

	// 创建 Elasticsearch 客户端
	client, err := elasticsearch.NewClient(cfg)
	if err != nil {
		fmt.Println("Error creating the client: ", err)
		return
	}
	//获取es数据
	// 构造查询请求
	req := esapi.SearchRequest{
		Index: []string{"alert"}, // 索引名称
	}

	res, err := req.Do(context.Background(), client)
	if err != nil {
		log.Fatalf("Error executing search request: %s", err)
	}
	defer res.Body.Close()

	// 处理响应结果
	if res.IsError() {
		log.Fatalf("Search request returned error code %d", res.StatusCode)
	}
	respjson := res.String()
	fmt.Println(respjson)
	fmt.Println(reflect.TypeOf(respjson))
	c.JSON(http.StatusOK, respjson)

}

func main() {
	router := gin.Default()

	router.POST("/receive", HandleJSON)
	router.GET("/search", SearchES)

	// 启动服务器
	router.Run(":8080")
}