基于Prometheus的client_golang库实现应用的自定义可观测监控

前文使用java+graalvm实现原生应用可观测监控： prometheus client_java实现进程的CPU、内存、IO、流量的可观测，但是部分java依赖包使用了复杂的反射功能，Graalvm编译可能失败，无法在运行过程中获取反射类，需要手动添加反射类，比较麻烦，容易出现编译原生失败的情况。
这里介绍基于Prometheus的client_golang库实现应用可观测监控，使用Go语言实现，编译更简单。
官方教程：[https://prometheus.io/docs/guides/go-application/](Instrumenting a Go application for Prometheus)
client_golang： https://github.com/prometheus/client_golang

1. 安装client_golang库

当前最新版本v1.20.5

go get github.com/prometheus/client_golang/prometheus
go get github.com/prometheus/client_golang/prometheus/promauto
go get github.com/prometheus/client_golang/prometheus/promhttp

2. 编写可观测监控代码

实现进程的CPU、内存、IO、流量的可观测监控，部分实现代码如下：

package metrics

import (
	"bufio"
	"container/list"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/prometheus/client_golang/prometheus"
	"os/exec"
	"task-exporter/common"
	"task-exporter/models"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

type TopMetrics struct{}

var Registry = prometheus.NewRegistry()

var upGauge = prometheus.NewGauge(prometheus.GaugeOpts{
	Name: METRICS_UP_USE,
	Help: "help",
})

var processCpuGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
	Name: METRICS_CPU_USE,
	Help: "help",
}, []string{LABEL_PID, LABEL_USER, LABEL_CMD, LABEL_INCLUDE})

var processMemGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
	Name: METRICS_MEM_USE,
	Help: "help",
}, []string{LABEL_PID, LABEL_USER, LABEL_CMD, LABEL_INCLUDE})

var processResGauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
	Name: METRICS_RES_USE,
	Help: "help",
}, []string{LABEL_PID, LABEL_USER, LABEL_CMD, LABEL_INCLUDE})

func init() {
	upGauge.Set(1)
	Registry.MustRegister(upGauge)

	Registry.MustRegister(processCpuGauge)
	Registry.MustRegister(processMemGauge)
	Registry.MustRegister(processResGauge)
	fmt.Println("=====init=====")
}

func (top *TopMetrics) Run() {
	topData, _ := top.handle()

	processList := topData.ProcessList
	var elements []models.ProcessInfo
	for e := processList.Front(); e != nil; e = e.Next() {
		elements = append(elements, e.Value.(models.ProcessInfo))
	}

	elements = top.topCpuMem(elements, "cpu")
	processCpuGauge.Reset()
	for row, v := range elements {
		if row < common.ConfigInfo.Limit {
			processCpuGauge.WithLabelValues(v.Pid, v.User, v.Cmd, v.In).Set(v.Cpu)
		}
	}

	// MEM,RES
	elements = top.topCpuMem(elements, "res")
	processMemGauge.Reset()
	processResGauge.Reset()
	for row, v := range elements {
		if row < common.ConfigInfo.Limit {
			processMemGauge.WithLabelValues(v.Pid, v.User, v.Cmd, v.In).Set(v.Mem)
			processResGauge.WithLabelValues(v.Pid, v.User, v.Cmd, v.In).Set(v.Res)
		}

	}

}

func (top *TopMetrics) handle() (models.TopData, error) {
	var topData = models.TopData{ProcessList: list.New()}
	// 创建一个 Command 对象，指定要执行的外部命令及其参数
	cmd := exec.Command("top", "-c", "-b", "-n", "1", "-w", "512")

	// 获取命令的标准输出
	stdout, err := cmd.StdoutPipe()
	defer stdout.Close()
	if err != nil {
		fmt.Println("Error creating StdoutPipe:", err)
		return topData, errors.New("Error creating StdoutPipe:")
	}

	// 启动命令
	if err := cmd.Start(); err != nil {
		fmt.Println("Error starting command:", err)
		return topData, errors.New("Error starting command")
	}

	// 使用 bufio.Scanner 逐行读取命令的输出
	scanner := bufio.NewScanner(stdout)
	//fmt.Println("\n=====start=====")
	row := 1
	for scanner.Scan() {
		line := scanner.Text()
		//fmt.Println(row, "======:", line)
		if row == 1 {
			//top.handleUptime(line, row, &topData)
		} else if row == 2 {
			//top.handleTask(line, row, &topData)
		} else if row == 3 {
			//top.handleCpu(line, row, &topData)
		} else if row == 4 {
			//top.handleMem(line, row, &topData)
		} else if row == 5 {
			//top.handleSwap(line, row, &topData)
		} else if row >= 8 {
			top.handleProcess(line, row, &topData)
		}

		row++
	}

	// 等待命令执行完成
	if err := cmd.Wait(); err != nil {
		fmt.Println("Error waiting for top command to finish:", err)
		return topData, errors.New("exec: not started")
	}

	// 检查扫描过程中是否有错误
	if err := scanner.Err(); err != nil {
		fmt.Println("Error reading output:", err)
	}
	return topData, nil
}

func (top *TopMetrics) handleProcess(line string, row int, topData *models.TopData) {
	//                                       pid      user      PR       NI    VIRT      RES     SHR     S        %CPU        %MEM         TIME    COMMAND
	processRegex := regexp.MustCompile("(\\d+)\\s+(\\S+)\\s+\\S+\\s+\\S+\\s+\\S+\\s+(\\S+)\\s+\\S+\\s+\\S+\\s+([\\d.]+)\\s+([\\d.]+)\\s+\\S+\\s(.+)")
	processMatches := processRegex.FindStringSubmatch(line)
	if len(processMatches) == 7 {
		process := models.ProcessInfo{In: "n"}
		process.Pid = processMatches[1]
		process.User = processMatches[2]
		res := processMatches[3]
		if strings.HasSuffix(res, "m") {
			process.Res, _ = strconv.ParseFloat(strings.Trim(res, "m"), 64)
			process.Res = process.Res * 1024
		} else if strings.HasSuffix(res, "g") {
			process.Res, _ = strconv.ParseFloat(strings.Trim(res, "g"), 64)
			process.Res = process.Res * 1024 * 1024
		} else {
			process.Res, _ = strconv.ParseFloat(res, 64)
		}

		process.Cpu, _ = strconv.ParseFloat(processMatches[4], 64)
		process.Mem, _ = strconv.ParseFloat(processMatches[5], 64)
		process.Cmd = strings.TrimSpace(processMatches[6])
		// 检查别名
		process.Cmd = common.FilterAlias(process.Cmd)
		topData.ProcessList.PushBack(process)
	} else {
		strB, _ := json.Marshal(processMatches)
		fmt.Println(row, "======processRegex found:", string(strB))
	}
}

/**
 * 采集进程信息CPU\MEM\RES
 */
func (top *TopMetrics) topCpuMem(processes []models.ProcessInfo, ptype string) []models.ProcessInfo {
	len := len(processes)
	//fmt.Println("======topCpuMem len:", len)
	for i := 0; i < len-1; i++ {
		for r := i + 1; r < len; r++ {
			if ptype == "cpu" {
				if processes[r].Cpu > processes[i].Cpu {
					processes[i], processes[r] = processes[r], processes[i]
				}
			} else if ptype == "mem" {
				if processes[r].Mem > processes[i].Mem {
					processes[i], processes[r] = processes[r], processes[i]
				}
			} else if ptype == "res" {
				if processes[r].Res > processes[i].Res {
					//fmt.Println("======res change:", i, processes[i], r, processes[r])
					processes[i], processes[r] = processes[r], processes[i]
				}
			}
		}
	}
	return processes
}

main.go

package main

import (
	"flag"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"log"
	"net/http"
	"task-exporter/metrics"
	"time"
)

var addr = flag.String("listen-address", ":8080", "The address to listen on for HTTP requests.")

func main() {

	registry := metrics.Registry
	go func() {
		for {
			top := metrics.TopMetrics{}
			top.Run()

			//io := metrics.IoMetrics{}
			//io.Run()

			//net := metrics.NethogsMetrics{}
			//net.Run()

			//port := metrics.NetstatMetrics{}
			//port.Run()

			time.Sleep(15 * time.Second)
		}
	}()

	http.Handle(
		"/metrics", promhttp.HandlerFor(
			registry,
			promhttp.HandlerOpts{
				EnableOpenMetrics: true,
			}),
	)
	log.Printf("Listening on http://localhost%s/metrics", *addr)
	log.Fatal(http.ListenAndServe(*addr, nil))
}

3. 运行效果

Grafana展示效果参看前文。

4. jar、graalvm、golang编译运行版本对比

jar、graalvm、golang生成文件大小对比

jar、graalvm、golang运行占用CPU、内存对比

go编译版本:task_exporter-linux-x86.zip