kube-prometheus数据持久化

前言

环境：centos7.9、kube-prometheus-release-0.10.zip
可参考这篇：https://blog.youkuaiyun.com/bzjoko/article/details/127213238
使用kube-prometheus安装的prometheus，默认是没有将数据持久化的，也是就是pod 重启数据就没了，本文将讲解kube-prometheus数据持久化。

kube-prometheus数据持久化

[root@matser manifests]# kubectl get pods -n monitoring 
NAME                                   READY   STATUS    RESTARTS   AGE
........................
prometheus-k8s-0                       2/2     Running   0          68m	#这个就是真正在跑prometheus-server的pod
prometheus-k8s-1                       2/2     Running   0          68m	#这个就是真正在跑prometheus-server的pod
[root@matser manifests]# 
#由此知prometheus的主服务其实就是使用pod启动的，其实就是prometheus-k8s-0、prometheus-k8s-1，而这种pod一看有序号就知道是sts创建的
[root@matser manifests]#  kubectl get sts -n monitoring 
NAME                READY   AGE
alertmanager-main   3/3     69m
prometheus-k8s      2/2     69m		#这个
[root@matser manifests]# 
# 但是我们查了一遍，发现当初创建kube-prometheus的yaml文件里面并没有创建StatefulSet资源，很奇怪。
# 后来百度发现，其实官方定义了一种叫做prometheus的资源，该资源创建了StatefulSet，如下：
 [root@matser manifests]#  kubectl get prometheus -n monitoring 
NAME   VERSION   REPLICAS   AGE
k8s    2.32.1    2          71m		#这种prometheus资源，由其创建的sts，创建k8s资源的文件是prometheus-prometheus.yaml
[root@matser manifests]# 

# prometheus-server 默认情况下没有配置数据持久化。所以我们需要做持久化存储。
#在线修改prometheus这种类型的资源，名称叫k8s（当然，你也可以修改prometheus-prometheus.yaml然后在apply亦可）
[root@matser manifests]#  kubectl  edit prometheus k8s  -n monitoring 
apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  creationTimestamp: "2022-11-16T08:50:10Z"
  generation: 2
..................
spec:
  alerting:
    alertmanagers:
    - apiVersion: v2
.....................
  podMonitorNamespaceSelector: {}
  podMonitorSelector: {}
  probeNamespaceSelector: {}
  probeSelector: {}
  replicas: 2
  resources:
    requests:
      memory: 400Mi
  retention: 15d		#加这个参数，表示prometheus数据保留的天数，默认会是1天而已
  ruleNamespaceSelector: {}
  ruleSelector: {}
  securityContext:
    fsGroup: 2000
    runAsNonRoot: true
    runAsUser: 1000
  serviceAccountName: prometheus-k8s
  serviceMonitorNamespaceSelector: {}
  serviceMonitorSelector: {}
  storage:										#加上下面这一段，一共9句
    volumeClaimTemplate:
      spec:
        accessModes:
        - ReadWriteOnce
        resources:
          requests:
            storage: 50M
        storageClassName: nfs-storageclass		#指定已有的存储类名称
  version: 2.32.1
[root@matser manifests]# 

# 然后sts的pod会被创建创建，去到nfs服务器就会看到目录已经创建了现在pvc，pv都创建好了
[root@matser manifests]# kubectl  get pvc,pv -n monitoring 
NAME                                                       STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS       AGE
persistentvolumeclaim/prometheus-k8s-db-prometheus-k8s-0   Bound    pvc-34541364-5d8b-4f24-a3bf-de6c8670aad8   50M        RWO            nfs-storageclass   19m
persistentvolumeclaim/prometheus-k8s-db-prometheus-k8s-1   Bound    pvc-3079da5e-c43a-49b3-a234-1381cf8690c7   50M        RWO            nfs-storageclass   19m

NAME                                                        CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                                           STORAGECLASS       REASON   AGE
persistentvolume/pvc-3079da5e-c43a-49b3-a234-1381cf8690c7   50M        RWO            Retain           Bound    monitoring/prometheus-k8s-db-prometheus-k8s-1   nfs-storageclass            19m
persistentvolume/pvc-34541364-5d8b-4f24-a3bf-de6c8670aad8   50M        RWO            Retain           Bound    monitoring/prometheus-k8s-db-prometheus-k8s-0   nfs-storageclass            19m
[root@matser manifests]# 

自此，prometheus持久化数据配置完成了。

grafana的持久化

[root@matser redis-cluster]# kubectl  get pods,deploy -n monitoring 
NAME                                       READY   STATUS    RESTARTS   AGE
pod/grafana-78d8cfccff-vd8mn               1/1     Running   0          110m
....................

NAME                                  READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/grafana               1/1     1            1           110m
............................
[root@matser redis-cluster]# 
#以上我们看到使用deployment部署了grafana，只有一个pod。

#手动创建grafana的pvc
[root@matser manifests]# cat > grafana-pvc.yaml <<'EOF'
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: grafana-pvc
  namespace: monitoring
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 100M
  storageClassName: nfs-storageclass
EOF
[root@matser manifests]# kubectl  apply  -f grafana-pvc.yaml 
persistentvolumeclaim/grafana-pvc created
[root@matser manifests]# kubectl  get pvc -n monitoring  | grep grafana
NAME                                 STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS       AGE
grafana-pvc                          Bound    pvc-6ad3f6a0-7ddf-47df-aa18-6a694b86fd00   100M       RWX            nfs-storageclass   6s
[root@matser manifests]# kubectl  get pv | grep grafana
pvc-6ad3f6a0-7ddf-47df-aa18-6a694b86fd00   100M       RWX            Retain           Bound    monitoring/grafana-pvc                          nfs-storageclass            49s
[root@matser manifests]# 

#编辑grafana的yaml文件，当然你可以在线edit它
[root@matser manifests]# vim grafana-deployment.yaml

      volumes:
#      - emptyDir: {}					#注释或删掉
#        name: grafana-storage			#注释或删掉
      - name: grafana-storage			#注意名字要和上面那句的名称相同
        persistentVolumeClaim:			
         claimName: grafana-pvc			#指定我们刚才创建的pvc
      - name: grafana-datasources		#这个原来的，不管
        secret:
          secretName: grafana-datasources

# 可以不做下面这一步，因为上面持久化之后不会丢失密码的变更
#为了固定grafana的登陆密码，可以添加环境变量
       readinessProbe:
          httpGet:
            path: /api/health
            port: http
        env:								#添加环境变量
        - name: GF_SECURITY_ADMIN_USER		#添加环境变量
          value: admin						#添加环境变量
        - name: GF_SECURITY_ADMIN_PASSWORD	#添加环境变量
          value: admin						#添加环境变量
        resources:
          limits:
            cpu: 200m
            memory: 200Mi

[root@matser manifests]# kubectl  replace  -f grafana-deployment.yaml

至此，grafana的数据持久化也做好了。可以看到后端的nfs服务器上已经有目录了。