Nginx-AccessLog监控【Loki】

saikey0379

已于 2024-06-09 00:37:38 修改

阅读量1.6k

点赞数 53

文章标签： nginx accesslog httpcode monitor loki grafana promtail

于 2024-06-08 23:59:13 首次发布

本文链接：https://blog.youkuaiyun.com/weixin_41558221/article/details/139553159

版权

在这里插入图片描述

一、Loki部署Kubernetes

1.1.仓库配置

helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

1.2.配置生成

helm template loki grafana/loki-stack \
--namespace=loki-stack \
--set grafana.enabled=false \
--set loki.persistence.enabled=true \
--set loki.persistence.storageClassName=blk-std \
--set loki.persistence.size=20Gi > loki-stack.yaml

# 配置nodeSelector

配置备份：

---
# Source: loki-stack/charts/loki/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: loki
    chart: loki-2.16.0
    release: loki
    heritage: Helm
  annotations:
    {}
  name: loki
  namespace: loki-stack
automountServiceAccountToken: true
---
# Source: loki-stack/charts/promtail/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: loki-promtail
  namespace: loki-stack
  labels:
    helm.sh/chart: promtail-6.15.5
    app.kubernetes.io/name: promtail
    app.kubernetes.io/instance: loki
    app.kubernetes.io/version: "2.9.3"
    app.kubernetes.io/managed-by: Helm
---
# Source: loki-stack/charts/loki/templates/secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: loki
  namespace: loki-stack
  labels:
    app: loki
    chart: loki-2.16.0
    release: loki
    heritage: Helm
data:
  loki.yaml: YXV0aF9lbmFibGVkOiBmYWxzZQpjaHVua19zdG9yZV9jb25maWc6CiAgbWF4X2xvb2tfYmFja19wZXJpb2Q6IDBzCmNvbXBhY3RvcjoKICBzaGFyZWRfc3RvcmU6IGZpbGVzeXN0ZW0KICB3b3JraW5nX2RpcmVjdG9yeTogL2RhdGEvbG9raS9ib2x0ZGItc2hpcHBlci1jb21wYWN0b3IKaW5nZXN0ZXI6CiAgY2h1bmtfYmxvY2tfc2l6ZTogMjYyMTQ0CiAgY2h1bmtfaWRsZV9wZXJpb2Q6IDNtCiAgY2h1bmtfcmV0YWluX3BlcmlvZDogMW0KICBsaWZlY3ljbGVyOgogICAgcmluZzoKICAgICAgcmVwbGljYXRpb25fZmFjdG9yOiAxCiAgbWF4X3RyYW5zZmVyX3JldHJpZXM6IDAKICB3YWw6CiAgICBkaXI6IC9kYXRhL2xva2kvd2FsCmxpbWl0c19jb25maWc6CiAgZW5mb3JjZV9tZXRyaWNfbmFtZTogZmFsc2UKICBtYXhfZW50cmllc19saW1pdF9wZXJfcXVlcnk6IDUwMDAKICByZWplY3Rfb2xkX3NhbXBsZXM6IHRydWUKICByZWplY3Rfb2xkX3NhbXBsZXNfbWF4X2FnZTogMTY4aAptZW1iZXJsaXN0OgogIGpvaW5fbWVtYmVyczoKICAtICdsb2tpLW1lbWJlcmxpc3QnCnNjaGVtYV9jb25maWc6CiAgY29uZmlnczoKICAtIGZyb206ICIyMDIwLTEwLTI0IgogICAgaW5kZXg6CiAgICAgIHBlcmlvZDogMjRoCiAgICAgIHByZWZpeDogaW5kZXhfCiAgICBvYmplY3Rfc3RvcmU6IGZpbGVzeXN0ZW0KICAgIHNjaGVtYTogdjExCiAgICBzdG9yZTogYm9sdGRiLXNoaXBwZXIKc2VydmVyOgogIGdycGNfbGlzdGVuX3BvcnQ6IDkwOTUKICBodHRwX2xpc3Rlbl9wb3J0OiAzMTAwCnN0b3JhZ2VfY29uZmlnOgogIGJvbHRkYl9zaGlwcGVyOgogICAgYWN0aXZlX2luZGV4X2RpcmVjdG9yeTogL2RhdGEvbG9raS9ib2x0ZGItc2hpcHBlci1hY3RpdmUKICAgIGNhY2hlX2xvY2F0aW9uOiAvZGF0YS9sb2tpL2JvbHRkYi1zaGlwcGVyLWNhY2hlCiAgICBjYWNoZV90dGw6IDI0aAogICAgc2hhcmVkX3N0b3JlOiBmaWxlc3lzdGVtCiAgZmlsZXN5c3RlbToKICAgIGRpcmVjdG9yeTogL2RhdGEvbG9raS9jaHVua3MKdGFibGVfbWFuYWdlcjoKICByZXRlbnRpb25fZGVsZXRlc19lbmFibGVkOiBmYWxzZQogIHJldGVudGlvbl9wZXJpb2Q6IDBz
---
# Source: loki-stack/charts/promtail/templates/secret.yaml
apiVersion: v1
kind: Secret
metadata:
  name: loki-promtail
  namespace: loki-stack
  labels:
    helm.sh/chart: promtail-6.15.5
    app.kubernetes.io/name: promtail
    app.kubernetes.io/instance: loki
    app.kubernetes.io/version: "2.9.3"
    app.kubernetes.io/managed-by: Helm
stringData:
  promtail.yaml: |
    server:
      log_level: info
      log_format: logfmt
      http_listen_port: 3101
      
    
    clients:
      - url: http://loki:3100/loki/api/v1/push
    
    positions:
      filename: /run/promtail/positions.yaml
    
    scrape_configs:
      # See also https://github.com/grafana/loki/blob/master/production/ksonnet/promtail/scrape_config.libsonnet for reference
      - job_name: kubernetes-pods
        pipeline_stages:
          - cri: {}
        kubernetes_sd_configs:
          - role: pod
        relabel_configs:
          - source_labels:
              - __meta_kubernetes_pod_controller_name
            regex: ([0-9a-z-.]+?)(-[0-9a-f]{8,10})?
            action: replace
            target_label: __tmp_controller_name
          - source_labels:
              - __meta_kubernetes_pod_label_app_kubernetes_io_name
              - __meta_kubernetes_pod_label_app
              - __tmp_controller_name
              - __meta_kubernetes_pod_name
            regex: ^;*([^;]+)(;.*)?$
            action: replace
            target_label: app
          - source_labels:
              - __meta_kubernetes_pod_label_app_kubernetes_io_instance
              - __meta_kubernetes_pod_label_instance
            regex: ^;*([^;]+)(;.*)?$
            action: replace
            target_label: instance
          - source_labels:
              - __meta_kubernetes_pod_label_app_kubernetes_io_component
              - __meta_kubernetes_pod_label_component
            regex: ^;*([^;]+)(;.*)?$
            action: replace
            target_label: component
          - action: replace
            source_labels:
            - __meta_kubernetes_pod_node_name
            target_label: node_name
          - action: replace
            source_labels:
            - __meta_kubernetes_namespace
            target_label: namespace
          - action: replace
            replacement: $1
            separator: /
            source_labels:
            - namespace
            - app
            target_label: job
          - action: replace
            source_labels:
            - __meta_kubernetes_pod_name
            target_label: pod
          - action: replace
            source_labels:
            - __meta_kubernetes_pod_container_name
            target_label: container
          - action: replace
            replacement: /var/log/pods/*$1/*.log
            separator: /
            source_labels:
            - __meta_kubernetes_pod_uid
            - __meta_kubernetes_pod_container_name
            target_label: __path__
          - action: replace
            regex: true/(.*)
            replacement: /var/log/pods/*$1/*.log
            separator: /
            source_labels:
            - __meta_kubernetes_pod_annotationpresent_kubernetes_io_config_hash
            - __meta_kubernetes_pod_annotation_kubernetes_io_config_hash
            - __meta_kubernetes_pod_container_name
            target_label: __path__
      
      
    
    limits_config:
      
    
    tracing:
      enabled: false
---
# Source: loki-stack/templates/datasources.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: loki-loki-stack
  namespace: loki-stack
  labels:
    app: loki-stack
    chart: loki-stack-2.10.2
    release: loki
    heritage: Helm
    grafana_datasource: "1"
data:
  loki-stack-datasource.yaml: |-
    apiVersion: 1
    datasources:
    - name: Loki
      type: loki
      access: proxy
      url: "http://loki:3100"
      version: 1
      isDefault: true
      jsonData:
        {}
---
# Source: loki-stack/templates/tests/loki-test-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: loki-loki-stack-test
  labels:
    app: loki-stack
    chart: loki-stack-2.10.2
    release: loki
    heritage: Helm
data:
  test.sh: |
    #!/usr/bin/env bash

    LOKI_URI="http://${LOKI_SERVICE}:${LOKI_PORT}"

    function setup() {
      apk add -u curl jq
      until (curl -s ${LOKI_URI}/loki/api/v1/label/app/values | jq -e '.data[] | select(. == "loki")'); do
        sleep 1
      done
    }

    @test "Has labels" {
      curl -s ${LOKI_URI}/loki/api/v1/labels | \
      jq -e '.data[] | select(. == "app")'
    }

    @test "Query log entry" {
      curl -sG ${LOKI_URI}/api/prom/query?limit=10 --data-urlencode 'query={app="loki"}' | \
      jq -e '.streams[].entries | length >=1'
    }

    @test "Push log entry" {
      local timestamp=$(date +%s000000000)
      local data=$(jq -n --arg timestamp "${timestamp}" '{"streams": [{"stream": {"app": "loki-test"}, "values": [[$timestamp, "foobar"]]}]}')

      curl -s -X POST -H "Content-Type: application/json" ${LOKI_URI}/loki/api/v1/push --data-raw "${data}"

      curl -sG ${LOKI_URI}/loki/api/v1/query_range?limit=1 --data-urlencode 'query={app="loki-test"}' | \
      jq -e '.data.result[].values[][1] == "foobar"'
    }
---
# Source: loki-stack/charts/promtail/templates/clusterrole.yaml
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: loki-promtail
  labels:
    helm.sh/chart: promtail-6.15.5
    app.kubernetes.io/name: promtail
    app.kubernetes.io/instance: loki
    app.kubernetes.io/version: "2.9.3"
    app.kubernetes.io/managed-by: Helm
rules:
  - apiGroups:
      - ""
    resources:
      - nodes
      - nodes/proxy
      - services
      - endpoints
      - pods
    verbs:
      - get
      - watch
      - list
---
# Source: loki-stack/charts/promtail/templates/clusterrolebinding.yaml
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: loki-promtail
  labels:
    helm.sh/chart: promtail-6.15.5
    app.kubernetes.io/name: promtail
    app.kubernetes.io/instance: loki
    app.kubernetes.io/version: "2.9.3"
    app.kubernetes.io/managed-by: Helm
subjects:
  - kind: ServiceAccount
    name: loki-promtail
    namespace: loki-stack
roleRef:
  kind: ClusterRole
  name: loki-promtail
  apiGroup: rbac.authorization.k8s.io
---
# Source: loki-stack/charts/loki/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: loki
  namespace: loki-stack
  labels:
    app: loki
    chart: loki-2.16.0
    release: loki
    heritage: Helm
rules:
- apiGroups:      ['extensions']
  resources:      ['podsecuritypolicies']
  verbs:          ['use']
  resourceNames:  [loki]
---
# Source: loki-stack/charts/loki/templates/rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: loki
  namespace: loki-stack
  labels:
    app: loki
    chart: loki-2.16.0
    release: loki
    heritage: Helm
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: loki
subjects:
- kind: ServiceAccount
  name: loki
---
# Source: loki-stack/charts/loki/templates/service-headless.yaml
apiVersion: v1
kind: Service
metadata:
  name: loki-headless
  namespace: loki-stack
  labels:
    app: loki
    chart: loki-2.16.0
    release: loki
    heritage: Helm
    variant: headless
spec:
  clusterIP: None
  ports:
    - port: 3100
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
  selector:
    app: loki
    release: loki
---
# Source: loki-stack/charts/loki/templates/service-memberlist.yaml
apiVersion: v1
kind: Service
metadata:
  name: loki-memberlist
  namespace: loki-stack
  labels:
    app: loki
    chart: loki-2.16.0
    release: loki
    heritage: Helm
spec:
  type: ClusterIP
  clusterIP: None
  publishNotReadyAddresses: true
  ports:
    - name: http
      port: 7946
      targetPort: memberlist-port
      protocol: TCP
  selector:
    app: loki
    release: loki
---
# Source: loki-stack/charts/loki/templates/service.yaml
apiVersion: v1
kind: Service
metadata:
  name: loki
  namespace: loki-stack
  labels:
    app: loki
    chart: loki-2.16.0
    release: loki
    heritage: Helm
  annotations:
    {}
spec:
  type: ClusterIP
  ports:
    - port: 3100
      protocol: TCP
      name: http-metrics
      targetPort: http-metrics
  selector:
    app: loki
    release: loki
---
# Source: loki-stack/charts/promtail/templates/daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: loki-promtail
  namespace: loki-stack
  labels:
    helm.sh/chart: promtail-6.15.5
    app.kubernetes.io/name: promtail
    app.kubernetes.io/instance: loki
    app.kubernetes.io/version: "2.9.3"
    app.kubernetes.io/managed-by: Helm
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: promtail
      app.kubernetes.io/instance: loki
  updateStrategy:
    {}
  template:
    metadata:
      labels:
        app.kubernetes.io/name: promtail
        app.kubernetes.io/instance: loki
      annotations:
        checksum/config: 0f49fcd7a8fab642f9644e0a4d67b9f2bf9ce3e2cbf1f2ebfa7a301dbd59a7e0
    spec:
      serviceAccountName: loki-promtail
      enableServiceLinks: true
      securityContext:
        runAsGroup: 0
        runAsUser: 0
      containers:
        - name: promtail
          image: "docker.io/grafana/promtail:2.9.3"
          imagePullPolicy: IfNotPresent
          args:
            - "-config.file=/etc/promtail/promtail.yaml"
          volumeMounts:
            - name: config
              mountPath: /etc/promtail
            - mountPath: /run/promtail
              name: run
            - mountPath: /var/lib/docker/containers
              name: containers
              readOnly: true
            - mountPath: /var/log/pods
              name: pods
              readOnly: true
          env:
            - name: HOSTNAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          ports:
            - name: http-metrics
              containerPort: 3101
              protocol: TCP
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            readOnlyRootFilesystem: true
          readinessProbe:
            failureThreshold: 5
            httpGet:
              path: '/ready'
              port: http-metrics
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
      tolerations:
        - effect: NoSchedule
          key: node-role.kubernetes.io/master
          operator: Exists
        - effect: NoSchedule
          key: node-role.kubernetes.io/control-plane
          operator: Exists
      nodeSelector:
        pool: ops
      volumes:
        - name: config
          secret:
            secretName: loki-promtail
        - hostPath:
            path: /run/promtail
          name: run
        - hostPath:
            path: /var/lib/docker/containers
          name: containers
        - hostPath:
            path: /var/log/pods
          name: pods
---
# Source: loki-stack/charts/loki/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: loki
  namespace: loki-stack
  labels:
    app: loki
    chart: loki-2.16.0
    release: loki
    heritage: Helm
  annotations:
    {}
spec:
  podManagementPolicy: OrderedReady
  replicas: 1
  selector:
    matchLabels:
      app: loki
      release: loki
  serviceName: loki-headless
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: loki
        name: loki
        release: loki
      annotations:
        checksum/config: 6902b877b72340bffc62fa7bf5935695632390783b723365ecf4d2a6f6787c35
        prometheus.io/port: http-metrics
        prometheus.io/scrape: "true"
    spec:
      serviceAccountName: loki
      securityContext:
        fsGroup: 10001
        runAsGroup: 10001
        runAsNonRoot: true
        runAsUser: 10001
      initContainers:
        []
      containers:
        - name: loki
          image: "grafana/loki:2.6.1"
          imagePullPolicy: IfNotPresent
          args:
            - "-config.file=/etc/loki/loki.yaml"
          volumeMounts:
            - name: tmp
              mountPath: /tmp
            - name: config
              mountPath: /etc/loki
            - name: storage
              mountPath: "/data"
              subPath: 
          ports:
            - name: http-metrics
              containerPort: 3100
              protocol: TCP
            - name: grpc
              containerPort: 9095
              protocol: TCP
            - name: memberlist-port
              containerPort: 7946
              protocol: TCP
          livenessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          readinessProbe:
            httpGet:
              path: /ready
              port: http-metrics
            initialDelaySeconds: 45
          resources:
            {}
          securityContext:
            readOnlyRootFilesystem: true
          env:
      nodeSelector:
        pool: ops
      affinity:
        {}
      tolerations:
        []
      terminationGracePeriodSeconds: 4800
      volumes:
        - name: tmp
          emptyDir: {}
        - name: config
          secret:
            secretName: loki
  volumeClaimTemplates:
  - metadata:
      name: storage
      labels:
        {}
      annotations:
        {}
    spec:
      accessModes:
        - ReadWriteOnce
      resources:
        requests:
          storage: "20Gi"
      storageClassName: blk-std
---
# Source: loki-stack/templates/tests/loki-test-pod.yaml
apiVersion: v1
kind: Pod
metadata:
  annotations:
    "helm.sh/hook": test-success
  labels:
    app: loki-stack
    chart: loki-stack-2.10.2
    release: loki
    heritage: Helm
  name: loki-loki-stack-test
spec:
  containers:
  - name: test
    image: "bats/bats:1.8.2"
    imagePullPolicy: ""
    args:
    - /var/lib/loki/test.sh
    env:
    - name: LOKI_SERVICE
      value: loki
    - name: LOKI_PORT
      value: "3100"
    volumeMounts:
    - name: tests
      mountPath: /var/lib/loki
  restartPolicy: Never
  volumes:
  - name: tests
    configMap:
      name: loki-loki-stack-test

1.3.部署安装

kubectl create ns loki-stack
kubectl apply -f loki-stack.yaml

在这里插入图片描述

二、Promtail安装

构建参考：https://blog.youkuaiyun.com/weixin_41558221/article/details/139553307
下载地址：https://download.youkuaiyun.com/download/weixin_41558221/89412360

rpm -ivh promtail-3.0.0-1.el9.x86_64.rpm

cat << "EOF" > /etc/promtail/config.yml
server:
  http_listen_port: 9080
  grpc_listen_port: 0
 
positions:
  filename: /var/lib/promtail/positions.yaml
 
clients:
  - url: http://loki.example.com/loki/api/v1/push
 
scrape_configs:
- job_name: nginx
  static_configs:
  - targets:
      - hostname
    labels:
      job: nginx
      host: hostname
      agent: promtail
      __path__: /var/log/nginx/*log
EOF

systemctl enable promtail
systemctl start promtail

systemctl status promtail

三、Nginx-GeoIP支持

参考：https://blog.youkuaiyun.com/leiwuhen92/article/details/131379717

$geoip_country_code：两个字母的国家代码，例如“ RU”、“ US”。
$geoip_country_code3：三字母国家代码，例如“ RUS”、“ USA”。
$geoip_country_name：国家名称，例如“ Russian Federation”、“ United States”。
$geoip_area_code：电话区号（仅限美国）。由于相应的数据库字段已弃用，此变量可能包含过时的信息。
$geoip_city_continent_code：两个字母的大洲代码，例如“ EU”、“ NA”。
$geoip_city_country_code：两个字母的国家代码，例如“ RU”、“ US”。
$geoip_city_country_code3：三字母国家代码，例如“ RUS”、“ USA”。
$geoip_city_country_name：国家名称，例如“ Russian Federation”、“ United
States”。
$geoip_dma_code：根据Google AdWords API 中的geotargeting，美国的 DMA
区域代码（也称为“都市圈代码”）。
$geoip_latitude：纬度。 $geoip_longitude：经度。
$geoip_region：双符号国家地区代码（地区、领土、州、省、联邦土地等），例如“ 48”、“ DC”。
$geoip_region_name：国家地区名称（地区、领地、州、省、联邦土地等），例如“ Moscow City”、“
District of Columbia”。 $geoip_city：城市名称，例如“ Moscow”、“ Washington”。
$geoip_postal_code：邮政编码。 $geoip_org：组织名称，例如“墨尔本大学”

3.1.安装ngx_http_geoip_module

wget https://launchpad.net/geoip/main/1.4.8/+download/GeoIP-1.4.8.tar.gz
tar -zxvf GeoIP-1.4.8.tar.gz 
cd GeoIP-1.4.8/
./configure 
make 
make install

echo '/usr/local/lib' > /etc/ld.so.conf.d/geoip.conf
ldconfig

3.2.下载GeoIP.dat

下载链接：https://www.miyuru.lk/geoiplegacy
GeoLiteCity.dat下载链接：https://download.youkuaiyun.com/download/weixin_41558221/89412368
GeoLiteCity.dat.gz下载链接：https://download.youkuaiyun.com/download/weixin_41558221/89412373
GeoIP.dat.gz下载链接：https://download.youkuaiyun.com/download/weixin_41558221/89412374

# GeoIP.dat
wget https://dl.miyuru.lk/geoip/maxmind/country/maxmind.dat.gz
gunzip maxmind.dat.gz
mv maxmind.dat  /usr/local/openresty/nginx/conf/GeoIP.dat

# GeoLiteCity.dat
wget https://dl.miyuru.lk/geoip/maxmind/city/maxmind.dat.gz
gunzip  maxmind.dat.gz 
mv maxmind.dat /usr/local/openresty/nginx/conf/GeoLiteCity.dat

3.3.Openresty/Nginx

参考：https://blog.youkuaiyun.com/weixin_41558221/article/details/139156178

./configure \
...
...
--with-http_geoip_module

配置

cat /usr/local/openresty/nginx/conf/nginx.conf
...
http {
...
    geoip_country /usr/local/openresty/nginx/conf/GeoIP.dat;
    geoip_city /usr/local/openresty/nginx/conf/GeoLiteCity.dat; 

    log_format main escape=json '{'
        '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
        '"connection": "$connection", ' # connection serial number
        '"connection_requests": "$connection_requests", ' # number of requests made in connection
        '"pid": "$pid", ' # process pid
        '"request_id": "$request_id", ' # the unique request id
        '"request_length": "$request_length", ' # request length (including headers and body)
        '"remote_addr": "$remote_addr", ' # client IP
        '"remote_user": "$remote_user", ' # client HTTP username
        '"remote_port": "$remote_port", ' # client port
        '"time_local": "$time_local", '
        '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
        '"request": "$request", ' # full path no arguments if the request
        '"request_uri": "$request_uri", ' # full path and arguments if the request
        '"args": "$args", ' # args
        '"status": "$status", ' # response status code
        '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
        '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
        '"http_referer": "$http_referer", ' # HTTP referer
        '"http_user_agent": "$http_user_agent", ' # user agent
        '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
        '"http_host": "$http_host", ' # the request Host: header
        '"server_name": "$server_name", ' # the name of the vhost serving the request
        '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
        '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
        '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
        '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
        '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
        '"upstream_response_length": "$upstream_response_length", ' # upstream response length
        '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
        '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
        '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
        '"scheme": "$scheme", ' # http or https
        '"request_method": "$request_method", ' # request method
        '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
        '"pipe": "$pipe", ' # "p" if request was pipelined, "." otherwise
        '"gzip_ratio": "$gzip_ratio", '
        '"http_cf_ray": "$http_cf_ray",'
        '"geoip_country_code": "$geoip_country_code"'
        '}';
...