logstash日志收集时间问题解决

1. 效果示例

目标存储格式如下，方便阅读与ES查询，可直接按同格式字符串日期范围查询

        "_source" : {
          "c_date" : "2024-09-05T09:19:44"
        }

{
  "nlp-log" : {
    "aliases" : { },
    "mappings" : {
      "dynamic" : "strict",
      "properties" : {
        "c_date" : {
          "type" : "date",
          "format" : "date_hour_minute_second"
        }
      }
    }
  }
}

2. logstash日志收集改造

input {
  file {
    path => "/home/sfxs/logs/*.log"
  }
}
filter {
  grok {
        # 多个正则匹配，匹配日期为timestamp
        match => [
                "message", "\[%{TIMESTAMP_ISO8601:timestamp}].*【.*\s+\|\s+%{DATA:class}\s+\|\s+%{DATA:row}\s+\|\s+%{DATA:msg}\s+\|\s+%{DATA:address}】.*",
                "message", "\[%{TIMESTAMP_ISO8601:timestamp}].*\[%{DATA:address}\]"
        ]
  }
  mutate {
     # 将上面timestamp逗号替换为点，以便解析毫秒
     gsub => ["timestamp", ",", "."]
  }
  date {
        # 将timestamp已log收集文件中的格式进行匹配出
        match => ["timestamp", "yyyy-MM-dd HH:mm:ss.SSS"]
        target => "parsed_timestamp"  # 将解析后的时间存储到新字段parsed_timestamp
        timezone => "Asia/Shanghai"
  }
  mutate{
    # 将新字段parsed_timestamp添加到c_date ES目标字段中
    add_field => { "c_date" => "%{+YYYY-MM-dd'T'HH:mm:ss}" }
    # 删除无用的中间字段
    remove_field => ["host", "path", "@version", "message", "timestamp", "@timestamp", "parsed_timestamp"]
  }
}
output {
  elasticsearch {
    hosts => ["IP:9200"]
    index => "nlp-log"
  }
}

欢迎关注公众号 算法小生