logstash自定义字段类型-优快云博客

本文介绍了如何在Logstash 7.17版本中自定义字段类型，通过编写conf文件和配置rocketmq.json及rocketmq.pattern文件来实现。这些文件分别位于指定目录中，为日志处理提供定制化的解析方案。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

说明

基于7.17版本

编写conf文件

rocketmq.json

# The # character at the beginning of a line indicates a comment. Use
# comments to describe your configuration.
input { 
        beats { 
                port => "5044"	#logstash监听端口
        }
}
# The filter part of this file is commented out to indicate that it is
# optional.
filter {
    grok {
        patterns_dir => "/etc/logstash/patterns"	#指定正则目录，用来在一些自定义的正则表达式，例如下面的LOG_TIME，METRIC_BROKER
        match => {"message" => ["%{LOG_TIME:logTime}\s%{WORD:level}\s-\s\[%{METRIC_BROKER:metric}]\s\[%{NOTSPACE:broker}] Stats In One Minute, SUM: %{INT:sum} TPS: %{NUMBER:tps}"]}
    }
    date {
      timezone => "Asia/Shanghai"	#解决时区问题
      match => ["logTime", "yyyy-MM-dd HH:mm:ss"] #匹配timestamp字段
      target => "@timestamp"  #将匹配到的数据写到@timestamp字段中
    }

}
output {
        #stdout { codec => rubydebug }
        #
        elasticsearch {
                hosts => [ "xxx.xxx.xxx.xxx:xxx" ]
                index => "rocketmq-%{[@metadata][version]}-%{+YYYY.MM.dd}" #索引生成规则
                user => "xx" #es username
                password => "xxx"	#es password
                template => "/etc/logstash/mappings/rocketmq.json"	#自定义mapping模板，用于定义字段类型，在kibana中会用到
                template_name => "rocketmq_template"
                template_overwrite => true
        }
}

rocketmq.pattern文件，存放于/etc/logstash/patterns

LOG_TIME \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}
METRIC_BROKER BROKER_PUT_NUMS|BROKER_GET_NUMS

rocketmq.json模板文件

{
  "index_patterns": "rocketmq-*",
  "settings": {
    "index.refresh_interval": "60s"
  },
  "mappings": {
    "properties": {
      "@timestamp": {
        "type": "date"
      },
      "@version": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      },
      "broker": {
        "type": "long",#自定义字段类型
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 256
          }
        }
      }
    }
  }
}