1.根据系统选择自己需要的版本下载(https://www.elastic.co/cn/downloads/logstash)

2.配置文件(新建logstash-7.3.1/config/logstash-test.conf)

3.配置文件如下
input{
kafka {
bootstrap_servers => "127.0.0.1:9092"
topics => "ip"
client_id => "ip_b"
group_id => "ip_a"
auto_offset_reset => "earliest"
consumer_threads => 2
# 从kafka获取json数据解析
codec => "json"
}
kafka {
bootstrap_servers => "127.0.0.1:9092"
topics => "ips"
client_id => "ips_b"
group_id => "ips_a"
auto_offset_reset => "earliest"
consumer_threads => 2
codec => "json"
}
}
filter{
grok {
# 移除不需要的字段
remove_field => ["@timestamp","@version"]
}
mutate {
# 增加一个request_time_format用于标识时间
add_field => {
"request_time_format" => ""
}
}
json {
source => "message"
}
# 根据标识字段判断使用不同的过滤规则
if[fields][log_source] == "ip" {
date {
# 这里是格式化时间2019-09-26 19:17:56
match => ["get_time" ,"yyyy-MM-dd HH:mm:ss"]
target => "request_time_format"
locale => "cn"
}
ruby {
# 真实的时间是比中国时间慢八个小时,这里把时间加上去
code => "event.set('request_time_format', event.get('request_time_format').time.localtime + 8*60*60)"
}
}
if[fields][log_source] == "ips" {
date {
# 格式化微秒时间UNIX_MS,具体请参看文档
match => [ "request_time" ,"yyyy-MM-dd HH:mm:ss,SSS" , "UNIX_MS" ]
# 过滤的时间格式赋值到request_time_format上
target => "request_time_format"
locale => "cn"
}
ruby {
code => "event.set('request_time_format', event.get('request_time_format').time.localtime + 8*60*60)"
}
}
}
output{
# elast参数配置
# 输出信息
if [action_method] == "ip" {
elasticsearch {
hosts => ["127.0.0.1:9200"]
# 索引的名称
index => "ip"
codec => line { format => "%{message}"}
}
}
# if[fields][log_source] == "ips" {
elasticsearch {
hosts => ["127.0.0.1:9200"]
# 索引的名称
index => "ips"
codec => line { format => "%{message}"}
}
}
# 打印输出传输到ElasticSearch
stdout{
codec => rubydebug
}
}
3.检查logstash的配置文件是否配置正确(logstash根目录下运行)
./bin/logstash -f ./config/logstash-test.conf --config.test_and_exit
4. 运行logstash根目录下运行)
./bin/logstash -f config/logstash-test.conf
5.Logstash配置含义