logstash 处理nginx 访问日志

最新推荐文章于 2024-08-10 12:00:00 发布

转载最新推荐文章于 2024-08-10 12:00:00 发布 · 191 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/zhaoyangjian724/p/6199318.html

文章标签：

#运维 #数据库 #大数据

本文介绍了使用Logstash进行前端访问日志收集、过滤及输出到Elasticsearch的具体配置方法。通过两个配置文件logstash_frontend.conf和logstash_indexer.conf，详细展示了不同类型前端日志的处理流程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

[root@dr-mysql01 frontend]# cat logstash_frontend.conf 
input {
        file {
                type => "zj_frontend_access"
                path => ["/data01/applog_backup/zjzc_log/zj-frontend0*access*"]
        }
    
       file { 
                type => "wj_frontend_access" 
                path => ["/data01/applog_backup/winfae_log/wj-frontend0*access*"] 
        } 

 
}
filter {
    grok {
        match => {
            "message" => "%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?<http_referer>\S+)\" \"(?<http_user_agent>\S+)\" \"(?<http_x_forwarded_for>\S+)\""
        }
    }   
}



output {
     if [type] == "zj_frontend_access" { 
        redis {
                host => "192.168.32.67"
                data_type => "list"
                key => "zj_frontend_access:redis"
                port=>"6379"
                password => "1234567"
        }
}
      else if [type] == "wj_frontend_access"{
       redis { 
                host => "192.168.32.67" 
                data_type => "list" 
                key => "wj_frontend_access:redis" 
                port=>"6379" 
                password => "1234567" 
        } 
}
}
  
  
  
  
  [root@dr-mysql01 frontend]# cat logstash_indexer.conf 
input {

        redis {
                host => "192.168.32.67"
                data_type => "list"
                key => "zj_frontend_access:redis"
                password => "1234567"
                port =>"6379"
        }


        redis {
                host => "192.168.32.67"
                data_type => "list"
                key => "wj_frontend_access:redis"
                password => "1234567"
                port =>"6379"
        }


}
output {
      if   [type] == "zj_frontend_access"{ 
        elasticsearch {
                hosts => "192.168.32.80:9200"
                index => "logstash-zjzc-frontend-%{+YYYY.MM.dd}"
        }
		stdout {
			codec => rubydebug
		}
      }  
      else if  [type] == "wj_frontend_access"{
      elasticsearch {
                hosts => "192.168.32.81:9200"
                index => "logstash-wj-frontend-%{+YYYY.MM.dd}"
        }
                stdout {
                        codec => rubydebug
                } 
  
  }

}

转载于:https://www.cnblogs.com/zhaoyangjian724/p/6199318.html