logstash 采集nginx 日志

简单安装nginx 

[root@elkstack03 ~]# yum install -y nginx

## 主配置文件
[root@elkstack03 ~]# cat /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 4096;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    include /etc/nginx/conf.d/*.conf;
}

## 子配置文件
[root@elkstack03 ~]# vim /etc/nginx/conf.d/www.conf
  
server{
        listen 80;
        server_name _;
        root /code;
        index index.html;
}

[root@elkstack03 ~]# mkdir /code
[root@elkstack03 ~]# echo 'test nginx' > /code/index.html
[root@elkstack03 ~]# systemctl start nginx
 

将nginx日志改成Json格式

之前我们讲了tomcat日志，在企业中，修改格式需要与开发商量，但是nginx我们不需要，如果需要原来的格式日志，我们可以将日志输出两份，一份 main格式，一份Json格式

http{
        ...
    log_format json '{"@timestamp":"$time_iso8601",'
           '"host":"$server_addr",'
           '"ipaddr":"$remote_addr",'
           '"login_user":"$remote_user",'
           '"size":$body_bytes_sent,'
           '"responsetime":$request_time,'
           '"upstreamtime":"$upstream_response_time",'
           '"upstreamhost":"$upstream_addr",'
           '"http_host":"$host",'
           '"url":"$uri",'
           '"domain":"$host",'
           '"xff":"$http_x_forwarded_for",'
           '"referer":"$http_referer",'
           '"status":"$status"}';
        ...
}

[root@elkstack03 conf.d]# vim www.conf 
server{
        listen 80;
        server_name www.zls.com;
        root /code;
        index index.html;
        access_log  /var/log/nginx/www.zls.com_access_json.log  json;
}

[root@elkstack03 conf.d]# cat /etc/nginx/conf.d/blog.conf 
server{
    listen 80;
    server_name blog.zls.com;
    root /blog;
    index index.html;
    access_log  /var/log/nginx/blog.zls.com_access_json.log  json;
}
 

使用Logstash收集nginx日志

[root@elkstack03 conf.d]# cat /etc/logstash/conf.d/nginx_file_es.conf
input{
    file{
        type => "www.zls.com_access"
        path => "/var/log/nginx/www.zls.com_access_json.log"
        start_position => "beginning"
    }
        file{
                type => "blog.zls.com_access"
                path => "/var/log/nginx/blog.zls.com_access_json.log"
                start_position => "beginning"
        }

}

filter{
    json{
        source => "message"
        remove_field => ["message"]
    }
}

output{
    elasticsearch{
        hosts => ["10.0.0.81:9200"]
        index => "%{type}-%{+yyyy.MM.dd}"
        codec => "json"
    }
}

[root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash --path.data=/var/lib/logstash/nginx -f /etc/logstash/conf.d/nginx_file_es.conf &
 

kibana 展示