本文介绍如何使用Elasticsearch、Logstash、Kibana（ELK）及Beats组件部署实时日志分析平台。从环境配置到各组件安装、配置及验证，提供了详细的步骤说明。
ELK（Elasticsearch, Logstash, Kibana），三个开源软件搭建的实时日志分析平台。
官网：
[url=https://www.elastic.co/products]https://www.elastic.co/products[/url]
版本：
[list][*]Elasticsearch 2.3.4 ： 日志索引、存储、查询
[*]Logstash 2.3.4 ： 日志收集、过滤、转发
[*]Kibana 4.5.3 ： 可视化查询 Elasticsearch 的数据
[*]Beats
Filebeat 1.2.3 ： 定时获取增量日志并转发给 Logstash
Topbeat 1.2.3 ： 定期收集系统信息并转发给 Logstash[/list]
前提：
设置hostname（做SSL证书需要）、安装Java（Elasticsearch和Logstash需要）
(1)导入GPG key
(2)添加Yum仓库
(3)安装Elasticsearch
确认Elasticsearch的索引
(4)安装Kibana
Kibana Dashboards
(5)安装Logstash
(6)安装Filebeat
(7)安装Topbeat
(8)收集Nginx日志
(9)收集Apache日志
参考：
[url=https://www.elastic.co/guide/index.html]https://www.elastic.co/guide/index.html[/url]
[url=https://www.digitalocean.com/community/tutorial_series/centralized-logging-with-logstash-and-kibana-on-centos-7]https://www.digitalocean.com/community/tutorial_series/centralized-logging-with-logstash-and-kibana-on-centos-7[/url]
官网：
[url=https://www.elastic.co/products]https://www.elastic.co/products[/url]
版本：
[list][*]Elasticsearch 2.3.4 ： 日志索引、存储、查询
[*]Logstash 2.3.4 ： 日志收集、过滤、转发
[*]Kibana 4.5.3 ： 可视化查询 Elasticsearch 的数据
[*]Beats
Filebeat 1.2.3 ： 定时获取增量日志并转发给 Logstash
Topbeat 1.2.3 ： 定期收集系统信息并转发给 Logstash[/list]
前提：
设置hostname（做SSL证书需要）、安装Java（Elasticsearch和Logstash需要）
(1)导入GPG key
rpm --import https://packages.elastic.co/GPG-KEY-elasticsearch(2)添加Yum仓库
# vi /etc/yum.repos.d/elasticsearch.repo
[elasticsearch-2.x]
name=Elasticsearch repository for 2.x packages
baseurl=https://packages.elastic.co/elasticsearch/2.x/centos
gpgcheck=1
gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
# vi /etc/yum.repos.d/kibana.repo
[kibana-4.5]
name=Kibana repository for 4.5.x packages
baseurl=http://packages.elastic.co/kibana/4.5/centos
gpgcheck=1
gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
# vi /etc/yum.repos.d/logstash.repo
[logstash-2.3]
name=Logstash repository for 2.3.x packages
baseurl=https://packages.elastic.co/logstash/2.3/centos
gpgcheck=1
gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
enabled=1
# vi /etc/yum.repos.d/beats.repo
[beats]
name=Elastic Beats Repository
baseurl=https://packages.elastic.co/beats/yum/el/$basearch
enabled=1
gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
gpgcheck=1(3)安装Elasticsearch
# yum -y install elasticsearch
# chkconfig --add elasticsearch
# chkconfig elasticsearch on
# rpm -qc elasticsearch
# vi /etc/elasticsearch/elasticsearch.yml
network.host: localhost
# service elasticsearch start
# service elasticsearch status
# netstat -nltp | grep -E '9200|9300'
# curl -X GET 'http://localhost:9200'确认Elasticsearch的索引
# curl http://localhost:9200/_cat/indices(4)安装Kibana
# yum -y install kibana
# chkconfig --add kibana
# chkconfig kibana on
# rpm -qc kibana
# vi /opt/kibana/config/kibana.yml
server.host: "localhost"
# service kibana start
# service kibana status
# netstat -nltp | grep 5601
# curl -X GET 'http://localhost:5601'Kibana Dashboards
# cd /usr/local/src/
# curl -L -O https://download.elastic.co/beats/dashboards/beats-dashboards-1.1.0.zip
# unzip beats-dashboards-1.1.0.zip
# cd beats-dashboards-1.1.0
# ./load.sh(5)安装Logstash
# yum -y install logstash
# chkconfig --add logstash
# chkconfig logstash on
# hostname
# cd /etc/pki/tls
# openssl req -subj '/CN=ELK_server_fqdn/' -x509 -days 3650 -batch -nodes -newkey rsa:2048 -keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt
# vi /etc/logstash/conf.d/02-beats-input.conf
input {
beats {
port => 5043
ssl => true
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}
# vi /etc/logstash/conf.d/10-syslog-filter.conf
filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}
# vi /etc/logstash/conf.d/30-elasticsearch-output.conf
output {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}
# service logstash configtest
# service logstash start
# service logstash status(6)安装Filebeat
# cp /tmp/logstash-forwarder.crt /etc/pki/tls/certs/logstash-forwarder.crt
# cd ~
# curl -O https://gist.githubusercontent.com/thisismitch/3429023e8438cc25b86c/raw/d8c479e2a1adcea8b1fe86570e42abab0f10f364/filebeat-index-template.json
# curl -X PUT 'http://localhost:9200/_template/filebeat?pretty' -d@filebeat-index-template.json
# yum -y install filebeat
# chkconfig --add filebeat
# chkconfig filebeat on
# vi /etc/filebeat/filebeat.yml
# egrep -v '^$|^#|^\s+#' /etc/filebeat/filebeat.yml
filebeat:
prospectors:
-
paths:
- /var/log/*.log
input_type: log
document_type: syslog
registry_file: /var/lib/filebeat/registry
output:
logstash:
hosts: ["localhost:5043"]
tls:
certificate_authorities: ["/etc/pki/tls/certs/logstash-forwarder.crt"]
shipper:
logging:
files:
rotateeverybytes: 10485760 # = 10MB
# service filebeat start
# service filebeat status
# curl -XGET 'http://localhost:9200/filebeat-*/_search?pretty'(7)安装Topbeat
# cd ~
# curl -O https://raw.githubusercontent.com/elastic/topbeat/master/etc/topbeat.template.json
# curl -XPUT 'http://localhost:9200/_template/topbeat' -d@topbeat.template.json
# yum -y install topbeat
# vi /etc/topbeat/topbeat.yml
# egrep -v '^$|^#|^\s+#' /etc/topbeat/topbeat.yml
output:
logstash:
hosts: ["localhost:5043"]
tls:
certificate_authorities: ["/etc/pki/tls/certs/logstash-forwarder.crt"]
# service topbeat start
# service topbeat status
# curl -XGET 'http://localhost:9200/topbeat-*/_search?pretty'(8)收集Nginx日志
# mkdir -p /opt/logstash/patterns
# chown logstash: /opt/logstash/patterns
# vi /opt/logstash/patterns/nginx
NGUSERNAME [a-zA-Z\.\@\-\+_%]+
NGUSER %{NGUSERNAME}
NGINXACCESS %{IPORHOST:clientip} %{NGUSER:ident} %{NGUSER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) (?:"(?:%{URI:referrer}|-)"|%{QS:referrer}) %{QS:agent}
# chown logstash: /opt/logstash/patterns/nginx
# vi /etc/filebeat/filebeat.yml
-
paths:
- /var/log/nginx/access.log
document_type: nginx-access
# service filebeat restart
# vi /etc/logstash/conf.d/11-nginx-filter.conf
filter {
if [type] == "nginx-access" {
grok {
match => { "message" => "%{NGINXACCESS}" }
}
}
}
# service logstash restart(9)收集Apache日志
# vi /etc/filebeat/filebeat.yml
-
paths:
- /var/log/apache2/access.log
document_type: apache-access
# service filebeat restart
# vi /etc/logstash/conf.d/12-apache.conf
filter {
if [type] == "apache-access" {
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
}
}
# service logstash restart参考：
[url=https://www.elastic.co/guide/index.html]https://www.elastic.co/guide/index.html[/url]
[url=https://www.digitalocean.com/community/tutorial_series/centralized-logging-with-logstash-and-kibana-on-centos-7]https://www.digitalocean.com/community/tutorial_series/centralized-logging-with-logstash-and-kibana-on-centos-7[/url]
扫一扫
1069
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
