ELk日志系统搭建

ELk日志系统搭建

转至元数据结尾

• 
  

#安装JDK

cd /data/soft/

tar zxf jdk-7u71-linux-x64.tar.gz

mkdir -p /data/mexue_apps/jdk/

/bin/cp -ap jdk1.7.0_71/* /data/mexue_apps/jdk/

cat > /etc/profile.d/java.sh << 'EOF'

export JAVA_HOME=/data/mexue_apps/jdk

export JRE_HOME=/data/mexue_apps/jdk/jre

export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib

export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin

EOF

source /etc/profile.d/java.sh

java -version

 

#源码下载

mkdir /data/soft

cd /data/soft

wget  https://download.elastic.co/logstash/logstash/packages/centos/logstash-2.4.0.noarch.rpm

wget  https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/rpm/elasticsearch/2.4.1/elasticsearch-2.4.1.rpm

wget  https://download.elastic.co/kibana/kibana/kibana-4.6.1-x86_64.rpm

yum -y localinstall logstash-2.4.0.noarch.rpm

yum -y localinstall elasticsearch-2.4.1.rpm

yum -y localinstall kibana-4.6.1-x86_64.rpm

 

#客户端日志搜集logstash配置

cat > /etc/logstash/conf.d/logstash_agent.conf << EOF

input {

        file {

                type => "web_main04_interfacemonitor"

                path => ["/data/mexue_logs/mexue*/interface_monitor.log"]

                codec => json

        }

        file {

                type => "web_main04_catalina.out"

                path => ["/data/mexue_apps/mexue*/tomcat/logs/Exception.log"]

                codec => multiline {

                    pattern => "^*Exception:"

                    negate => true

                    what => "previous"

                }

        }

}

 

output {

        kafka { 

                bootstrap_servers => "10.174.8.98:9092"

                topic_id => "nginx-access"

                compression_type => "snappy" 

        }

}

EOF

 

#服务端logstash配置

cat > /etc/logstash/conf.d/logstash_indexer.conf << 'EOF'

input {

    kafka {

        zk_connect => "10.174.8.98:2181"

        topic_id => "nginx-access"

        codec => json

        type => "nginx-access"

        consumer_threads => 64

        decorate_events => false

    }

}

 

output {

    if [message] =~ /Exception/ {

        if [message] =~ /OutOfMemoryError/ {

            exec {

                command => "python /scripts/MyEmail2/pyemail.py  logusers@mexue.com '%{type} OutOfMemoryError' '%{message}'"

            }

        }

 

        else if [message] =~ /IllegalStateException/ {

            exec {

                command => "python /scripts/MyEmail2/pyemail.py  logusers@mexue.com '%{type} IllegalStateException' '%{message}'"

            }

        }

 

        else if [message] =~ /NullPointerException/ {

            if [message] =~ /APR error: -32/ {

            }

 

            else if [message] =~ /APR error: -104/ {

            }

 

            else {

                exec {

                    command => "python /scripts/MyEmail2/pyemail.py  logusers@mexue.com '%{type} NullPointerException' '%{message}'"

                }

            }

        }

 

        else {

            exec {

                    command => "python /scripts/MyEmail2/pyemail.py  logusers@mexue.com '%{type} UnknownException' '%{message}'"

             }     

        }

    }

 

    elasticsearch {

        hosts => ["10.174.8.98:9200"]

        index => "logstash-%{type}-%{+YYYY.MM.dd}"

        document_type => "%{type}"

        workers => 4

        flush_size => 20000

        idle_flush_time => 10

        template_overwrite => true

    }

}

EOF

 

 

#客户端logstash使用redis的配置

output {

        redis {

                host => "10.174.8.98"

                port => "6379"

                data_type => "list"

                key => "logstash:redis"

        }

}

 

#服务端logstash使用redis的配置

input {

        redis {

                host => "192.168.0.112"

                port => "6379"

                data_type => "list"

                key => "logstash:redis"

                type => "redis-input"

                codec => "json"

           threads => 5

        }

}

 

 

#elasticsearch和kibana的配置文件

 

ll /etc/elasticsearch/elasticsearch.yml

ll /opt/kibana/config/kibana.yml

客户端安装：

cat > /etc/logstash/conf.d/logstash_agent.conf << 'EOF'

input {

        file {

                type => "web_mexuemain_web02_catalina.out"

                path => ["/data/mexue_apps/mexue*/tomcat/logs/Exception.log"]

                codec => multiline {

                    pattern => "^*Exception:"

                    negate => true

                    what => "previous"

                }

        }

}

output {

        redis {

                host => "10.174.9.246"

                port => "6379"

                data_type => "list"

                key => "logstash:redis"

        }

}

EOF

/etc/init.d/logstash configtest

touch /data/mexue_apps/mexueMain/tomcat/logs/Exception.log

echo "*/1 * * * * /bin/bash /scripts/logrotate.sh" >> /var/spool/cron/root

cat > /scripts/logrotate.sh << 'EOF'

#!/bin/bash  

log="/data/mexue_apps/mexue*/tomcat/logs/catalina.out"

size=`ls -lrt $log | cut -d " " -f 5`

if [ $size -ge 30000000 ]; then

    egrep "at |Exception"  $log >> /data/mexue_apps/mexue*/tomcat/logs/Exception.log

    /usr/sbin/logrotate -f /scripts/tomcat;

fi

EOF

cat > /scripts/tomcat << 'EOF'

/data/mexue_apps/mexue*/tomcat/logs/catalina.out {

        rotate 300

        missingok

        notifempty

        copytruncate

        size = 30M

}

EOF

/etc/init.d/logstash start

ps -ef | grep java

###########注意事项

logstash每次启动防止从之前记录位置读取

可以删除记录日志位置的文件

ll /var/lib/logstash/.sincedb_*

rm -rf /var/lib/logstash/.sincedb_*

ll /var/lib/logstash/.sincedb_*

/etc/init.d/logstash start