ELK 通过 Logtsash 收集 Tomcat 和 Java 日志

一、服务器部署 Tomcat 服务

cd /apps
wget https://archive.apache.org/dist/tomcat/tomcat-8/v8.5.68/bin/apache-tomcat-8.5.68.tar.gz
tar xvf apache-tomcat-8.5.68.tar.gz
cd apache-tomcat-8.5.68

# 修改配置文件中，日志名称及后缀
vim conf/server.xml
prefix="tomcat_ access_ Log" suffix=". Log"

# 启动服务
./bin/catalina.sh run

尝试访问页面

二、收集 Tomcat 访问日志

2.1 Tomcat 日志转 JSON 格式

# 修改配置文件中，日志格式
vim conf/server.xml
        <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
               prefix="tomcat_access_log" suffix=".log"
               pattern="{&quot;clientip&quot;:&quot;%h&quot;,&quot;ClientUser&quot;:&quot;%l &quot;,&quot;authenticated&quot;:&quot;%u&quot;,&quot;AccessTime&quot;:&quot;%t&quot;,&quot;method&quot;:&quot;%r&quot;,&quot;status&quot;:&quot;%s&quot;,&quot;SendBytes&quot;:&quot;%b&quot;,&quot;Query?string&quot;:&quot;%q&quot;,&quot;partner&quot;:&quot;%{Referer}i&quot;,&quot;AgentVersion&quot;:&quot;%{User-Agent}i&quot;}"/>

# 启动服务
./bin/catalina.sh run

尝试访问页面

查看日志

root@logstash1:/apps/apache-tomcat-8.5.68# tail -f logs/tomcat_access_log.2021-08-28.log
10.0.0.1 - - [28/Aug/2021:05:17:51 +0000] "GET /bg-upper.png HTTP/1.1" 200 3103
10.0.0.1 - - [28/Aug/2021:05:17:51 +0000] "GET /favicon.ico HTTP/1.1" 200 21630
{"clientip":"10.0.0.1","ClientUser":"- ","authenticated":"-","AccessTime":"[28/Aug/2021:05:27:53 +0000]","method":"GET / HTTP/1.1","status":"200","SendBytes":"11156","Query?string":"","partner":"-","AgentVersion":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"}
{"clientip":"10.0.0.1","ClientUser":"- ","authenticated":"-","AccessTime":"[28/Aug/2021:05:27:53 +0000]","method":"GET /favicon.ico HTTP/1.1","status":"200","SendBytes":"21630","Query?string":"","partner":"http://10.0.0.37:8080/","AgentVersion":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36"}

2.2 验证日志是否 JOSN 格式

2.3 Tomcat 服务器安装 Logstash

将 logstash-7.12.1-amd64.deb 软件包传到 /usr/local/src 目录下，并进行安装

# 安装 elasticsearch
dpkg -i /usr/local/src/logstash-7.12.1-amd64.deb

2.4 编辑 Logstash 配置文件

root@logstash1:~# vim /etc/logstash/conf.d/test.conf

input {
  file {
    path => "/var/log/syslog"
    type => "systemlog"
    start_position => "beginning"
    stat_interval => "3 second"
  }
  file {
    path => "/var/log/vmware*.log"
    type => "vmwarelog"
    start_position => "beginning"
    stat_interval => "3 second"
  }
  file {
    path => "/apps/apache-tomcat-8.5.68/logs/tomcat_access_log.*.log"
    type => "tomcat-accesslog"
    start_position => "end"
    stat_interval => "3 second"
    codec => "json"
  }
}

output {
  if [type] == "systemlog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-testindex"
    }
  }
  if [type] == "vmwarelog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-vmwarelog-%{+YYYY.MM.dd}"
    }
  }
  if [type] == "tomcat-accesslog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-tomcat-accesslog-%{+YYYY.MM.dd}"
    }
  }
}

2.5 检测配置文件语法是否正确

root@logstash1:~# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf -t

2.6 启动服务，并验证

systemctl restart logstash.service

2.7 创建索引方便查询日志

三、收集 java 日志

3.1 被收集的文件

root@logstash1:~# ll /var/log/java/lck-cluster.log
-rw-r--r-- 1 root root 11600 Aug 28 10:09 /var/log/java/lck-cluster.log

3.2 编辑 Logstash 配置文件

    codec => multiline { 
      pattern => "^\["    # 正则表达式
      negate => true      # 匹配正则表达式
      what => "previous"  # 开头为 "[" 到另外一行 开头为 "["，称为一行
    }

root@logstash1:~# vim /etc/logstash/conf.d/test.conf
input {
  file {
    path => "/var/log/syslog"
    type => "systemlog"
    start_position => "beginning"
    stat_interval => "3 second"
  }
  file {
    path => "/var/log/vmware*.log"
    type => "vmwarelog"
    start_position => "beginning"
    start_position => "end"
    stat_interval => "3 second"
    codec => "json"
  }
  file {
    path => "/var/log/java/lck-cluster.log"
    type => "java-errorlog"
    start_position => "beginning"
    stat_interval => "3 second"
    codec => multiline {
      pattern => "^\["
      negate => true
      what => "previous"
    }
  }
}

output {
  if [type] == "systemlog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-testindex"
    }
  }
  if [type] == "vmwarelog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-vmwarelog-%{+YYYY.MM.dd}"
    }
  }
  if [type] == "tomcat-accesslog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-tomcat-accesslog-%{+YYYY.MM.dd}"
    }
  }
  if [type] == "java-errorlog" {
    elasticsearch {
      hosts => ["10.0.0.31:9200"]
      index => "logstash-lck-java-errorlog-%{+YYYY.MM.dd}"
    }
  }
}

3.3 检测配置文件语法是否正确

root@logstash1:~# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/test.conf -t

3.4 启动服务，并验证

systemctl restart logstash.service

3.5 创建索引方便查询日志