logstash 7.11.1 收集tomcat catalina日志

注：配置文件里配置是多数据源

catalina日志样例：

25-Apr-2021 00:38:29.431 INFO [Thread-8] org.apache.coyote.AbstractProtocol.pause Pausing ProtocolHandler ["http-nio-8080"]
25-Apr-2021 00:38:29.439 INFO [Thread-8] org.apache.catalina.core.StandardService.stopInternal Stopping service [Catalina]
25-Apr-2021 00:38:29.484 INFO [localhost-startStop-2] org.apache.catalina.core.StandardWrapper.unload Waiting for [14] instance(s) to be deallocated for Servlet [dispatcher]
25-Apr-2021 00:38:30.491 INFO [localhost-startStop-2] org.apache.catalina.core.StandardWrapper.unload Waiting for [14] instance(s) to be deallocated for Servlet [dispatcher]
25-Apr-2021 00:38:31.496 INFO [localhost-startStop-2] org.apache.catalina.core.StandardWrapper.unload Waiting for [14] instance(s) to be deallocated for Servlet [dispatcher]
25-Apr-2021 00:38:31.861 INFO [localhost-startStop-2] org.geoserver.GeoserverInitStartupListener.contextDestroyed Beginning GeoServer cleanup sequence
25-Apr-2021 00:38:31.862 INFO [localhost-startStop-2] org.geoserver.GeoserverInitStartupListener.contextDestroyed Unregistered JDBC driver org.hsqldb.jdbc.JDBCDriver@3b3a8a5a
25-Apr-2021 00:38:31.862 INFO [localhost-startStop-2] org.geoserver.GeoserverInitStartupListener.contextDestroyed Unregistered JDBC driver org.sqlite.JDBC@7f606890
25-Apr-2021 00:38:31.862 INFO [localhost-startStop-2] org.geoserver.GeoserverInitStartupListener.contextDestroyed Unregistered JDBC driver org.postgresql.Driver@55422263
25-Apr-2021 00:38:31.863 INFO [localhost-startStop-2] org.geoserver.GeoserverInitStartupListener.contextDestroyed Unregistered JDBC driver org.h3.Driver@1616cfe9
25-Apr-2021 00:40:21.812 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [geoserver] appears to have started a thread named [FileSystemWatcher-1] but has failed to stop it. This is very likely to create a memory leak. Stack trace of thread:
 sun.misc.Unsafe.park(Native Method)
 java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
 java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
 java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
 java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
 java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
 java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
 java.lang.Thread.run(Thread.java:748)
25-Apr-2021 00:40:21.815 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [geoserver] appears to have started a thread named [Thread-6] but has failed to stop it. This is very likely to create a memory leak. Stack trace of thread:
 java.lang.Thread.sleep(Native Method)
 org.geoserver.system.status.OSHISystemInfoCollector.lambda$new$0(OSHISystemInfoCollector.java:86)
 org.geoserver.system.status.OSHISystemInfoCollector$$Lambda$200/539131315.run(Unknown Source)
 java.lang.Thread.run(Thread.java:748)
25-Apr-2021 00:40:21.816 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [geoserver] appears to have started a thread named [EMF Reference Cleaner] but has failed to stop it. This is very likely to create a memory leak. Stack trace of thread:
 java.lang.Object.wait(Native Method)
 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144)
 java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165)
 org.eclipse.emf.common.util.CommonUtil$1ReferenceClearingQueuePollingThread.run(CommonUtil.java:70)
25-Apr-2021 00:40:21.820 WARNING [localhost-startStop-2] org.apache.catalina.loader.WebappClassLoaderBase.clearReferencesThreads The web application [geoserver] appears to have started a thread named [GuavaAuthCache-0-1] but has failed to stop it. This is very likely to create a memory leak. Stack trace of thread:
 sun.misc.Unsafe.park(Native Method)
 java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
 java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
 java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
 java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
 java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
 java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
 java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
 java.lang.Thread.run(Thread.java:748)

logstash.cnf配置文件

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
    file {
        type => "clickhouselog"
        path => "/home/its/logs/clickhouse/*.log"
        discover_interval => 10
        start_position => "beginning"
    }
    file {
        type => "tmlog"
        path => "/home/soft/app/tomcat/data/logs/catalina.2021-04-25.log"
		#如果遇到异常，以时间为规则区分一条数据。
        codec => multiline {
                        pattern => "^%{MONTHDAY}[./-]%{MONTH}[./-]%{YEAR}[- ]%{HOUR}:%{MINUTE}:%{SECOND}"
                        negate => true
                        what => "previous"
                }
	discover_interval => 10
        start_position => "beginning"
    }
}
filter {
	if [type] == "clickhouselog" {
		grok {
     		   match => ["message", "(?<time>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{HOUR}:%{MINUTE}:%{SECOND})"]
		}
		grok {
                  match =>{
                        "message" => "(?<logLevel>(?<=<).*?(?=>))"
                 }
                }
		grok {
                  match =>{
                        "message" => "(?<logContent>(?<=>)(.*)/?)"
                 }
                }
		ruby {
			code => "
			    event.set('collet_time', Time.parse(event.get('time'))+8*60*60)
			"
    		}
		mutate {
		  remove_field => [ "host","timestamp","time" ]
		}
	}
	
	if [type] == "tmlog" {
		grok{
			match => ["message", "(?<tmtime>%{MONTHDAY}[./-]%{MONTH}[./-]%{YEAR}[- ]%{HOUR}:%{MINUTE}:%{SECOND})"]
		}
		grok{
                        match => ["message", "(?<logLevel>%{LOGLEVEL})"]
                }
		 grok{
		        match =>{
                         "message" => "(?<logContent>(?<=])(.*)/?)"
                 	}

                }
		ruby {
                        code => "
                            event.set('collet_time', Time.parse(event.get('tmtime'))+8*60*60)
                        "
                }
		 mutate {
                  remove_field => [ "tmtime" ]
                }


	}
} 
output {
    elasticsearch {
    index => "log-%{+YYYY.MM.dd}"
    hosts => ["192.168.100.41:8200"]
    }
    stdout {codec => rubydebug}
}

 参考：

https://blog.youkuaiyun.com/sinat_23030553/article/details/89946707

https://www.cnblogs.com/stozen/p/5638369.html

http://www.sunrisenan.com/docs/elkstack/elk01.html