Logstash与ELK日志收集实践-优快云博客

一。直接上logstash的配置文件output_redis.conf

input {
        
		log4j {
			type => "log4j-json-service"
			port => 5560
		}
		log4j {
			type => "log4j-json-web"
			port => 5561
		}
		log4j {
			type => "log4j-json-mobile"
			port => 5562
		}
		log4j {
			type => "log4j-json-app"
			port => 5563
		}
		log4j {
			type => "log4j-json-manager"
			port => 5564
		}
		log4j {
			type => "log4j-json-pay"
			port => 5565
		}
		log4j {
			type => "log4j-json-search"
			port => 5566
		}
		log4j {
			type => "log4j-json-restful"
			port => 5567
		}
		udp {
		    type => "log4j-json-node"
			port => 5568
		 }
}

output {
    if [type] == "nginxlog"{
		elasticsearch {
			 hosts => "127.0.0.1"
			 index => "logstash-%{type}-%{+YYYY.MM.dd}"
			}	
	}
	if [type] == "log4j-json"{
		elasticsearch {
				 hosts => "127.0.0.1"
				 index => "logstash-log4j-%{+YYYY.MM.dd}"
				}
	}
	if [type] =~ "log4j-json-*"{
		elasticsearch {
				 hosts => "127.0.0.1"
				 index => "logstash-log4j-local-%{+YYYY.MM.dd}"
				}
	}
			
}

最新版的logstash默认给json格式化了，省去不少功夫，唯一郁闷的是没找到ip转geoip的方法，有人弄好了麻烦告知一声- -～

input模块里接收的是java log4j的日志，因为日志分模块输出，所有每个模块需要一个log4j{}来接收，udp是接收的node日志, nginx的删了.....

output模块是输出到elasticsearch的配置，根据不同的type，生成不同的index（索引）。

二。Java端和node端配置

java 这里只贴其中一个模块的日志，其他相同，只是Port改一下就好了，这里统一输出到内网ip:192.168.1.250, log4j.properties:

log4j.rootLogger=DEBUG, logstash
log4j.appender.logstash=org.apache.log4j.net.SocketAppender
log4j.appender.logstash.Port=5560
log4j.appender.logstash.RemoteHost=192.168.1.250
log4j.appender.logstash.ReconnectionDelay=60000
log4j.appender.logstash.LocationInfo=true

log4j.logger.freemarker=INFO
log4j.logger.java.sql.ResultSet=DEBUG
log4j.logger.java.sql.Statement=DEBUG
log4j.logger.java.sql.Connection=DEBUG
log4j.logger.java.sql.PreparedStatement=DEBUG
log4j.logger.org.springframework=INFO
log4j.logger.com.alibaba.dubbo=INFO
log4j.logger.org.apache.zookeeper=INFO
log4j.logger.com.101tec=INFO

node 这里统一输出到内网ip:192.168.1.250:

var helper = {};
exports.helper = helper;

var log4js = require('log4js');
var fs = require("fs");
var path = require("path");

// 加载配置文件
var objConfig = JSON.parse(fs.readFileSync("./config/log4js.json", "utf8"));

var objConfig = {
    "appenders": [
        {
            "type": "console",
            "category": "myLogger"
        },
        {
            "host": "192.168.1.250",
            "port": 5568,
            "type": "logstashUDP",
            "logType": "ppim", // Optional, defaults to 'category'
            "fields": {             // Optional, will be added to the 'fields' object in logstash
                "field1": "value1",
                "field2": "value2"
            },
            "layout": {
                "type": "pattern",
                "pattern": "%m"
            },
            "category": "myLogger"
        }
    ]
};

// 目录创建完毕，才加载配置，不然会出异常
log4js.configure(objConfig);
var logDebug = log4js.getLogger('myLogger');
var logInfo = log4js.getLogger('myLogger');
var logWarn = log4js.getLogger('myLogger');
var logErr = log4js.getLogger('myLogger');


helper.writeDebug = function(msg){
    if(msg == null)
        msg = "";
    logDebug.debug(msg);
};

helper.writeInfo = function(msg){
    if(msg == null)
        msg = "";
    logInfo.info(msg,'11','22');
};

helper.writeWarn = function(msg){
    if(msg == null)
        msg = "";
    logWarn.warn(msg);
};

helper.writeErr = function(msg, exp){
    if(msg == null)
        msg = "";
    if(exp != null)
        msg += "\r\n" + exp;
    logErr.error(msg);
};

// 配合express用的方法
exports.use = function(app) {
    //页面请求日志, level用auto时,默认级别是WARN
    app.use(log4js.connectLogger(logInfo, {level:'debug', format:':method :url'}));
};

三。经过上面两步，运行ELK，就可以在浏览器看到日志输出了。如果有疑问，可以查看文档，地址如下：

http://udn.yyuap.com/doc/logstash-best-practice-cn/index.html

特别说明一点，安装logstash插件的命令是 logstash-plugin 。比如查询插件列表:

四。kibana的dashboard使用

然后保存，在dashboard点+号，就可以加入到dashboard里了。其他的图表类似操作就可以，要理解x轴和y轴的作用

五。搜索

先上代码,这是去除静态文件访问的一个搜索语句，类似sql,

它的查询语法关键字跟lucene一样DSL查询(Query DSL)，下面只是抛砖引玉，各位自行探索

!document_uri:*css AND !document_uri: '-'   AND !document_uri:*.png AND !document_uri:*.jpg AND !document_uri:*.gif AND !document_uri:*.js  AND !document_uri:*.css

转载于:https://my.oschina.net/rockfate/blog/794369