log4j与flume整合配置(实现故障转移FailoverLog4jAppender)及问题总结

实现功能:
该功能重点在log4j端
启动两个相同的flume,log4j往flume1发数据,当flume1冗机后,可以继续往flume2中发送数据.达到故障转移,达到高可用

1、依赖jar包导入，通过maven管理，pom文件增加如下配置：

<!-- log4j输出日志到flume的appender依赖包 -->  
<dependency>  
    <groupId>org.apache.flume.flume-ng-clients</groupId>  
    <artifactId>flume-ng-log4jappender</artifactId>  
    <version>1.6.0</version>  
</dependency>  

<!-- log4j 依赖包-->  
<dependency>  
    <groupId>log4j</groupId>  
    <artifactId>log4j</artifactId>  
    <version>1.2.17</version>  
</dependency>  
<dependency>  
    <groupId>org.slf4j</groupId>  
    <artifactId>slf4j-api</artifactId>  
    <version>1.7.5</version>  
</dependency>  
<dependency>  
    <groupId>org.slf4j</groupId>  
    <artifactId>slf4j-log4j12</artifactId>  
    <version>1.7.5</version>  
</dependency>  

2、log4j配置文件：

#log4j输出到控制台
log4j.rootLogger=INFO,stdout,writelog,flume 
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%m%n

#log4j输出到flume单机模式配置  
log4j.appender.flume = org.apache.flume.clients.log4jappender.Log4jAppender
log4j.appender.flume.layout=org.apache.log4j.PatternLayout
log4j.appender.flume.Hostname = 192.168.109.133
log4j.appender.flume.Port = 6501
log4j.appender.flume.UnsafeMode = true

#log4j输出到flume故障转移方式配置  
log4j.appender.writelog = org.apache.flume.clients.log4jappender.FailoverLog4jAppender
log4j.appender.writelog.Hosts = m02:6501 m03:6501
log4j.appender.writelog.MaxAttempts = 2
log4j.appender.writelog.MaxIoWorkers = 2
log4j.appender.writelog.UnsafeMode = false
log4j.appender.writelog.layout=org.apache.log4j.PatternLayout

3.问题
采用改方式配置打印日志时，出现循环打印出“Using default maxIOWorkers”导致栈溢出问题：
在getProperties方法中增加对maxIOWorkers初始化：

props.setProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS,
(Runtime.getRuntime().availableProcessors() * 2)+"");

需要重写类(FailoverLog4jAppender).重新打包

修改的如下:

/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package org.apache.flume.clients.log4jappender;

import org.apache.commons.lang.StringUtils;
import org.apache.flume.FlumeException;
import org.apache.flume.api.RpcClientConfigurationConstants;
import org.apache.flume.api.RpcClientFactory;
import org.apache.flume.api.RpcClientFactory.ClientType;
import org.apache.log4j.Logger;
import org.apache.log4j.helpers.LogLog;
import org.apache.log4j.spi.LoggingEvent;

import java.util.Properties;

/**
 * author:edwardsbean
 */
public class FailoverLog4jAppender extends Log4jAppender {

    private String hosts;
    private String maxAttempts;
    private String maxIoWorkers;
    private boolean configured = false;

    public void setHosts(String hostNames) {
        this.hosts = hostNames;
    }

    public void setMaxAttempts(String maxAttempts) {
        this.maxAttempts = maxAttempts;
    }


    public void setMaxIoWorkers(String maxIoWorkers) {
        this.maxIoWorkers = maxIoWorkers;
    }


    Logger logger = Logger.getLogger(FailoverLog4jAppender.class);

    /**
     * 报错机制
     * @param event
     */
    @Override
    public synchronized void append(LoggingEvent event) {
        if (!configured) {
            String errorMsg = "Flume Log4jAppender not configured correctly! Cannot" +
                    " send events to Flume.";
            LogLog.error(errorMsg);
            if (getUnsafeMode()) {
                return;
            }
            //jie- 配置文件报错
            throw new FlumeException(errorMsg);
        }
        super.append(event);
    }


    /**
     * 重连机制
     * @throws FlumeException if the FailoverRpcClient cannot be instantiated.
     */
    @Override
    public void activateOptions() throws FlumeException {
        try {
            final Properties properties = getProperties(hosts, maxAttempts, maxIoWorkers,getTimeout());
            rpcClient = RpcClientFactory.getInstance(properties);
            if (layout != null) {
                layout.activateOptions();
            }
            configured = true;
        } catch (Exception e) {
            String errormsg = "RPC client creation failed! " + e.getMessage();
            LogLog.error(errormsg);
            logger.error("RPC client creation failed!" + "连接错误");
            if (getUnsafeMode()) {
                return;
            }
            throw new FlumeException(e);
        }

    }

    //配置必要的参数 说到底,就是配置文件
    /**
     * jie:ps自己重写获取配置文件信息,将多个网络端口配置起,是用FAILOVER(故障转移)
     * @param hosts        多个主机名
     * @param maxAttempts  主机数量
     * @param timeout      连接超时时间
     * @return
     * @throws FlumeException
     */
    private Properties getProperties(String hosts, String maxAttempts,String maxIoWorkers, long timeout) throws FlumeException {
        //获取多个主机名的hostsname
        if (StringUtils.isEmpty(hosts)) {
            throw new FlumeException("hosts must not be null");
        }


        Properties props = new Properties();
        //获取hostsname数组,按正则切分, \\s -> 空格   \\s+ ->  多个空格
        //datanode4:6501 datanode6:6501
        String[] hostsAndPorts = hosts.split("\\s+");
        StringBuilder names = new StringBuilder();
        for (int i = 0; i < hostsAndPorts.length; i++) {
            String hostAndPort = hostsAndPorts[i];
            String name = "h" + i;
            props.setProperty(RpcClientConfigurationConstants.CONFIG_HOSTS_PREFIX + name,
                    hostAndPort);
            names.append(name).append(" ");
        }
        props.put(RpcClientConfigurationConstants.CONFIG_HOSTS, names.toString());
        //客户端连接方式: CONFIG_CLIENT_TYPE -> client.type
        //DEFAULT_FAILOVER : 故障转移   (NettyAvroRpcClient(普通连接),FAILOVER(故障转移),LOADBALANCE(负载均衡),THRIFT(节约模式?))
        props.put(RpcClientConfigurationConstants.CONFIG_CLIENT_TYPE,
                ClientType.DEFAULT_FAILOVER.toString());

        if (StringUtils.isEmpty(maxAttempts)) {
            throw new FlumeException("hosts must not be null");
        }

        //一样
        props.put(RpcClientConfigurationConstants.CONFIG_MAX_ATTEMPTS, maxAttempts);
        //一样
        props.setProperty(RpcClientConfigurationConstants.CONFIG_CONNECT_TIMEOUT,
                String.valueOf(timeout));
        //一样
        props.setProperty(RpcClientConfigurationConstants.CONFIG_REQUEST_TIMEOUT,
                String.valueOf(timeout));
      /*  props.setProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS,
                String.valueOf(maxIoWorkers));   */
      //与slf4j框架bug 冲突
      //采用负载均衡方式配置打印日志时，出现循环打印出“Using default maxIOWorkers”导致栈溢出问题
      //在getProperties方法中增加对maxIOWorkers初始化
        props.setProperty(RpcClientConfigurationConstants.MAX_IO_WORKERS,
                (Runtime.getRuntime().availableProcessors() * 2)+"");


        return props;
    }
}

测试代码:

package log;

import org.apache.log4j.Logger;

import java.io.IOException;
import java.net.Socket;
import java.text.SimpleDateFormat;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * Created by Administrator on 2016/11/2.
 */
public class log4j {

    public static void main(String[] args) throws Exception {
        Logger logger = Logger.getLogger(log4j.class);
        //通过配置文件，将信息，发送到flume source中
        String url = "测试数据";
        System.out.println("开始发送");
        logger.info("log4j7测试数据" + " thread send message  on -");

        thread mTh1 = new thread("A");
        try {
            mTh1.start();
        } catch (Exception e) {
            logger.info("程序出现错误,骚等片刻100S:" + e.toString());
            try {
                mTh1.sleep(100000);
            } catch (InterruptedException e1) {
                e1.printStackTrace();
            }
        }

    }
}

class thread extends Thread {

    String message = "----->testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest" +
            "testtesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttesttest";

    Logger logger = Logger.getLogger(thread.class);
    Logger bpe1 = Logger.getLogger("vehicle_exam_event_his");
    Logger bpe2 = Logger.getLogger("vehicle_exam_score_his");
    Logger dse1 = Logger.getLogger("device_statushis");
    Logger dse2 = Logger.getLogger("device_statushis_invisible");
    Logger dse3 = Logger.getLogger("device_statushis_vehicle");
    Logger dse4 = Logger.getLogger("device_statushis_vehicle_invisible");
    Logger dse5 = Logger.getLogger("vehicle_behavior_point");
    Logger dse6 = Logger.getLogger("vehicle_behavior_process");

    private String name;

    public thread(String name) {
        this.name = name;
    }

    @Override
    public void run() {
        Socket socket = null;
        try {
            socket = new Socket("datanode4",41414);
        } catch (IOException e) {
            e.printStackTrace();
        }

        SimpleDateFormat time = new SimpleDateFormat("HH:mm:ss");
        String format = null;
        Long currtime = null;
        String test = "是否丢失 数据 log4j7测试数据";
        String mess = "thread send message  on -";
        String what = "- :  ---" + name + " for ";
        int i = 0;

        while (true) {
            try {
                if(socket.isConnected()){
                    currtime = System.currentTimeMillis();
                    format = time.format(currtime);
                    logger.info(test + mess + format + what + i++ + message);
                    System.out.println("41414端口开放");
                    sleep(2000);
                }else {
                    System.out.println("没有连接");
                    sleep(2000);
                }

            } catch (Exception e) {
                System.out.println("here");
                System.out.println(e);
            }

        }


    }
}