微服务日志输出如何处理？

Api日志

为了方便日志跟踪，api日志一般会要求显示请求路径，方法类型，执行时间，请求的入参，请求头，以及请求响应。如果是微服务之间的调用，需要设置全局的请求id用于标识整个链路请求。基于控制层的拦截器实现，定义LogFilter实现日志的输出。微服务之间请求的全局id通过header传递，使用HttpServlet的装饰类进行扩展header的设置。源码如下：

public class LogFilter implements Filter {

    private static final Set<String> LOCAL_HOST_SET = Sets.newHashSet(LOCAL_HOST_IP, LOCAL_HOST);
    @Override
    public void init(FilterConfig filterConfig) throws ServletException {
    }

    @Override
    public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {

        if (this.isLocalhostRequest(request.getLocalAddr())
                || this.isFile(request.getContentType())) {

            chain.doFilter(request, response);
            return;
        }

        HttpServletRequest httpServletRequest = (HttpServletRequest) request;
        HttpServletResponse httpServletResponse = (HttpServletResponse) response;
        RequestLogWrapper requestLogWrapper = new RequestLogWrapper(httpServletRequest);
        ResponseLogWrapper responseLogWrapper = new ResponseLogWrapper(httpServletResponse);
        // 在header中定义全局的请求id
        String requestId;
        if (StringUtils.isBlank(requestId = requestLogWrapper.getHeader(REQUEST_ID))) {

            requestId = UUID.randomUUID().toString();
            // 重新设置requestId
            requestLogWrapper.setHeader(REQUEST_ID, requestId);
        }

        // 定义服务执行开始结束时间
        LocalDateTime startAt = DateUtils.now();
        chain.doFilter(requestLogWrapper, responseLogWrapper);
        LocalDateTime finishAt = DateUtils.now();

        HttpLog httpLog = HttpLogUtil.obtainResponseLog(requestLogWrapper, responseLogWrapper, startAt, finishAt, requestId);
        ServletOutputStream outputStream = response.getOutputStream();
        outputStream.write(responseLogWrapper.getResponseData());
        outputStream.flush();
        outputStream.close();

        // 打印response
        log.info("----------------------http log -------------------------");
        String requestParams;
        String headers;
        log.info("requestId: [{}],methodName: [{}],url: [{}],startAt: [{}],finishAt: [{}],duration: [{}ms],status: [{}],headers:[{}],request: [{}],response: [{}]",
                httpLog.getRequestId(),
                httpLog.getMethodName(),
                httpLog.getUrl(),
                DateUtils.formatDate(httpLog.getStartAt(), DateUtils.DATE_TIME_MILE),
                DateUtils.formatDate(httpLog.getFinishAt(), DateUtils.DATE_TIME_MILE),
                httpLog.getDuration(),
                httpLog.getStatus(),
                StringUtils.isNotBlank(headers = httpLog.getHeaders()) ? HttpLogUtil.subStringIfRequired(headers) : StringUtils.EMPTY,
                StringUtils.isNotBlank(requestParams = httpLog.getRequest()) ? HttpLogUtil.subStringIfRequired(requestParams) : StringUtils.EMPTY,
                HttpLogUtil.subStringIfRequired(httpLog.getResponse()));
    }

    private boolean isLocalhostRequest(String localAddr) {

        for (String localHost : LOCAL_HOST_SET) {
            if (localHost.equalsIgnoreCase(localAddr)) {
                return true;
            }
        }
        return false;
    }

    private boolean isFile(String contentType){
        if(StringUtils.isNotEmpty(contentType)&&contentType.contains(FILE_CONTENT_TYPE)){
            return true;
        }
        return false;
    }

    @Override
    public void destroy() {
    }
}



public class RequestLogWrapper extends HttpServletRequestWrapper {

    /**
     * 所有header参数集合,在HttpServletRequest内部，header名称已经全部给转换小写
     */
    private final Map<String, String> headerMap;
    /**
     * 请求body
     */
    private final byte[] body;

    public RequestLogWrapper(HttpServletRequest request) {
        super(request);
        this.body = this.initBody(request);
        this.headerMap = this.initHeaderMap(request);
    }

    private byte[] initBody(HttpServletRequest request) {

        String bodyString;
        return StringUtils.isNotBlank(bodyString = this.getRequestBodyString(request))
                ? bodyString.getBytes(Charset.defaultCharset()) : new byte[]{};
    }

    private Map<String, String> initHeaderMap(HttpServletRequest request) {

        Map<String, String> headerMap = new HashMap<>();
        Enumeration<String> headerNames = request.getHeaderNames();
        while (headerNames.hasMoreElements()) {
            String headerName = headerNames.nextElement();
            headerMap.put(headerName, request.getHeader(headerName));
        }
        return headerMap;
    }

    public void setHeader(String header, String value) {

        if (StringUtils.isBlank(header) || StringUtils.isBlank(value)) {
            return;
        }
        this.headerMap.put(header, value);
    }

    @Override
    public String getHeader(String name) {
        return StringUtils.isNotBlank(name) ? this.getHeaderIgnoreCase(name) : null;
    }

    /**
     * header名称已经全部给转换小写，现根据源格式获取，如果获取不到再转换为小写格式
     */
    private String getHeaderIgnoreCase(String name) {
        String value;
        if (StringUtils.isNotBlank(value = this.headerMap.get(name))) {
            return value;
        }
        return this.headerMap.get(name.toLowerCase());
    }

    /**
     * 封装request id 的header
     */
    @Override
    public Enumeration<String> getHeaderNames() {
        return Collections.enumeration(this.headerMap.keySet());
    }

    private String getRequestBodyString(HttpServletRequest request) {

        StringBuffer sb = new StringBuffer();
        BufferedReader bufferedReader = null;
        try {

            ServletInputStream inputStream = request.getInputStream();
            bufferedReader = new BufferedReader(new InputStreamReader(inputStream, Charset.defaultCharset()));
            String lineStr;
            while (StringUtils.isNotBlank(lineStr = bufferedReader.readLine())) {
                sb.append(lineStr);
            }
        } catch (IOException e) {

            log.error("log filter  parse request exception!", e);
            throw new RuntimeException(e);
        } finally {
            if (Objects.nonNull(bufferedReader)) {
                try {
                    bufferedReader.close();
                } catch (IOException e) {

                    log.error("log filter  close inputStream exception!", e);
                    throw new RuntimeException(e);
                }
            }
        }
        return sb.toString();
    }

    /**
     * 获取request 的json的请求参数
     */
    public String getBodyJson() {
        return ArrayUtils.isNotEmpty(body) ? new String(body, Charset.defaultCharset()) : StringUtils.EMPTY;
    }

    @Override
    public ServletInputStream getInputStream() {

        final ByteArrayInputStream inputStream = new ByteArrayInputStream(body);
        return new ServletInputStream() {
            @Override
            public int read() throws IOException {
                return inputStream.read();
            }

            @Override
            public boolean isFinished() {
                return false;
            }

            @Override
            public boolean isReady() {
                return false;
            }

            @Override
            public void setReadListener(ReadListener readListener) {
            }
        };
    }

    @Override
    public BufferedReader getReader() throws IOException {
        return new BufferedReader(new InputStreamReader(this.getInputStream()));
    }
}

 

Feign日志

在springcloud框架中，通常使用feign实现服务的相互之间的调用。Feign输出日志只会答应出debugger级别的日志，并且日志的类型分为如下四类：

• NONE：不打印日志
• BASIC：只打印基本信息，包括请求方法、请求地址、响应状态码、请求时长
• HEADERS：在 BASIC 基础信息的基础之上，增加请求头、响应头
• FULL：打印完整信息，包括请求和响应的所有信息。

自定义Slf4jFeignLogger实现info级别的日志输出，并且根据日志类别打印不同类型日志。源码如下：

public class Slf4jFeignLogger extends feign.Logger {

    private final Logger logger;

    public Slf4jFeignLogger(Class<?> clazz) {
        this(LoggerFactory.getLogger(clazz));
    }

    public Slf4jFeignLogger(String name) {
        this(LoggerFactory.getLogger(name));
    }

    public Slf4jFeignLogger() {
        this(feign.Logger.class);
    }

    public Slf4jFeignLogger(Logger logger) {
        this.logger = logger;
    }


    @Override
    protected void log(String configKey, String format, Object... args) {

        if (logger.isInfoEnabled()) {

            logger.info(String.format(methodTag(configKey), args) + format);
        }
    }

    @Override
    protected void logRequest(String configKey, Level logLevel, Request request) {

        Map<String, Collection<String>> headers = request.headers();
        HttpLog httpLog = HttpLog.builder()
                .headers(MapUtils.isNotEmpty(headers) ? JSON.toJSONString(headers) : StringUtils.EMPTY)
                .methodName(request.httpMethod().name())
                .request(this.getBodyText(request))
                .url(request.url())
                .startAt(DateUtils.now())
                .build();
        HttpLogHolder.set(httpLog);
    }

    private String getBodyText(Request request) {

        byte[] body;
        if (ArrayUtils.isNotEmpty(body = request.body())) {

            Charset charset;
            return Objects.nonNull(charset = request.charset()) ? new String(body, charset) : null;
        }
        return null;
    }


    @Override
    protected Response logAndRebufferResponse(String configKey,
                                              Level logLevel,
                                              Response response,
                                              long elapsedTime) throws IOException {

        HttpLog httpLog = HttpLogHolder.getAndRelease();
        if (Objects.nonNull(httpLog)) {

            httpLog.setFinishAt(DateUtils.now());
            httpLog.setDuration(httpLog.getFinishAt().toInstant(ZoneOffset.UTC).toEpochMilli() - httpLog.getStartAt().toInstant(ZoneOffset.UTC).toEpochMilli());
            httpLog.setStatus(response.status());

            String msg;
            switch (logLevel) {

                case FULL:

                    String bodyData;
                    if (StringUtils.isNotEmpty(bodyData = IOUtils.toString(response.body().asInputStream(), StandardCharsets.UTF_8.name()))) {

                        // 从response中获取数据
                        httpLog.setResponse(bodyData);
                        response = response.toBuilder().body(bodyData, StandardCharsets.UTF_8).build();
                    }
                    msg = String.format("url: [%s],methodName: [%s],startAt: [%s],finishAt: [%s],duration: [%sms],status: [%s],headers:[%s],request: [%s],response: [%s]"
                            , httpLog.getUrl(), httpLog.getMethodName(),
                            getMills(httpLog.getStartAt()), getMills(httpLog.getFinishAt()), httpLog.getDuration(), httpLog.getStatus(),
                            HttpLogUtil.subStringIfRequired(httpLog.getHeaders()), HttpLogUtil.subStringIfRequired(httpLog.getRequest()), HttpLogUtil.subStringIfRequired(httpLog.getResponse()));

                    break;
                default:
                    msg = String.format("url: [%s],methodName: [%s],startAt: [%s],finishAt: [%s],duration: [%sms],status: [%s],headers:[%s]"
                            , httpLog.getUrl(), httpLog.getMethodName(),
                            getMills(httpLog.getStartAt()), getMills(httpLog.getFinishAt()), httpLog.getDuration(), httpLog.getStatus(),
                            HttpLogUtil.subStringIfRequired(httpLog.getHeaders()));
                    break;
            }
            log(configKey, "----------------- feign log -----------------");
            log(configKey, msg);
        }
        return response;
    }
}

 

日志收集

随着微服务数量的增加，对于日志的收集通常采用ELK方式（ELK日志采集分析）进行统一处理。