py正则分析日志 自己用的

2019独角兽企业重金招聘Python工程师标准>>>

日后在做整理

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Created by weilai on 2016/12/21
# 自动分析火车票发送和接受日志工具，wl 201611221
# 用于分析火车票发送和收通知的内容 和时差
# v0.1


import os
import datetime
import time
import json
import re


# 定义基础目录
path = 'X:\\AccDetail'
send_log_base_path = 'X:\\InteractionDetail'
receive_log_base_path = 'X:\\InteractionDetail'

now = datetime.datetime.now()
today_dir = '20161221' #now.strftime('%Y%m%d')
# 定义基础目录和时间目录组合
work_dir = path + '\\' + today_dir
send_log_path = send_log_base_path + '\\' + today_dir
receive_log_path = receive_log_base_path + '\\' + today_dir
# 定义发送，接收，结果集合
book_list = []
receive_list = []
ticket_list = []

pattern_log_body = r'\[BEGIN\]([\s\S.]*?)\[END\]'
pattern_log_time = r'\[L_B\]时间戳:(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})\[L_E\]'
pattern_log_run_time = r'\[L_B\]时间间隔:(\d{7}\.\d{3})\[L_E\]'
pattern_log_func_name = r'\[L_B\](自定义消息:book)\[L_E\]'
pattern_log_send_content = r'\[L_B\]发送内容:j=([\s\S.]*?)\[L_E\]'
pattern_log_receive_content = r'\[L_B\]接收内容:([\s\S.]*?)\[L_E\]'

pattern_log_notice_func_name = r'\[L_B\](自定义消息:预定结果通知)\[L_E\]'
pattern_log_notice_send_content = r'\[L_B\]发送内容:([\s\S.]*?)\[L_E\]'
pattern_log_notice_receive_content = r'\[L_B\]接收内容:j=([\s\S.]*?)\[L_E\]'


class InteractiveInfo:
    """初级分析结果类"""
    log_time = ''
    log_name = ''
    log_run_time = ''
    log_param = ''
    log_result = ''
    flow_id = ''
    order_id = ''
    out_order_id = ''
    msg = ''
    log_file = ''
    valid_data = False
    train = ''
    passager = ''
    ticket = ''


class TicketInfo:
    """最后分析结果类"""
    ticket_order_id = ''
    ticket_out_order_id = ''
    ticket_flow_id = ''
    ticket_send_time = ''
    ticket_receive_time = ''
    time_diff = 0
    receive_times = 0
    info_from_file = ''
    msg = ''
    source_send_data = None
    source_receive_data = None

# ------------------------------------------------


def regex_text(pattern_str, txt):
    pattern = re.compile(pattern_str, re.M)
    match = pattern.findall(txt)
    if match:
        return match
    return None


def regex_log_body(txt):
    return regex_text(pattern_log_body, txt)


def regex_log_time(txt):
    return regex_text(pattern_log_time, txt)


def regex_log_run_time(txt):
    return regex_text(pattern_log_run_time, txt)


def regex_log_func_name(txt):
    return regex_text(pattern_log_func_name, txt)


def regex_log_send_content(txt):
    return regex_text(pattern_log_send_content, txt)


def regex_log_receive_content(txt):
    return regex_text(pattern_log_receive_content, txt)


def regex_log_notice_func_name(txt):
    return regex_text(pattern_log_notice_func_name, txt)


def regex_log_notice_receive_content(txt):
    return regex_text(pattern_log_notice_receive_content, txt)
# ----------------------------------------------


def analyse_send_file(log_file_path):
    with open(log_file_path, 'r', encoding='utf-8') as f:
        print(log_file_path)
        log_file_str = f.read()
        logs = regex_log_body(log_file_str)
        for log in logs:
            log_item = log
            log_name = regex_log_func_name(log_item)
            if log_name is not None:
                send_log = InteractiveInfo()
                send_log.log_name = log_name[0]
                send_log.log_file = log_file_path
                # print(log_name[0])
                log_time = regex_log_time(log_item)
                send_log.log_time = log_time[0]
                # print(log_time[0])
                log_run_time = regex_log_run_time(log_item)
                send_log.log_run_time = log_run_time[0]
                # print(log_run_time[0])
                send_content = regex_log_send_content(log_item)
                get_send_json(send_content[0], send_log)
                # print(send_content[0])
                receive_content = regex_log_receive_content(log_item)
                get_send_result_json(receive_content[0], send_log)
                # print(receive_content[0])
                book_list.append(send_log)
# ---------------------------------------------------


def get_send_result_json(json_str, send_obj):
    json_val = json.loads(json_str)
    send_obj.flow_id = str(json_val['i'])
    send_obj.msg += json_val['m'] + "|"


def get_send_json(json_str, send_obj):
    json_val = json.loads(json_str)
    send_obj.order_id = json_val['d']['n']
    x1 = lambda x: x['pn']
    send_obj.passager = ','.join([x1(y) for y in json_val['d']['p']])
    send_obj.train = '{0}|{1}|{2}|{3}'.format(json_val['d']['c'], json_val['d']['d'], json_val['d']['fn'], json_val['d']['tn'])
    # print(send_obj.train)
# -----------------------------------------------


def analyse_receive_file(log_file_path):
    with open(log_file_path, 'r', encoding='utf-8') as f:
        print(log_file_path)
        log_file_str = f.read()
        logs = regex_log_body(log_file_str)
        for log in logs:
            log_item = log
            log_name = regex_log_notice_func_name(log_item)
            if log_name is not None:
                receive_log = InteractiveInfo()
                receive_log.log_name = log_name[0]
                # print(receive_log.log_name)
                receive_log.log_file = log_file_path
                # print(log_name[0])
                log_time = regex_log_time(log_item)
                receive_log.log_time = log_time[0]
                # print(log_time[0])
                log_run_time = regex_log_run_time(log_item)
                receive_log.log_run_time = log_run_time[0]
                # print(log_run_time[0])
                # 无需这两句 不分解发送内容
                # send_content = regex_log_send_content(log_item)
                # get_receive_json(send_content[0], receive_log)
                # print(send_content[0])
                receive_content = regex_log_notice_receive_content(log_item)
                get_receive_result_json(receive_content[0], receive_log)
                # print(receive_content[0])
                receive_list.append(receive_log)


def get_receive_result_json(json_str, receive_obj):
    json_val = json.loads(json_str)
    receive_obj.flow_id = str(json_val['i'])
    receive_obj.out_order_id = str(json_val['d']['i'])
    receive_obj.order_id = str(json_val['d']['n'])
    receive_obj.msg += json_val['m'] + "|"
    x1 = lambda x: x['tn']+'|'+x['si']+'|'+x['pr']
    if 't' in json_val['d']:
        receive_obj.ticket = json_val['d']['t'] + '|'
    if 'p' in json_val['d']:
        receive_obj.ticket = ','.join([x1(y) for y in json_val['d']['p']])
    receive_obj.ticket.replace('\n', '')
    # print(receive_obj.ticket)


def get_receive_json(json_str, send_obj):
    pass
    # -----------------------------------------------
# -----------------------------------------------


def analyse_file(analyse_dir, dir_str, analyse_func):
    """分析日志"""
    file_list = os.listdir(analyse_dir)
    for dir_name in file_list:
        if dir_name.find(dir_str) >= 0:
            full_path = analyse_dir + '\\' + dir_name
            # print(full_path)
            files = os.listdir(full_path)
            for file in files:
                file_path = full_path + '\\' + file
                analyse_func(file_path)
# ----------------------------------------------


def analyse_send_log(analyse_dir):
    analyse_file(analyse_dir, '火车票交互服务', analyse_send_file)


def analyse_receive_log(analyse_dir):
    analyse_file(analyse_dir, '火车票业务收通知网站基线', analyse_receive_file)


def show_send_list(source_list):
    """显示分析的发送日志"""
    global book_list
    book_list = sorted(source_list, key=lambda x: x.log_time)
    i = 0
    p_template = '{0})=={1}=={2}=={3}=={4}=={5};'
    for item in book_list:
        i += 1
        p_str = p_template.format(str(i), item.log_time, item.log_name, item.flow_id, item.order_id, item.msg)
        print(p_str)


def show_receive_list(source_list):
    """显示分析的接收日志"""
    global receive_list
    receive_list = sorted(source_list, key=lambda x: x.log_time)
    i = 0
    p_template = '{0})=={1}=={2}=={3}=={4}=={5};'
    for item in receive_list:
        i += 1
        m = item.msg
        if item.msg == '|':
            m += item.ticket
        p_str = p_template.format(str(i), item.log_time, item.log_name, item.flow_id, item.order_id, m)
        print(p_str)
# ----------------------------------------------


def analyse_diff(send_list, result_list):
    """对比发送数据和接收数据，组合对照数据，并计算运行时差"""
    for b_item in send_list:
        for r_item in result_list:
            if b_item.flow_id == r_item.flow_id and b_item.flow_id != '': # 不能为空字符串
                has_it = [x for x in ticket_list if x.ticket_flow_id == b_item.flow_id]
                if len(has_it) <= 0:
                    t = TicketInfo()
                    t.ticket_send_time = b_item.log_time
                    t.ticket_receive_time = r_item.log_time
                    t.ticket_flow_id = b_item.flow_id
                    t.ticket_order_id = b_item.order_id
                    t.ticket_out_order_id = r_item.out_order_id
                    t.msg = b_item.msg + '|' + r_item.msg
                    t.receive_times = 1
                    t.time_diff = 0
                    t.info_from_file = '{f1}|{f2}'.format(f1=b_item.log_file, f2=r_item.log_file)
                    time_a1 = time.strptime(t.ticket_receive_time, "%Y-%m-%d %H:%M:%S.%f")
                    time_stamp1 = datetime.datetime(*time_a1[:6])
                    time_a2 = time.strptime(t.ticket_send_time, "%Y-%m-%d %H:%M:%S.%f")
                    time_stamp2 = datetime.datetime(*time_a2[:6])
                    t.time_diff = (time_stamp1 - time_stamp2).total_seconds()
                    # print(t.time_diff)
                    ticket_list.append(t)
                else:
                    has_it[0].receive_times += 1
# -------------------------------------------------


def show_result_list(source_list):
    """显示分析组合结果"""
    global ticket_list
    ticket_list = sorted(source_list, key=lambda x: x.ticket_send_time)
    i = 0
    p_template = '{0})={1}=={2}=={3}-{4}[{5}秒]{6}次'
    for item in ticket_list:
        i += 1
        p_str = p_template.format(str(i), item.ticket_flow_id, item.ticket_order_id, item.ticket_send_time,
                                  item.ticket_receive_time, str(item.time_diff), str(item.receive_times), item.info_from_file)
        print(p_str)
# ---------------------------------------------------


print('/***********火车票发送交互服务日志[{0}]***********/'.format(today_dir))
analyse_send_log(send_log_path)
show_send_list(book_list)
print('/***********火车票接收通知日志[{0}]***********/'.format(today_dir))
analyse_receive_log(receive_log_path)
show_receive_list(receive_list)
print('/***********火车票占座时差计算[{0}]***********/'.format(today_dir))
analyse_diff(book_list, receive_list)
show_result_list(ticket_list)

# 样品
xx = """[BEGIN]
[L_B]日志ID:bb1f4d67-162f-47d0-90b0-c27bfbe218cb[L_E]
[L_B]时间点:636179106143493242[L_E]
[L_B]跟踪ID:00517WCF16122109492672990[L_E]
[L_B]时间戳:2016-12-21 09:50:14.349[L_E]
[L_B]时间间隔:0047620.052[L_E]
[L_B]应用程序名:火车票交互服务v0029[L_E]
[L_B]模块名:火车票交互[L_E]
[L_B]方法名:[L_E]
[L_B]方法说明:[L_E]
[L_B]进程ID:25968[L_E]
[L_B]线程ID:27[L_E]
[L_B]发送地址:http://121.41.85.136/a.ashx[L_E]
[L_B]异常ID:[L_E]
[L_B]异常级别:[L_E]
[L_B]自定义消息:book[L_E]
[L_B]交互类型:[L_E]
[L_B]关键信息1:1612210950140910f0500236688[L_E]
[L_B]关键信息2:[L_E]
[L_B]机器名:WIN-VDHFP5GHJ89[L_E]
[L_B]本地IP:172.17.1.242[L_E]
[L_B]发送内容:j={"a": true,"b":[{"PassagerID":1,"TicketKeyID":"1688"}],"c": "s55666","d": {  "c":"D2244","d":"2016-12-31","n":"161221095016770236688","w":false,"p":[{"pi":"555555555555555554","pn":"某","po":1,"ic":"1","xc":"O","pc":"1"}],"fc":"ICW","fn":"成都东","tc":"NIW","tn":"遂宁"    },     "f": "book",     "r":"http://9999.com:1401/Notify",   "s": "c722f24f1ec88271a3c0aed7e70f6835",            "t": "20161221095014",        "w": 30000 }[L_E]
[L_B]接收内容:{
  "c": 0,
  "d": null,
  "m": "请等待异步",
  "s": true,
  "w": true,
  "i": 2016122100000000298
}[L_E]
[L_B]路由:r+zYFqAUxzhIEINyz9RyMppo9SMJ1oVwbqlQwyzn0Wol+lxGUB1MwjcWz88Pa5ah7StDIUP/0XBirsjsZMiW0BfC8lHJzAILcpwLva4Q8/zrzra0mjlKFg@@[L_E]
[END]"""
y = re.findall(r'\[BEGIN\]([\s\S.]*?)\[END\]', xx, re.M)
print(y)

 

 

转载于:https://my.oschina.net/raddleoj/blog/811141