将压缩日志解压并根据关键字取出对应的日志行python

######################################################
#file_name : gather check
#
#function target: gather file error select
#
#author:  zL
#time  :  2018/04
#flow  :  get porcess path       || (tar_file_analysis)
#      :  unprocess file         || (un_cab, un_tar)
#      :  vol gather file check  || (vogather_check)
#      :  get error line         || (target_file_analysis, get_error_line)
#      :   agent lrt ensure      ||  ()
#######################################################

#import rarfile
#import pandas
import os
#import time

#import sys
#import importlib
#importlib.reload(sys)
#reload(sys)
#sys.setdefaultencoding('utf8')
#importlib.setdefaultencoding('utf8')

#save CAB file
tarZ_path = []
tarZ_name = []
CAB_path = []
CAB_name = []
bz2_path = []
bz2_name = []

#add filename here
trace_file = ["aPP_Log.log", "aPPmaind.trc"]
trace_file.append("aPPmsgdd.trc");

#add check key worlds here
key_world = ["error", "warning"]#, "aPPwebd start", "aPPmaind", "aPPsvr start"
ignore_world = ["error=0", "errno=<2>", "errno=<16>"]
trace_path = []

#flag 
result_file_flag = 1  # start

#######################################
#funtion: check if path exists
#in:      dir
#out:     1: exists  do nothing
#         0: not exists, mkdir
#######################################
def zl_WinRAR(file, path):
    if os.path.exists("C:\Program Files\WinRAR\WinRAR.exe"):
        cmd = "\"C:\Program Files\WinRAR\WinRAR.exe\" x " + file + " " + path
    elif os.path.exists("C:\Program Files (x86)\WinRAR\WinRAR.exe"):
        cmd = "\"C:\Program Files (x86)\WinRAR\WinRAR.exe\" x " + file + " " + path
    else:
        print("There is no WinRAR in your computer!\n")
        exit(0)
    os.system(cmd)

#######################################
#funtion: check if path exists
#in:      dir
#out:     1: exists  do nothing
#         0: not exists, mkdir
#######################################
def zl_mkdir(dir):
    if os.path.isdir(dir):
        return 1
    else:
        os.mkdir(dir)
        return 0

#######################################
#funtion: check it is targetfile
#in:      filename
#out:     yes/no
#######################################
def isTargetFile(filename):
    for value in trace_file:
        if filename == value:
            return 1
    return 0

#######################################
#funtion: get compress file path and name
#in:      now dir
#out:     NULL
#######################################
def tar_file_analysis(file_dir):
    for root, dirs, files in os.walk(file_dir): 
        #print(root) #当前目录路径    
        #print(dirs) #当前路径下所有子目录  
        #print(files) #当前路径下所有非目录子文件
        
        #CAB
        for file_name in files:
            print(file_name)
            if ".CAB" in file_name and "aPPgather-" in file_name:
                CAB_path.append(root)
                CAB_name.append(file_name)
                print(file_name)
                break
            if  "tar.Z" in file_name and "aPPgather-" in file_name:
                tarZ_path.append(root)
                tarZ_name.append(file_name)
                print(file_name)
            if  "tar.gz" in file_name and "aPPgather-" in file_name:
                tarZ_path.append(root)
                tarZ_name.append(file_name)
                print(file_name)
            if  ".tar.bz2" in file_name and "Gather" in file_name:
                bz2_path.append(root)
                bz2_name.append(file_name)
                print(file_name)

#######################################
#funtion: find target file path
#in:      now dir
#out:     NULL
#######################################
def target_file_analysis(file_dir):
    for root, dirs, files in os.walk(file_dir):
        for file_name in files:
            if isTargetFile(file_name):
                get_error_line(root, file_name)

#######################################
#funtion: find error in target file
#in:      now dir
#out:     NULL
#######################################
def get_error_line(file_path, file_name):
    #open(os.path.join(root, file),'r', encoding='cp932', errors='ignore')
    global result_file_flag
    if result_file_flag == 1:
        result_file_flag = 0
        resultFile = open('errorSelect.txt', 'w', encoding='utf-8')
    else:
        resultFile = open('errorSelect.txt', 'a', encoding='utf-8')
    
    file = file_path + "\\" + file_name
    resultFile.writelines(file)
    resultFile.writelines("  \n")
    print("\n========start=====\n", file_path, file_name, "\n=======end======\n")
    targetFile = open(file, 'r', encoding = 'utf-8', errors='replace')

    for line in targetFile:
        #print(line)
        flag = 0
        for value in ignore_world:
            if value in line:
                flag = 1
                break

        if flag == 1:
            break

        for value in key_world:
            if value in line:
                #print(line)
                resultFile.writelines(line)
                resultFile.writelines("  \n")
                break

    resultFile.close()
    targetFile.close()

#######################################
#funtion: get info in agent target file
#in:      now dir
#out:     NULL
#######################################    
def agent_target_check(file_path):
    file_agcomma = [] 
    file_agcommp = []
    file_agmaind = ["aPPagmaind0.log"]
    resultFile = open('errorSelect.txt', 'a', encoding='utf-8')

    #get targetfile
    for root, dirs, files in os.walk(file_path):
        for file_name in files:
                file_dir =  root + "\\" + file_name
                if "aPPagcomma" in file_name and "_" not in file_name:
                    file_agcomma.append(file_dir)
                elif "aPPagcommp" in file_name:
                    file_agcommp.append(file_dir)
    
    #file_agcomma.append("aPPagmaind0.trc")
    #print(file_agcomma)
    #print(file_agcommp)
    #find key world
    resultFile = open('errorSelect.txt', 'a', encoding='utf-8')
    for dir in file_agcomma:
        resultFile.writelines(dir)
        resultFile.writelines("  \n")
        targetFile = open(dir, 'r', encoding = 'utf-8', errors='replace')
        for line in targetFile:
            if ("error" in line and "error=0" not in line) or ("Command was send" in line) or ("econnect" in line and "OK" not in line):
                resultFile.writelines(line)
                resultFile.writelines("  \n")
        targetFile.close()

    for dir in file_agcommp:
        resultFile.writelines(dir)
        resultFile.writelines("  \n")
        targetFile = open(dir, 'r', encoding = 'utf-8', errors='replace')
        for line in targetFile:
            if ("error" in line and "error=0" not in line) or ("ip" in line and "state" in line and "READY" not in line):
                resultFile.writelines(line)
                resultFile.writelines("  \n")
        targetFile.close()

    resultFile.close()

#######################################
#funtion: check if syslog exists
#in:      gather file path
#out:     NULL
#######################################
def vogather_check(file_path, file_name):
    fileVogather = ""
    
    for root, dirs, files in os.walk(file_path):
        for values in files:
            if "aPPvolgather.CAB" in values or "aPPvolgather-" in values:
                fileVogather = root + "\\" + values
                break
    
    if fileVogather == "":
        return
    print(fileVogather)
    #time.sleep(3)
    #untar vogather
    if "CAB" in fileVogather:
        dir = fileVogather[0:-4]
    elif ".tar.gz" in fileVogather:
        dir = fileVogather[0:-7]
    else:
        dir = fileVogather[0:-6]

    if zl_mkdir(dir) == 0:
        zl_WinRAR(fileVogather, dir)
    
    #check syslog
    if "CAB" in fileVogather:
        dir = dir + "\\" + "aPPvolgather_syslog"
        if zl_mkdir(dir) == 0:
            zl_WinRAR(dir + ".CAB", dir)
        
        syslog = dir + "\\" + "SysLog.evt"
        applog = dir + "\\" + "AppLog.evt"
        if (os.path.exists(syslog) and os.path.exists(applog)):
            return
        else:
            print("target: \n", syslog ,"is not exists")
    else:
        syslog = dir + "\\" + "aPPvolgather_syslog\\messages"
        syslog_log = dir + "\\" + "aPPvolgather_syslog\\syslog.log"
        if os.path.exists(syslog) or os.path.exists(syslog_log):
            return
        else:
            print("target: \n", syslog , syslog_log, "is not exists")

#######################################
#funtion: unpair compress file(CAB) 
#in:      file path and name
#out:     NULL
#######################################
def un_cab(file_path, file_name):
    """unrar cab file"""
    file = file_path + "\\" + file_name

    #mkdir for every tar file
    dir = file_path + "\\" + file_name[0:-4]
    if zl_mkdir(dir) == 0:
        #untar cab file
        zl_WinRAR(file, dir)
    

    #untar aPPgater_tarce.CAB
    file_trace =  dir + "\\" + "aPPgather_trace.CAB"
    dir_trace  =  dir + "\\" + "aPPgather_trace"
    if zl_mkdir(dir_trace) == 0:
        zl_WinRAR(file_trace, dir_trace)
    
    #untar aPPgather_log.CAB
    file_log =  dir + "\\" + "aPPgather_log.CAB"
    dir_log  =  dir + "\\" + "aPPgather_log"
    if zl_mkdir(dir_log) == 0:
        zl_WinRAR(file_log, dir_log)

    #add trace_path
    trace_path.append(dir)

    #vogather check
    vogather_check(file_path, file_name)

#######################################
#funtion: unpair compress file(tar.Z) 
#in:      file path and name
#out:     NULL
#######################################
def un_tar(file_path, file_name):
    """unrar tar.Z file"""
    file = file_path + "\\" + file_name

    #mkdir for every tar file
    dir = file_path + "\\" + file_name[0:-6]
    if zl_mkdir(dir) == 0:
        zl_WinRAR(file, dir)
        
    #add trace_path
    #dir = dir + "\\" + "trace"
    trace_path.append(dir)

    #vogather check
    vogather_check(file_path, file_name)

#######################################
#funtion: unpair compress file(.tar.bz2) 
#in:      file path and name
#out:     NULL
#######################################
def un_bz2(file_path, file_name):
    """unrar tar.Z file"""
    file = file_path + "\\" + file_name

    #mkdir for every tar file
    dir = file_path + "\\" + file_name[0:-8]
    if zl_mkdir(dir) == 0:
        zl_WinRAR(file, dir)
    
    #add trace_path
    #dir = dir + "\\" + "trace"
    trace_path.append(dir)

    #vogather check
    for root, dirs, files in os.walk(dir):
        for file_name in files:
                if "messages" == file_name:
                    return 1
    return 0
#print(os.getcwd()) #获取当前工作目录路径
#print(os.path.abspath('.')) #获取当前工作目录路径
#print(os.path.abspath('test.txt')) #获取当前目录文件下的工作目录路径
#print(os.path.abspath('..')) #获取当前工作的父目录 ！注意是父目录路径
#print(os.path.abspath(os.curdir)) #获取当前工作目录路径

 

def main():
    print("Now paht is :", os.getcwd())
    tar_file_analysis(os.getcwd())
    
    print("----------------unprocess start---------------------")
    for i in range(len(CAB_path)):
        print("-------------------------------------")
        print(CAB_path[i],CAB_name[i])
        un_cab(CAB_path[i], CAB_name[i])
       
    
    for i in range(len(tarZ_path)):
        print("-------------------------------------")
        print(tarZ_path[i],tarZ_name[i])
        un_tar(tarZ_path[i], tarZ_name[i])
    print("----------------unprocess end---------------------")

    for i in range(len(bz2_path)):
        print("-------------------------------------")
        print(bz2_path[i], bz2_name[i])
        if 0 == un_bz2(bz2_path[i], bz2_name[i]):
            print("\nsyslog is not exsits\n")
    print("----------------unprocess end---------------------")    #get trace path
    #print(trace_path)
    
    print("----------------get error info  start---------------------")
    for tarcePath in trace_path:
        #print(tarcePath)
        target_file_analysis(tarcePath)
    print("----------------get error info  end---------------------")

    print("----------------agent LRT get info start---------------------")
    for tarcePath in trace_path:
        agent_target_check(tarcePath)
    print("----------------agent LRT get info end---------------------")

if __name__=="__main__":
   main()