日志工具-part1

import os

# table_list = ["logs"]
# table_list = ["logs", "logs2"]
table_list = ["logs", "logs2","logs3"]
# sour_list = ['source1']
# sour_list = ['source1', 'source2']
sour_list = ['source1', 'source2','source3']
config_default = [
    '  else if [source] == "{}" {{',  # 注意这里的 {{ 和 }}，双花括号
    '    jdbc {{',
    '      connection_string => "jdbc:mysql://127.0.0.1:3306/logs"',
    '      driver_class => "com.mysql.cj.jdbc.Driver"',
    '      driver_jar_path => "/opt/module/logstash-7.17.3/lib/mysql-connector-j-8.1.0.jar"',
    '      username => "root"',
    '      password => "123456"',
    '      statement => [',
    '        "INSERT INTO {} (timestamp, log_offset, log_file_path, error_message, error_type, ecs_version, input_type, tags, host_name, agent_type, agent_version, agent_id, agent_name, agent_hostname, message) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",',
    '        "@timestamp", "log_offset", "log_file_path", "error_message", "error_type", "ecs_version", "input.type", "tags", "host_name", "agent_type", "agent_version", "agent_id", "agent_name", "agent_hostname", "message"',
    '      ]',
    '    }}',
    '  }}'  # 同样，这里使用了双花括号
]

local_dir = r'E:\DjangoPros\Log'
read_path = os.path.join(local_dir, r'E:\DjangoPros\Log\save\logstash_init_.conf')
save_path = os.path.join(local_dir, r'E:\DjangoPros\Log\save\logstash.conf')
config_part = '\n'.join(text for text in config_default)
config_list=[]

# try:
with open(read_path, 'r', encoding='utf-8') as config_file:
    config_ori = config_file.read().strip()

for index in range(len(table_list)):
    #从第二个开始，第一个已经在配置文件中(if，这里是else if的配置)
    if index == 0:
        continue
    config_new = config_part.format(sour_list[index],table_list[index])
    config_list.append(config_new)
config_all='\n'.join(text for text in config_list)
if len(config_all)==0:
    config_all=''
config_ori = config_ori.format( sour_list[0],table_list[0], config_all)

with open(save_path, 'w', encoding='utf-8') as file:
    file.write(config_ori)

print(f"Logstash 配置文件已生成: {save_path}")

logstash原始配置文件

input {{
  beats {{
    port => 5044
  }}
}}

filter {{
  ruby {{
    code => "event.timestamp.time.localtime"
  }}
  # 保留source字段在fields中，且不影响其他字段
  if [fields][source] {{
    mutate {{
      add_field => {{"source" => "%{{[fields][source]}}" }}
    }}
  }}
  if [error] {{
    mutate {{
      add_field => {{ "error_message" => "%{{[error][message]}}" }}
      add_field => {{ "error_type" => "%{{[error][type]}}" }}
    }}
  }}
  if [log] {{
    mutate {{
      add_field => {{ "log_offset" => "%{{[log][offset]}}" }}
      add_field => {{ "log_file_path" => "%{{[log][file][path]}}" }}
    }}
  }}
  if [agent] {{
    mutate {{
      add_field => {{ "agent_type" => "%{{[agent][type]}}" }}
      add_field => {{ "agent_version" => "%{{[agent][version]}}" }}
      add_field => {{ "agent_id" => "%{{[agent][id]}}" }}
      add_field => {{ "agent_name" => "%{{[agent][name]}}" }}
      add_field => {{ "agent_hostname" => "%{{[agent][hostname]}}" }}
    }}
  }}
  if [host] {{
    mutate {{
      add_field => {{ "host_name" => "%{{[host][name]}}" }}
    }}
  }}
  if [ecs] {{
    mutate {{
      add_field => {{ "ecs_version" => "%{{[ecs][version]}}" }}
    }}
  }}
  mutate {{
    remove_field => ["@version", "input", "tags", "host", "agent", "log", "error", "ecs"]
  }}
}}

output {{
  if [source] == "{}" {{
    jdbc {{
      connection_string => "jdbc:mysql://127.0.0.1:3306/logs"
      driver_class => "com.mysql.cj.jdbc.Driver"
      driver_jar_path => "/opt/module/logstash-7.17.3/lib/mysql-connector-j-8.1.0.jar"
      username => "root"
      password => "123456"
      statement => [
        "INSERT INTO {} (timestamp, log_offset, log_file_path, error_message, error_type, ecs_version, input_type, tags, host_name, agent_type, agent_version, agent_id, agent_name, agent_hostname, message) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
        "@timestamp", "log_offset", "log_file_path", "error_message", "error_type", "ecs_version", "input.type", "tags", "host_name", "agent_type", "agent_version", "agent_id", "agent_name", "agent_hostname", "message"
      ]
    }}
  }}
 {}
}}