Flume实例二 | 监控服务器Lynis日志上传远端hbase | 联级

文章目录

联级agent

联级agent

采集资产漏洞扫描日志/var/log/lynis.log，agent端agent.properties，大数据端hbase.properties。
启动两个采集服务，客户端start-agent.sh和大数据端start-h.sh

创建hbase表

create 'T_WARNING_LYNIS', {NAME => 'BASIC_INFO', VERSIONS => 3, BLOCKCACHE => false, COMPRESSION => 'SNAPPY'}, SPLITS => ['1','2','3','4','5','6','7','8','9','10']

创建目录

客户端create.sh

mkdir -p /home/dataFlume/data_lynis/checkpoint
mkdir -p /home/dataFlume/data_lynis/datadir
mkdir -p /home/dataFlume/data_lynis/backup_checkpoint

大数据端create-h.sh

mkdir -p /home/hadoop/dataFlume/data_lynis/checkpoint
mkdir -p /home/hadoop/dataFlume/data_lynis/datadir
mkdir -p /home/hadoop/dataFlume/data_lynis/backup_checkpoint

客户端flume配置

agent.properties

#list all the compents of this agent 
type_agent.sources = r1
type_agent.sinks = k1
type_agent.channels = c1
#10.lynis
# list the properties of source
type_agent.sources.r1.type = exec
type_agent.sources.r1.command = tail -F /var/log/lynis.log | sed -u "s/^/$HOSTNAME /g"
type_agent.sources.r1.shell = /bin/bash -c

# list the properties of channels
type_agent.channels.c1.type = file
type_agent.channels.c1.checkpointDir = /home/dataFlume/data_lynis/checkpoint
# Using multiple directories on separate disks can improve file channel peformance
type_agent.channels.c1.dataDirs = /home/dataFlume/data_lynis/datadir
type_agent.channels.c1.useDualCheckpoints = true
type_agent.channels.c1.backupCheckpointDir = /home/dataFlume/data_lynis/backup_checkpoint

# list the properties of sinks
type_agent.sinks.k1.type = avro
type_agent.sinks.k1.hostname = master
type_agent.sinks.k1.port = 16080
type_agent.sinks.k1.compression-type = deflate
type_agent.sinks.k1.batch-size = 16

# link the flow
type_agent.sources.r1.channels = c1
type_agent.sinks.k1.channel = c1

其中tail -F /var/log/lynis.log | sed -u "s/^/$HOSTNAME /g"后面得sed用于在每行行首添加HOSTNAME

start-agent.sh

nohup flume-ng agent -n type_agent -c $FLUME_HOME/conf -f $FLUME_HOME/conf/Inventory/Lynis/agent.properties &

记得服务sh文件执行权限

chmod -R 744 start-agent.sh

运行命令

nohup ./start-agent.sh 1>out.log 2>&1

大数据端

hbase.properties

#list all the compents of this agent 
type_hbase.sources = r1
type_hbase.sinks = k1
type_hbase.channels = c1
#10.lynis
# list the properties of source
type_hbase.sources.r1.type = avro
type_hbase.sources.r1.bind = 0.0.0.0
type_hbase.sources.r1.port = 16080
type_hbase.sources.r1.threads = 10
type_hbase.sources.r1.compression-type = deflate

# list the properties of channels
type_hbase.channels.c1.type = file
type_hbase.channels.c1.checkpointDir = /home/hadoop/dataFlume/data_lynis/checkpoint
# Using multiple directories on separate disks can improve file channel peformance
type_hbase.channels.c1.dataDirs = /home/hadoop/dataFlume/data_lynis/datadir
type_hbase.channels.c1.useDualCheckpoints = true
type_hbase.channels.c1.backupCheckpointDir = /home/hadoop/dataFlume/data_lynis/backup_checkpoint

# list the properties of sinks
type_hbase.sinks.k1.type = hbase2
type_hbase.sinks.k1.serializer = org.apache.flume.sink.hbase2.RegexHBase2EventSerializer
type_hbase.sinks.k1.columnFamily = BASIC_INFO
# first regex expression
#type_hbase.sinks.k1.serializer.regex = ([^ ]*) ([0-9]*-[0-9]*-[0-9]*) ([0-9]*:[0-9]*:[0-9]*) (Warning|Suggestion): (.+)
#type_hbase.sinks.k1.serializer.colNames = DST_IP,YMD,TIME,TYPE,INFO
type_hbase.sinks.k10.serializer.regex = ([^ ]*) ([0-9]*-[0-9]*-[0-9]*) ([0-9]*:[0-9]*:[0-9]*) (Warning|Suggestion): (.*?) \\[test:(.*?)-(.*?)\\] \\[details:(.*?)\\] \\[solution:(.*?)\\]
type_hbase.sinks.k10.serializer.colNames = DST_IP,YMD,TIME,TYPE,INFO,TEST,TEST_ID,DETAIL,SOLUTION

type_hbase.sinks.k1.table = T_WARNING_LYNIS

# link the flow
type_hbase.sources.r1.channels = c1
type_hbase.sinks.k10.channel = c1

start-h.sh

nohup flume-ng agent -n type_hbase -c $FLUME_HOME/conf -f $FLUME_HOME/conf/Inventory/Lynis/hbase.properties &

记得服务sh文件执行权限

chmod -R 744 start-h.sh

运行命令

nohup ./start-h.sh 1>out.log 2>&1