flume日志

1. 故障转移(failover)

a.sources = s1
a.sinks = k1 k2
a.channels = c1
a.sinkgroups = g1
a.sources.s1.type =exec
a.sources.s1.command = tail -F /tmp/test.log

a.sinks.k1.type = avro
a.sinks.k1.hostname = localhost
a.sinks.k1.port = 44444
a.sinks.k2.type = avro
a.sinks.k2.hostname = localhost
a.sinks.k2.port = 44445

a.channels.c1.type = memory
a.channels.c1.capacity=1000
a.channels.c1.transactionCapacity=100
a.sources.s1.channels = c1
#sinkgroups配置
a.sinkgroups.g1.sinks = k1 k2
a.sinkgroups.g1.processor.type = failover
#赋予k1,k2权重，数据正常进入权重高的sinks
a.sinkgroups.g1.processor.priority.k1 = 5
a.sinkgroups.g1.processor.priority.k2 = 10
a.sinkgroups.g1.processor.maxpenalty = 300000
a.sinks.k1.channels = c1
a.sinks.k2.channels = c1

2. 负载均衡(load_balance)

a.sources = s1
a.sinks = k1 k2
a.channels = c1
a.sinkgroups = g1
a.sources.s1.type =exec
a.sources.s1.command = tail -F /tmp/test.log

a.sinks.k1.type = avro
a.sinks.k1.hostname = localhost
a.sinks.k1.port = 44444
a.sinks.k2.type = avro
a.sinks.k2.hostname = localhost
a.sinks.k2.port = 44445

a.channels.c1.type = memory
a.channels.c1.capacity=1000
a.channels.c1.transactionCapacity=100
a.sources.s1.channels = c1
#sinkgroups配置
a.sinkgroups.g1.sinks = k1 k2
a.sinkgroups.g1.processor.type = load_balance
a1.sinkgroups.g1.processor.backoff = true
#random：随机  round_robin:轮询
a1.sinkgroups.g1.processor.selector = random
a.sinks.k1.channels = c1
a.sinks.k2.channels = c1

注意：一般情况下，如果数据sources通过channels进入两个不同的sinks，需要保证sinks数据一致的话，必须每个sinks有自己单独的channel，共用一个channel会产生一个sinkgroups，sinkgroups的几种模式内部：failover/load_balance或者default，一般都是轮询或者随机发送数据到sinks,造成数据不一致。（failover按照权重，服务正常情况下，全部发送给权重高的sinks）