flume集群

flume-ng agent --conf /root/flume-1.4.0/conf --conf-file  /root/flume-1.4.0/conf/a.conf --name a1 -Dflume.root.logger=INFO,console

curl -X get -d '[{  "headers" :{"namenode" :  "namenode.example.com","datanode" :  "random_datanode.example.com"},"body" :  "really_random_body"}]' http://vm006:44444

节点1

a2.sources = r1
a2.sinks = k1
a2.channels = c1


# Describe/configure the source
a2.sources.r1.type = avro
a2.sources.r1.bind = 0.0.0.0
a2.sources.r1.port = 44444
a2.sources.r1.handler = org.apache.flume.source.http.JSONHandler


# Describe the sink
a2.sinks.k1.type = file_roll
a2.sinks.k1.sink.directory = /data/flume/


# Use a channel which buffers events in memory
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100


# Bind the source and sink to the channel
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1

节点2

a2.sources = r1
a2.sinks = k1
a2.channels = c1


# Describe/configure the source
a2.sources.r1.type = avro
a2.sources.r1.bind = 0.0.0.0
a2.sources.r1.port = 44444
a2.sources.r1.handler = org.apache.flume.source.http.JSONHandler


# Describe the sink
a2.sinks.k1.type = file_roll
a2.sinks.k1.sink.directory = /data/flume


# Use a channel which buffers events in memory
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100


# Bind the source and sink to the channel
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1

负载均衡节点

a2.sources = r1
a2.sinks = k1
a2.channels = c1


# Describe/configure the source
a2.sources.r1.type = avro
a2.sources.r1.bind = 0.0.0.0
a2.sources.r1.port = 44444
a2.sources.r1.handler = org.apache.flume.source.http.JSONHandler


# Describe the sink
a2.sinks.k1.type = file_roll
a2.sinks.k1.sink.directory = /data/flume


# Use a channel which buffers events in memory
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100


# Bind the source and sink to the channel
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1

客户端机器暂时使用flume-ng的agent发送，配置如下：

[html]  view plain copy

1. # Define a memory channel called c1 on a1  
2. a1.channels.c1.type = file   
3. a1.channels.c1.checkpointDir = /tmp/flume/checkpoint  
4. a1.channels.c1.dataDirs = /tmp/flume/data  
5.   
6.   
7. a1.sources.r1.channels = c1  
8. a1.sources.r1.type = exec  
9. a1.sources.r1.command = cat /tmp/linux.log  
10.   
11. a1.sinks.k1.type = avro   
12. a1.sinks.k1.channel = c1  
13. a1.sinks.k1.hostname = host1  
14. a1.sinks.k1.port = 41415  
15.   
16. a1.channels = c1  
17. a1.sources = r1  
18. a1.sinks = k1