flume主节点配置
############################################
# producer config
###########################################
#agent section
producer.sources = s
producer.channels = c
producer.sinks = r
#source section
producer.sources.s.type = avro
producer.sources.s.channels = c
producer.sources.s.bind = 192.168.200.100
producer.sources.s.port = 60000
# Each sink's type must be defined
producer.sinks.r.type = org.apache.flume.plugins.KafkaSink
producer.sinks.r.metadata.broker.list=127.0.0.1:9092
producer.sinks.r.partition.key=0
producer.sinks.r.partitioner.class=org.apache.flume.plugins.SinglePartition
producer.sinks.r.serializer.class=kafka.serializer.StringEncoder
producer.sinks.r.request.required.acks=0
producer.sinks.r.max.message.size=1000000
producer.sinks.r.producer.type=sync
producer.sinks.r.custom.encoding=UTF-8
producer.sinks.r.custom.topic.name=cicLogCoinsTopic
#Specify the channel the sink should use
producer.sinks.r.channel = c
# Each channel's type is defined.
producer.channels.c.type = memory
producer.channels.c.capacity = 1000
############################################
# consumer config
###########################################
consumer.sources = s
consumer.channels = c
consumer.sinks = r
consumer.sources.s.type = seq
consumer.sources.s.channels = c
consumer.sinks.r.type = logger
consumer.sinks.r.channel = c
consumer.channels.c.type = memory
consumer.channels.c.capacity = 100
consumer.sources.s.type = org.apache.flume.plugins.KafkaSource
consumer.sources.s.zookeeper.connect=127.0.0.1:2181
consumer.sources.s.group.id=testGroup
consumer.sources.s.zookeeper.session.timeout.ms=400
consumer.sources.s.zookeeper.sync.time.ms=200
consumer.sources.s.auto.commit.interval.ms=1000
consumer.sources.s.custom.topic.name=kafkaToptic
consumer.sources.s.custom.thread.per.consumer=5
flume一级节点配置
# Describe the source
a1.sources = r1
a1.sinks = k1
a1.channels = c1
# Describe/configure the source
a1.sources.r1.type = avro
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
# Describe the sink
a1.sinks.k1.type = avro
a1.sinks.k1.hostname = 192.168.200.100 中央节点的地址
a1.sinks.k1.port= 60000
# Use a channel which buffers events in memory
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
# Bind the source and sink to the channel
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
相关启动命令
一级flume:
自己给自己传递数据后台启动agent
bin/flume-ng agent -n a1 -c conf -f conf/cic.conf.properties -Dflume.root.logger=INFO,console > /dev/null 2>/tmp/tmp.log &
前台启动agent,可以看传递的数据
bin/flume-ng agent -n producer -c conf -f conf/cic.conf.properties -Dflume.root.logger=INFO,console
传递数据
./bin/flume-ng avro-client -c conf -H localhost -p 41411 -F test
二级flume:
后台启动agent./bin/flume-ng agent --conf conf --conf-file conf/flume-cic.properties -n producer -Dflume.root.logger=INFO,console > /dev/null 2>/tmp/error.log &
前台启动agent,可以看传递的数据
./bin/flume-ng agent --conf conf --conf-file conf/flume-cic.properties -n producer -Dflume.root.logger=INFO,console