flume collection配置

最新推荐文章于 2020-10-11 15:52:59 发布

原创最新推荐文章于 2020-10-11 15:52:59 发布 · 195 阅读

0 ·

CC 4.0 BY-SA版权

本文详细解析了 Apache Flume 的配置文件，介绍了如何通过 Flume 实现数据从多个源到不同目标的数据流管道配置。具体包括源配置、通道配置、选择器配置以及 Kafka 和 HDFS 沉淀配置。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

#master0 name
master0.channels = c1 c2
master0.sources = r1
master0.sinks = k1 h1

#set sources
master0.sources.r1.channels = c1 c2
master0.sources.r1.type = avro
master0.sources.r1.bind = 0.0.0.0
master0.sources.r1.port = 52020

#set channel
master0.channels.c1.type = memory
master0.channels.c1.capacity = 10000
master0.channels.c1.transactionCapacity = 100

master0.channels.c2.type = memory
master0.channels.c2.capacity = 10000
master0.channels.c2.transactionCapacity = 100

#set channel selector
master0.sources.r1.selector.type = multiplexing
master0.sources.r1.selector.header = routekey
master0.sources.r1.selector.mapping.KF = c1
master0.sources.r1.selector.mapping.HF = c2
master0.sources.r1.selector.mapping.KF-HF = c1 c2
master0.sources.r1.selector.mapping.HF-KF = c1 c2
master0.sources.r1.selector.default = c1

# set sink1 kafuka
master0.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
master0.sinks.k1.topic = %{topic}
master0.sinks.k1.brokerList = 10.3.0.248:9092,10.3.0.249:9092,10.3.0.250:9092
master0.sinks.k1.requiredAcks = 1
master0.sinks.k1.batchSize = 20
master0.sinks.k1.channel = c1

#set sink2 hdfs
master0.sinks.h1.type = hdfs
master0.sinks.h1.channel = c2
master0.sinks.h1.hdfs.path = /data/flumelog/%{filename}/%{host}/%y-%m-%d/
master0.sinks.h1.hdfs.filePrefix = s0-
master0.sinks.h1.hdfs.fileType=DataStream
master0.sinks.h1.hdfs.writeFormat = Text
master0.sinks.h1.hdfs.rollInterval = 0
master0.sinks.h1.hdfs.rollSize = 61440000
master0.sinks.h1.hdfs.rollCount = 0
master0.sinks.h1.hdfs.idleTimeout=10800