flume-ng hdfs配置文件

最新推荐文章于 2025-11-23 15:59:15 发布

原创最新推荐文章于 2025-11-23 15:59:15 发布 · 226 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#flume

flume 专栏收录该内容

3 篇文章

订阅专栏

本文详细介绍了 Flume 数据采集系统的配置方法，包括 Avro 类型的数据源设置、内存通道参数配置、以及 HDFS 沉淀数据的具体选项。通过具体实例展示了如何设置拦截器、通道类型与容量、HDFS 的路径、文件前缀等关键参数。



collection73Agent.sources=source73
collection73Agent.sinks=sink73-1 sink73-2
collection73Agent.channels=channel73

#configuration source
collection73Agent.sources.source73.type=avro
collection73Agent.sources.source73.channels=channel73
collection73Agent.sources.source73.bind=10.21.3.73
collection73Agent.sources.source73.port=4141
collection73Agent.sources.source73.interceptors = i1 i2
collection73Agent.sources.source73.interceptors.i1.type = org.apache.flume.interceptor.HostInterceptor$Builder
collection73Agent.sources.source73.interceptors.i1.preserveExisting = false
collection73Agent.sources.source73.interceptors.i1.hostHeader = hostname
collection73Agent.sources.source73.interceptors.i2.type = org.apache.flume.interceptor.TimestampInterceptor$Builder

#configuration channel
collection73Agent.channels.channel73.type=memory
collection73Agent.channels.channel73.capacity=2000
collection73Agent.channels.channel73.transactionCapacity=1000
collection73Agent.channels.channel73.keep-alive=10

#confituration sinks
collection73Agent.sinks.sink73-1.type=hdfs
collection73Agent.sinks.sink73-1.channel=channel73
collection73Agent.sinks.sink73-1.hdfs.path=hdfs://mycluster1/flume/%Y-%m
collection73Agent.sinks.sink73-1.hdfs.filePrefix=syslog1.%Y-%m-%d
collection73Agent.sinks.sink73-1.hdfs.fileSuffix=.log
#collection73Agent.sinks.sink73-1.hdfs.round=true
#collection73Agent.sinks.sink73-1.hdfs.roundValue=10
#collection73Agent.sinks.sink73-1.hdfs.roundUnit=minute
collection73Agent.sinks.sink73-1.hdfs.rollInterval=0
collection73Agent.sinks.sink73-1.hdfs.rollSize=0
collection73Agent.sinks.sink73-1.hdfs.batchSize=10000
collection73Agent.sinks.sink73-1.hdfs.rollCount=0
collection73Agent.sinks.sink73-1.hdfs.fileType = DataStream
collection73Agent.sinks.sink73-1.hdfs.writeFormat=Text
collection73Agent.sinks.sink73-1.hdfs.callTimeout=30000
collection73Agent.sinks.sink73-1.channel=channel73

#confituration sinks
collection73Agent.sinks.sink73-2.type=hdfs
collection73Agent.sinks.sink73-2.channel=channel73
collection73Agent.sinks.sink73-2.hdfs.path=hdfs://mycluster1/flume/%Y-%m
collection73Agent.sinks.sink73-2.hdfs.filePrefix=syslog2.%Y-%m-%d
collection73Agent.sinks.sink73-2.hdfs.fileSuffix=.log
#collection73Agent.sinks.sink73-2.hdfs.round=true
#collection73Agent.sinks.sink73-2.hdfs.roundValue=10
#collection73Agent.sinks.sink73-2.hdfs.roundUnit=minute
collection73Agent.sinks.sink73-2.hdfs.rollInterval=0
collection73Agent.sinks.sink73-2.hdfs.rollSize=0
collection73Agent.sinks.sink73-2.hdfs.batchSize=10000
collection73Agent.sinks.sink73-2.hdfs.rollCount=0
collection73Agent.sinks.sink73-2.hdfs.fileType = DataStream
collection73Agent.sinks.sink73-2.hdfs.writeFormat=Text
collection73Agent.sinks.sink73-2.hdfs.callTimeout=30000
collection73Agent.sinks.sink73-2.channel=channel73