Flume 安装, 测试

最新推荐文章于 2024-10-03 20:57:37 发布

原创最新推荐文章于 2024-10-03 20:57:37 发布 · 7.8w 阅读

0 ·

CC 4.0 BY-SA版权

flume 专栏收录该内容

2 篇文章

订阅专栏

本文介绍如何安装和配置Apache Flume进行日志收集。包括选择版本、解压缩、配置文件设置及启动流程。通过具体配置示例，帮助读者快速上手。

1. 安装

选择适合自己的版本：[下载](https://flume.apache.org/download.html)

2. 解压

gunzip flume-ng-1.6.0-cdh5.5.1.tar.gz

3. 测试

     3.1 mkdir test
     3.2 cd test && mkdir logs
     3.3 touch flume-conf.properties

test.sources = s1
test.channels = c1
test.sinks = s1

test.sources.s1.type = TAILDIR
test.sources.s1.filegroups = t1
test.sources.s1.filegroups.t1= /data/flume/test/test.*.log
test.sources.s1.batchSize = 100
test.sources.s1.positionFile = /data/flume/test/position.json
test.sources.s1.channels = c1
test.sources.s1.writePosInterval = 5000
test.sources.s1.skipToEnd = false

test.channels.c1.type = SpillableMemory
test.channels.c1.memoryCapacity = 100000
test.channels.c1.byteCapacity = 134217728
test.channels.c1.checkpointDir = /data/flume/test/checkpoint
test.channels.c1.dataDirs = /data/flume/test/data

test.sinks.s1.type = logger
test.sinks.s1.channel = c1
3.4 touch flume-env.sh
HTTP_MONITORING_PORT=34520
export JAVA_OPTS=”$JAVA_OPTS -Xms256m -Xmx256m -Dflume.project.dir=$opt_conf -Dflume.monitoring.type=http -Dflume.monitoring.port=\$HTTP_MONITORING_PORT”
3.5 touch log4j.properties
flume.root.logger=INFO,LOGFILE
flume.log.dir=${flume.project.dir}/logs
flume.log.file=flume.log

log4j.logger.org.apache.flume.lifecycle = INFO
log4j.logger.org.jboss = WARN
log4j.logger.org.mortbay = INFO
log4j.logger.org.apache.avro.ipc.NettyTransceiver = WARN
log4j.logger.org.apache.hadoop = INFO
log4j.logger.org.apache.hadoop.hive = ERROR

log4j.rootLogger=${flume.root.logger}
log4j.appender.LOGFILE=org.apache.log4j.RollingFileAppender
log4j.appender.LOGFILE.MaxFileSize=100MB
log4j.appender.LOGFILE.MaxBackupIndex=10
log4j.appender.LOGFILE.File=${flume.log.dir}/${flume.log.file}
log4j.appender.LOGFILE.layout=org.apache.log4j.PatternLayout
log4j.appender.LOGFILE.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-5p [%t] (%C.%M:%L) %x - %m%n

log4j.appender.DAILY=org.apache.log4j.rolling.RollingFileAppender
log4j.appender.DAILY.rollingPolicy=org.apache.log4j.rolling.TimeBasedRollingPolicy
log4j.appender.DAILY.rollingPolicy.ActiveFileName=${flume.log.dir}/${flume.log.file}
log4j.appender.DAILY.rollingPolicy.FileNamePattern=${flume.log.dir}/${flume.log.file}.%d{yyyy-MM-dd}
log4j.appender.DAILY.layout=org.apache.log4j.PatternLayout
log4j.appender.DAILY.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-5p [%t] (%C.%M:%L) %x - %m%n

log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d (%t) [%p - %l] %m%n

4. 启动

/data/flume/apache-flume-1.6.0-cdh5.5.1-bin/bin/flume-ng agent -name test --conf . --conf-file flume-conf.properties
tail -f logs/flume.log