Flume入门

安装&配置

wget http://mirror.bit.edu.cn/apache/flume/1.6.0/apache-flume-1.6.0-bin.tar.gz
tar -xzvf apache-flume-1.6.0-bin.tar.gz
ln -s apache-flume-1.6.0-bin flume
cd flume
cp conf/flume-env.sh.template conf/flume-env.sh
cp conf/flume-conf.properties.template conf/flume-conf.properties

其中，conf/flume-env.sh内容修改为：

export JAVA_HOME=/usr/java/jdk1.8.0_45
export FLUME_HOME=../
export JAVA_OPTS="-Xms256m -Xmx2g"
FLUME_CLASSPATH=".:$FLUME_HOME/lib/*"

添加agent conf文件conf/flume-conf.properties

#spooling directory
#agent1.sources.source1.type=spooldir
#agent1.sources.source1.spoolDir=/data/flumetest/data
#agent1.sources.source1.deserializer.maxLineLength=20480000
#agent1.sources.source1.channels=channel1
#agent1.sources.source1.fileHeader=false
#agent1.sources.source1.interceptors=i1
#agent1.sources.source1.interceptors.i1.type=timestamp

agent1.sources.source1.channels = sc
agent1.sources.source1.type = exec
agent1.sources.source1.command = tail -F /data/flumetest/access.log.20160102
agent1.sources.source1.channels=channel1

agent1.sinks.slink1.type=hdfs
agent1.sinks.slink1.hdfs.path=hdfs://nameserver:port/flume/
agent1.sinks.slink1.hdfs.fileType=DataStream
agent1.sinks.slink1.hdfs.writeFormat=TEXT
agent1.sinks.slink1.hdfs.rollInterval=30
agent1.sinks.slink1.channel=channel1
agent1.sinks.slink1.hdfs.filePrefix=%Y-%m-%d
agent1.sinks.slink1.hdfs.batchSize=1000
agent1.sinks.slink1.hdfs.rollSize = 67108864
agent1.sinks.slink1.hdfs.rollCount = 0
agent1.sinks.slink1.hdfs.txnEventMax = 1000
agent1.sinks.slink1.hdfs.callTimeout = 60000
agent1.sinks.slink1.hdfs.appendTimeout = 60000
agent1.sinks.slink1.hdfs.useLocalTimeStamp = true

agent1.channels.channel1.type=file
agent1.channels.channel1.checkpointDir=/data/flumetest/cp/point
agent1.channels.channel1.dataDirs=/data/flumetest/cp
agent1.channels.channel1.capacity=10000
agent1.channels.channel1.transactionCapacity=10000

agent1.sources=source1
agent1.sinks=slink1
agent1.channels=channel1

启动agent：

bin/flume-ng agent -n agent1 -f conf/flume-conf.properties -Dflume.root.logger=DEBUG,LOGFILE,console -Dflume.monitoring.type=http

其中，-n 参数的值必须是conf/flume-conf.properties文件中agent的名字