一、拦截器
java写完,打包放入flume的lib文件夹下。
package com.nj;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class InterceptorDemo implements Interceptor {
private List<Event> addHeaderEvents;
@Override
public void initialize() {
addHeaderEvents=new ArrayList<>();
}
@Override
public Event intercept(Event event) {
Map<String,String> headers=event.getHeaders();
String body=new String(event.getBody());
if (body.startsWith("gree")){
headers.put("type","lhy");
}else{
headers.put("type","hmq");
}
return event;
}
@Override
public List<Event> intercept(List<Event> list) {
addHeaderEvents.clear();
for (Event event:list)
{
addHeaderEvents.add(intercept(event));
}
return addHeaderEvents;
}
@Override
public void close() {
}
public static class Builder implements Interceptor.Builder{
@Override
public Interceptor build() {
return new InterceptorDemo();
}
@Override
public void configure(Context context) {
}
}
}
创建/netcat-flume-loggerhdfs.conf文件。写入代码:
a2.sources = r1
a1.channels = c1 c2
a1.sinks = k1 k2
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = com.nj.InterceptorDemo$Builder
a1.sources.r1.selector.type = multiplexing
a1.sources.r1.selector.header = type
a1.sources.r1.selector.mapping.hmq = c1
a1.sources.r1.selector.mapping.lhy = c2
a1.channels.c1.type = memory
a1.channels.c2.type = memory
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.fileType = Datastream
a1.sinks.k1.hdfs.fileProfix = lhy
a1.sinks.k1.hdfs.filesSuffix = .csv
a1.sinks.k1.hdfs.path = hdfs://192.168.255.11:9000/user/lhyDemo//%Y-%m-%d
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.batchSize = 640
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.rollSize = 64000000
a1.sinks.k1.hdfs.rollInterval = 3
a1.sinks.k2.type = hdfs
a1.sinks.k2.hdfs.fileType = Datastream
a1.sinks.k2.hdfs.fileProfix = hmq
a1.sinks.k2.hdfs.filesSuffix = .csv
a1.sinks.k2.hdfs.path = hdfs://192.168.255.11:9000/user/hmqDemo//%Y-%m-%d
a1.sinks.k2.hdfs.useLocalTimeStamp = true
a1.sinks.k2.hdfs.batchSize = 640
a1.sinks.k2.hdfs.rollCount = 0
a1.sinks.k2.hdfs.rollSize = 64000000
a1.sinks.k2.hdfs.rollInterval = 3
a1.sources.r1.channels = c1 c2
a1.sinks.k1.channel = c1
a1.sinks.k2.channel = c2
二、数据收集上传hdfs
ser_friends.sources = userFriendsSources
user_friends.channels = userFriendsChannel
user_friends.sinks = userFriendsSink
user_friends.sources.userFriendsSources.type = spooldir
user_friends.sources.userFriendsSources.spoolDir = /opt/data/kb07/user_friends
user_friends.sources.userFriendsSources.deserializer = LINE
user_friends.sources.userFriendsSources.deserializer.maxLineLength = 600000
user_friends.sources.userFriendsSources.includePattern = user_friends_[0-9]{4}_[0-9]{2}_[0-9]{2}.csv
user_friends.channels.userFriendsChannel.type = file
user_friends.channels.userFriendsChannel.checkpointDir = /opt/data/kb07/flumeFile/checkpoint/userFriends
user_friends.channels.userFriendsChannel.dataDirs = /opt/data/kb07/flumeFile/data/userFriends
user_friends.sinks.userFriendsSink.type = hdfs
user_friends.sinks.userFriendsSink.hdfs.fileType = DataStream
user_friends.sinks.userFriendsSink.hdfs.filePrefix = userFriend
user_friends.sinks.userFriendsSink.hdfs.fileSuffix = .csv
user_friends.sinks.userFriendsSink.hdfs.path = hdfs://192.168.255.11:9000/user/userFriend/%Y-%m-%d
user_friends.sinks.userFriendsSink.hdfs.useLocalTimeStamp = true
user_friends.sinks.userFriendsSink.hdfs.batchSize = 640
user_friends.sinks.userFriendsSink.hdfs.rollCount = 0
user_friends.sinks.userFriendsSink.hdfs.rollSize = 64000000
user_friends.sinks.userFriendsSink.hdfs.rollInterval = 30
user_friends.sources.userFriendsSources.channels = userFriendsChannel
user_friends.sinks.userFriendsSink.channel = userFriendsChannel
三、启动Flume
上传文件到Linux中,解压缩。进入文件夹下。
cd ./flume
cd ./conf/
拷贝文件,修改配置
cp flume-env.sh.template flume-env.sh
vi ./flume-env.sh
export JAVA_HOME=/opt/jdk
export JAVA_OPTS="-Xms2000m -Xmx2000m -Dcom.sun.management.jmxremote"
在flume的conf文件夹下,创建文件夹job
mkdir job
[root@qqq job]# yum install -y nc
[root@qqq job]# nc -lk 44444
[root@qqq ~]# yum list telnet* 列出telnet相关的安装包
[root@qqq ~]# yum install telnet-server 安装telnet服务
[root@qqq ~]# yum install telnet.* 安装telnet客户端
[root@qqq ~]# telnet localhost 44444
// 读取电脑指定端口44444内容
[root@qqq job]# vi netcat-flume-logger.conf
a1.sources = r1
a1.sinks = k1
a1.channels = c1
a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444
a1.sinks.k1.type = logger
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
[root@qqq flume]# ./bin/flume-ng agent --name a1 --conf conf/ --conf-file conf/job/netcat-flume-logger.conf -Dflume.root.logger=INFO,console
// 读取指定文件内容
[root@qqq job]# vi file-flume-logger.conf
a2.sources = r1
a2.sinks = k1
a2.channels = c1
a2.sources.r1.type = exec
a2.sources.r1.command = tail -f /opt/bigdata/flume160514/conf/job/tmp.txt
a2.sinks.k1.type = logger
a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100
a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1
[root@qqq flume]# ./bin/flume-ng agent --name a2 --conf conf/ --conf-file conf/job/file-flume-logger.conf -Dflume.root.logger=INFO,console
[root@qqq prodata]# wc -l events.csv 查看文件总行数
// 读取指定文件夹中复合正则表达式的文件内容
[root@qqq job]# vi ./events-flume-logger.conf
events.sources = eventsSource
events.channels = eventsChannel
events.sinks = eventsSink
events.sinks.eventsSink.type = logger
events.sources.eventsSource.type = spooldir
events.sources.eventsSource.spoolDir = /opt/kb07file/flumeFile/events
events.sources.eventsSource.deserializer = LINE
events.sources.eventsSource.deserializer.maxLineLength = 32000
events.sources.eventsSource.includePattern = events_[0-9]{4}-[0-9]{2}-[0-9]{2}.csv
events.channels.eventsChannel.type = file
events.channels.eventsChannel.checkpointDir = /opt/kb07file/flumeFile/checkpoint/events
events.channels.eventsChannel.dataDirs = /opt/kb07file/flumeFile/data/events
events.sources.eventsSource.channels = eventsChannel
events.sinks.eventsSink.channel = eventsChannel
[root@qqq flume]# ./bin/flume-ng agent --name events --conf conf/ --conf-file conf/job/events-flume-logger.conf -Dflume.root.logger=INFO,console