Flume日志收集系统的常用命令和常用操作

最新推荐文章于 2024-07-01 13:05:14 发布

不想当和尚

最新推荐文章于 2024-07-01 13:05:14 发布

阅读量331

点赞数

CC 4.0 BY-SA版权

分类专栏：整理文章标签：大数据 flume

本文链接：https://blog.youkuaiyun.com/lhmqy/article/details/108059660

整理专栏收录该内容

20 篇文章

订阅专栏

本文详细介绍了如何使用Flume进行数据采集、处理及上传至HDFS的过程。包括自定义拦截器实现数据预处理，配置文件详解，以及通过Netcat、Spool Dir等源收集数据的具体步骤。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

一、拦截器

java写完，打包放入flume的lib文件夹下。

package com.nj;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class InterceptorDemo implements Interceptor {

    private List<Event> addHeaderEvents;




    @Override
    public void initialize() {
        addHeaderEvents=new ArrayList<>();
    }

    @Override
    public Event intercept(Event event) {
        Map<String,String> headers=event.getHeaders();
        String body=new String(event.getBody());
        if (body.startsWith("gree")){
            headers.put("type","lhy");
        }else{
            headers.put("type","hmq");
        }
        return event;
    }

    @Override
    public List<Event> intercept(List<Event> list) {
        addHeaderEvents.clear();
        for (Event event:list)
        {
            addHeaderEvents.add(intercept(event));
        }
        return addHeaderEvents;
    }

    @Override
    public void close() {

    }

    public static class Builder implements Interceptor.Builder{

        @Override
        public Interceptor build() {
            return new InterceptorDemo();
        }

        @Override
        public void configure(Context context) {

        }
    }
}

创建/netcat-flume-loggerhdfs.conf文件。写入代码：

a2.sources = r1
a1.channels = c1 c2
a1.sinks = k1 k2

a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444

a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = com.nj.InterceptorDemo$Builder

a1.sources.r1.selector.type = multiplexing
a1.sources.r1.selector.header = type
a1.sources.r1.selector.mapping.hmq = c1
a1.sources.r1.selector.mapping.lhy = c2

a1.channels.c1.type = memory
a1.channels.c2.type = memory

a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.fileType = Datastream
a1.sinks.k1.hdfs.fileProfix = lhy
a1.sinks.k1.hdfs.filesSuffix = .csv
a1.sinks.k1.hdfs.path = hdfs://192.168.255.11:9000/user/lhyDemo//%Y-%m-%d
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.batchSize = 640
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.rollSize = 64000000
a1.sinks.k1.hdfs.rollInterval = 3

a1.sinks.k2.type = hdfs
a1.sinks.k2.hdfs.fileType = Datastream
a1.sinks.k2.hdfs.fileProfix = hmq
a1.sinks.k2.hdfs.filesSuffix = .csv
a1.sinks.k2.hdfs.path = hdfs://192.168.255.11:9000/user/hmqDemo//%Y-%m-%d
a1.sinks.k2.hdfs.useLocalTimeStamp = true
a1.sinks.k2.hdfs.batchSize = 640
a1.sinks.k2.hdfs.rollCount = 0
a1.sinks.k2.hdfs.rollSize = 64000000
a1.sinks.k2.hdfs.rollInterval = 3

a1.sources.r1.channels = c1 c2
a1.sinks.k1.channel = c1
a1.sinks.k2.channel = c2

二、数据收集上传hdfs

ser_friends.sources = userFriendsSources
user_friends.channels = userFriendsChannel
user_friends.sinks = userFriendsSink

user_friends.sources.userFriendsSources.type = spooldir
user_friends.sources.userFriendsSources.spoolDir = /opt/data/kb07/user_friends
user_friends.sources.userFriendsSources.deserializer = LINE
user_friends.sources.userFriendsSources.deserializer.maxLineLength = 600000
user_friends.sources.userFriendsSources.includePattern = user_friends_[0-9]{4}_[0-9]{2}_[0-9]{2}.csv

user_friends.channels.userFriendsChannel.type = file
user_friends.channels.userFriendsChannel.checkpointDir = /opt/data/kb07/flumeFile/checkpoint/userFriends
user_friends.channels.userFriendsChannel.dataDirs = /opt/data/kb07/flumeFile/data/userFriends

user_friends.sinks.userFriendsSink.type = hdfs
user_friends.sinks.userFriendsSink.hdfs.fileType = DataStream
user_friends.sinks.userFriendsSink.hdfs.filePrefix = userFriend
user_friends.sinks.userFriendsSink.hdfs.fileSuffix = .csv
user_friends.sinks.userFriendsSink.hdfs.path = hdfs://192.168.255.11:9000/user/userFriend/%Y-%m-%d
user_friends.sinks.userFriendsSink.hdfs.useLocalTimeStamp = true
user_friends.sinks.userFriendsSink.hdfs.batchSize = 640
user_friends.sinks.userFriendsSink.hdfs.rollCount = 0
user_friends.sinks.userFriendsSink.hdfs.rollSize = 64000000
user_friends.sinks.userFriendsSink.hdfs.rollInterval = 30

user_friends.sources.userFriendsSources.channels = userFriendsChannel
user_friends.sinks.userFriendsSink.channel = userFriendsChannel

三、启动Flume

上传文件到Linux中，解压缩。进入文件夹下。

cd ./flume
cd ./conf/

拷贝文件，修改配置

cp flume-env.sh.template flume-env.sh
vi ./flume-env.sh

export JAVA_HOME=/opt/jdk
export JAVA_OPTS="-Xms2000m -Xmx2000m -Dcom.sun.management.jmxremote"

在flume的conf文件夹下，创建文件夹job

mkdir job

[root@qqq job]# yum install -y nc
[root@qqq job]# nc -lk 44444

[root@qqq ~]# yum list telnet* 列出telnet相关的安装包
[root@qqq ~]# yum install telnet-server 安装telnet服务
[root@qqq ~]# yum install telnet.* 安装telnet客户端

[root@qqq ~]# telnet localhost 44444

// 读取电脑指定端口44444内容
[root@qqq job]# vi netcat-flume-logger.conf
a1.sources = r1
a1.sinks = k1
a1.channels = c1

a1.sources.r1.type = netcat
a1.sources.r1.bind = localhost
a1.sources.r1.port = 44444

a1.sinks.k1.type = logger

a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

[root@qqq flume]# ./bin/flume-ng agent --name a1 --conf conf/ --conf-file conf/job/netcat-flume-logger.conf -Dflume.root.logger=INFO,console

// 读取指定文件内容
[root@qqq job]# vi file-flume-logger.conf
a2.sources = r1
a2.sinks = k1
a2.channels = c1

a2.sources.r1.type = exec
a2.sources.r1.command = tail -f /opt/bigdata/flume160514/conf/job/tmp.txt

a2.sinks.k1.type = logger

a2.channels.c1.type = memory
a2.channels.c1.capacity = 1000
a2.channels.c1.transactionCapacity = 100

a2.sources.r1.channels = c1
a2.sinks.k1.channel = c1

[root@qqq flume]# ./bin/flume-ng agent --name a2 --conf conf/ --conf-file conf/job/file-flume-logger.conf -Dflume.root.logger=INFO,console

[root@qqq prodata]# wc -l events.csv 查看文件总行数

// 读取指定文件夹中复合正则表达式的文件内容
[root@qqq job]# vi ./events-flume-logger.conf
events.sources = eventsSource
events.channels = eventsChannel
events.sinks = eventsSink

events.sinks.eventsSink.type = logger

events.sources.eventsSource.type = spooldir
events.sources.eventsSource.spoolDir = /opt/kb07file/flumeFile/events
events.sources.eventsSource.deserializer = LINE
events.sources.eventsSource.deserializer.maxLineLength = 32000
events.sources.eventsSource.includePattern = events_[0-9]{4}-[0-9]{2}-[0-9]{2}.csv

events.channels.eventsChannel.type = file
events.channels.eventsChannel.checkpointDir = /opt/kb07file/flumeFile/checkpoint/events
events.channels.eventsChannel.dataDirs = /opt/kb07file/flumeFile/data/events

events.sources.eventsSource.channels = eventsChannel
events.sinks.eventsSink.channel = eventsChannel

[root@qqq flume]# ./bin/flume-ng agent --name events --conf conf/ --conf-file conf/job/events-flume-logger.conf -Dflume.root.logger=INFO,console