flume上传hdfs的配置文件

最新推荐文章于 2024-04-02 23:15:55 发布

我不想学习啊

最新推荐文章于 2024-04-02 23:15:55 发布

阅读量265

点赞数

分类专栏：大数据学习文章标签： flume

本文链接：https://blog.youkuaiyun.com/wznb12138/article/details/108055505

版权

大数据学习专栏收录该内容

8 篇文章

订阅专栏

user_friends.sources = userFriendsSource
user_friends.channels = userFriendsChannel
user_friends.sinks = userFriendsSink

user_friends.sources.userFriendsSource.type = spooldir
user_friends.sources.userFriendsSource.spoolDir = /opt/kb07/flumeFile/user_friends
user_friends.sources.userFriendsSource.deserializer = LINE
user_friends.sources.userFriendsSource.deserializer.maxLineLength = 600000
user_friends.sources.userFriendsSource.includePattern = userFriends_[0-9]{4}-[0-9]{2}-[0-9]{2}.csv

user_friends.channels.userFriendsChannel.type = file
user_friends.channels.userFriendsChannel.checkpointDir = /opt/kb07/flumeFile/checkpoint/userFriends
user_friends.channels.userFriendsChannel.dataDirs = /opt/kb07/flumeFile/data/userFriends

user_friends.sinks.userFriendsSink.type = hdfs
user_friends.sinks.userFriendsSink.hdfs.fileType = DataStream
user_friends.sinks.userFriendsSink.hdfs.filePrefix = userFriend
user_friends.sinks.userFriendsSink.hdfs.fileSuffix = .csv
user_friends.sinks.userFriendsSink.hdfs.path = hdfs://192.168.238.140:9000/user/userFriend/%Y-%m-%d
user_friends.sinks.userFriendsSink.hdfs.useLocalTimeStamp = true
user_friends.sinks.userFriendsSink.hdfs.batchSize = 640
user_friends.sinks.userFriendsSink.hdfs.rollCount = 0
user_friends.sinks.userFriendsSink.hdfs.rollSize = 64000000
user_friends.sinks.userFriendsSink.hdfs.rollInterval = 30

user_friends.sources.userFriendsSource.channels = userFriendsChannel
user_friends.sinks.userFriendsSink.channel = userFriendsChannel