flumeng收集日志并存储到HDFS

最新推荐文章于 2021-04-04 03:33:23 发布

XiaoGuang-Xu

最新推荐文章于 2021-04-04 03:33:23 发布

阅读量735

点赞数

分类专栏： Flume-Ng

Flume-Ng 专栏收录该内容

10 篇文章

订阅专栏

flumeng收集日志并存储到HDFS

转载：http://blog.youkuaiyun.com/lskyne/article/details/37567489

两台机器agent1收集日志，传送到agent3，agent3上传日志到HDFS

agent1配置：

[plain]view plaincopy 
   
 agent1.sources=s1  
 agent1.channels=c1  
 agent1.sinks=k1  
   
   
 agent1.sources.s1.type = spooldir  
 agent1.sources.s1.spoolDir = /var/log/hutxlog  
 agent1.sources.s1.fileHeader =false  
 agent1.sources.s1.channels = c1  
   
   
 agent1.sinks.k1.channel=c1  
 agent1.sinks.k1.type=avro  
 agent1.sinks.k1.hostname=10.1.72.97  
 agent1.sinks.k1.port=12343  
 agent1.channels.c1.type=memory  
   
   
 agent1.sinks.sink-1.type = hdfs  
 agent1.sinks.sink-1.hdfs.kerberosPrincipal=$KERBEROS_PRINCIPAL  
 agent1.sinks.sink-1.hdfs.kerberosKeytab =$KERBEROS_KEYTAB  
 agent1.sinks.sink-1.hdfs.proxyUser = weblogs  

agent3配置

[plain]view plaincopy 
   
 agent3.channels.ch1.type = memory  
 agent3.sources.avro-source1.channels = ch1  
 agent3.sources.avro-source1.type = avro  
 agent3.sources.avro-source1.bind = 0.0.0.0  
 agent3.sources.avro-source1.port = 12343  
   
 agent3.sinks.log-sink1.type=hdfs  
 agent3.sinks.log-sink1.channel=ch1  
 agent3.sinks.log-sink1.hdfs.path=hdfs://10.1.72.94:8020/hutx  
 agent3.sinks.log-sink1.hdfs.fileType=DataStream  
 agent3.sinks.log-sink1.hdfs.writeFormat=TEXT  
 agent3.sinks.log-sink1.hdfs.rollInterval=4  
   
 agent3.channels = ch1  
 agent3.sources = avro-source1  
 agent3.sinks = log-sink1  
   
 agent3.sinks.log-sink1.type = hdfs  
 agent3.sinks.log-sink1.hdfs.kerberosPrincipal=$KERBEROS_PRINCIPAL  
 agent3.sinks.log-sink1.hdfs.kerberosKeytab =$KERBEROS_KEYTAB  

结果

[plain]view plaincopy 
   
 下午3点05:51.301   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Shutdown Metric for type: CHANNEL, name: ch1. channel.capacity == 100  
 下午3点05:51.301   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Shutdown Metric for type: CHANNEL, name: ch1. channel.current.size == 95  
 下午3点05:51.301   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Shutdown Metric for type: CHANNEL, name: ch1. channel.event.put.attempt == 95  
 下午3点05:51.301   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Shutdown Metric for type: CHANNEL, name: ch1. channel.event.put.success == 95  
 下午3点05:51.301   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Shutdown Metric for type: CHANNEL, name: ch1. channel.event.take.attempt == 47  
 下午3点05:51.301   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Shutdown Metric for type: CHANNEL, name: ch1. channel.event.take.success == 0  
 下午3点05:51.301   INFO    org.mortbay.log   
 Stopped SelectChannelConnector@0.0.0.0:41414  
 下午3点06:03.092   INFO    org.apache.flume.node.PollingPropertiesFileConfigurationProvider      
 Configuration provider starting  
 下午3点06:03.122   INFO    org.apache.flume.node.PollingPropertiesFileConfigurationProvider      
 Reloading configuration file:/var/run/cloudera-scm-agent/process/1444-flume-AGENT/flume.conf  
 下午3点06:03.127   INFO    org.apache.flume.conf.FlumeConfiguration      
 Processing:log-sink1  
 下午3点06:03.128   INFO    org.apache.flume.conf.FlumeConfiguration      
 Processing:log-sink1  
 下午3点06:03.128   INFO    org.apache.flume.conf.FlumeConfiguration      
 Processing:log-sink1  
 下午3点06:03.129   INFO    org.apache.flume.conf.FlumeConfiguration      
 Processing:log-sink1  
 下午3点06:03.129   INFO    org.apache.flume.conf.FlumeConfiguration      
 Processing:log-sink1  
 下午3点06:03.129   INFO    org.apache.flume.conf.FlumeConfiguration      
 Added sinks: log-sink1 Agent: agent3  
 下午3点06:03.129   INFO    org.apache.flume.conf.FlumeConfiguration      
 Processing:log-sink1  
 下午3点06:03.129   INFO    org.apache.flume.conf.FlumeConfiguration      
 Processing:log-sink1  
 下午3点06:03.129   INFO    org.apache.flume.conf.FlumeConfiguration      
 Processing:log-sink1  
 下午3点06:03.155   INFO    org.apache.flume.conf.FlumeConfiguration      
 Post-validation flume configuration contains configuration for agents: [agent3]  
 下午3点06:03.156   INFO    org.apache.flume.node.AbstractConfigurationProvider   
 Creating channels  
 下午3点06:03.166   INFO    org.apache.flume.channel.DefaultChannelFactory    
 Creating instance of channel ch1 type memory  
 下午3点06:03.172   INFO    org.apache.flume.node.AbstractConfigurationProvider   
 Created channel ch1  
 下午3点06:03.173   INFO    org.apache.flume.source.DefaultSourceFactory      
 Creating instance of source avro-source1, type avro  
 下午3点06:03.196   INFO    org.apache.flume.sink.DefaultSinkFactory      
 Creating instance of sink: log-sink1, type: hdfs  
 下午3点06:03.786   INFO    org.apache.flume.sink.hdfs.HDFSEventSink      
 Hadoop Security enabled: true  
 下午3点06:03.816   INFO    org.apache.flume.sink.hdfs.HDFSEventSink      
 { Sink type:HDFSEventSink, name:log-sink1 }: Attempting kerberos login as principal (flume/gcstat3@CDH) from keytab file (/var/run/cloudera-scm-agent/process/1444-flume-AGENT/flume.keytab)  
 下午3点06:04.107   INFO    org.apache.hadoop.security.UserGroupInformation   
 Login successful for user flume/gcstat3@CDH using keytab file /var/run/cloudera-scm-agent/process/1444-flume-AGENT/flume.keytab  
 下午3点06:04.107   INFO    org.apache.flume.sink.hdfs.HDFSEventSink      
 Auth method: KERBEROS  
 下午3点06:04.107   INFO    org.apache.flume.sink.hdfs.HDFSEventSink      
 User name: flume/gcstat3@CDH  
 下午3点06:04.107   INFO    org.apache.flume.sink.hdfs.HDFSEventSink      
 Using keytab: true  
 下午3点06:04.108   INFO    org.apache.flume.sink.hdfs.HDFSEventSink      
 Logged in as user flume/gcstat3@CDH  
 下午3点06:04.112   INFO    org.apache.flume.node.AbstractConfigurationProvider   
 Channel ch1 connected to [avro-source1, log-sink1]  
 下午3点06:04.121   INFO    org.apache.flume.node.Application     
 Starting new configuration:{ sourceRunners:{avro-source1=EventDrivenSourceRunner: { source:Avro source avro-source1: { bindAddress: 0.0.0.0, port: 12343 } }} sinkRunners:{log-sink1=SinkRunner: { policy:org.apache.flume.sink.DefaultSinkProcessor@2f93e4a8 counterGroup:{ name:null counters:{} } }} channels:{ch1=org.apache.flume.channel.MemoryChannel{name: ch1}} }  
 下午3点06:04.125   INFO    org.apache.flume.node.Application     
 Starting Channel ch1  
 下午3点06:04.180   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Monitored counter group for type: CHANNEL, name: ch1: Successfully registered new MBean.  
 下午3点06:04.180   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Component type: CHANNEL, name: ch1 started  
 下午3点06:04.181   INFO    org.apache.flume.node.Application     
 Starting Sink log-sink1  
 下午3点06:04.182   INFO    org.apache.flume.node.Application     
 Starting Source avro-source1  
 下午3点06:04.182   INFO    org.apache.flume.source.AvroSource    
 Starting Avro source avro-source1: { bindAddress: 0.0.0.0, port: 12343 }...  
 下午3点06:04.183   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Monitored counter group for type: SINK, name: log-sink1: Successfully registered new MBean.  
 下午3点06:04.183   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Component type: SINK, name: log-sink1 started  
 下午3点06:04.238   INFO    org.mortbay.log   
 Logging to org.slf4j.impl.Log4jLoggerAdapter(org.mortbay.log) via org.mortbay.log.Slf4jLog  
 下午3点06:04.305   INFO    org.mortbay.log   
 jetty-6.1.26  
 下午3点06:04.336   INFO    org.mortbay.log   
 Started SelectChannelConnector@0.0.0.0:41414  
 下午3点06:04.579   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Monitored counter group for type: SOURCE, name: avro-source1: Successfully registered new MBean.  
 下午3点06:04.579   INFO    org.apache.flume.instrumentation.MonitoredCounterGroup    
 Component type: SOURCE, name: avro-source1 started  
 下午3点06:04.580   INFO    org.apache.flume.source.AvroSource    
 Avro source avro-source1 started.  
 下午3点06:33.731   INFO    org.apache.avro.ipc.NettyServer   
 [id: 0x03072115, /10.1.72.94:59951 => /10.1.72.97:12343] OPEN  
 下午3点06:33.734   INFO    org.apache.avro.ipc.NettyServer   
 [id: 0x03072115, /10.1.72.94:59951 => /10.1.72.97:12343] BOUND: /10.1.72.97:12343  
 下午3点06:33.734   INFO    org.apache.avro.ipc.NettyServer   
 [id: 0x03072115, /10.1.72.94:59951 => /10.1.72.97:12343] CONNECTED: /10.1.72.94:59951  
 下午3点06:34.197   INFO    org.apache.flume.sink.hdfs.HDFSDataStream     
 Serializer = TEXT, UseRawLocalFileSystem = false  
 下午3点06:34.311   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Creating hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194198.tmp  
 下午3点06:35.644   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Renaming hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194198.tmp to hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194198  
 下午3点06:35.709   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Creating hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194199.tmp  
 下午3点06:35.826   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Renaming hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194199.tmp to hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194199  
 下午3点06:35.879   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Creating hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194200.tmp  
 下午3点06:35.993   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Renaming hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194200.tmp to hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194200  
 下午3点06:36.040   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Creating hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194201.tmp  
 下午3点06:36.151   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Renaming hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194201.tmp to hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194201  
 下午3点06:36.201   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Creating hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194202.tmp  
 下午3点06:36.301   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Renaming hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194202.tmp to hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194202  
 下午3点06:36.344   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Creating hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194203.tmp  
 下午3点06:36.460   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Renaming hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194203.tmp to hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194203  
 下午3点06:36.505   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Creating hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194204.tmp  
 下午3点06:40.568   INFO    org.apache.flume.sink.hdfs.BucketWriter   
 Renaming hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194204.tmp to hdfs://10.1.72.94:8020/hutx/FlumeData.1404803194204  
 下午3点06:40.574   INFO    org.apache.flume.sink.hdfs.HDFSEventSink      
 Writer callback called.  