彷徨 | 打印日志文件并将其上传存到HDFS中

最新推荐文章于 2024-11-16 19:00:01 发布

原创最新推荐文章于 2024-11-16 19:00:01 发布 · 644 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#日志采集与上传

大数据同时被 2 个专栏收录

66 篇文章

订阅专栏

Hadoop

27 篇文章

订阅专栏

本文介绍了一个完整的日志管理系统，包括日志生成、采集、上传至HDFS及日志清理过程。系统通过定时任务实现日志文件的自动管理，利用log4j进行日志记录，并将日志文件按需上传至HDFS集群。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

在本地打印日志 , 对日志文件进行分析 , 把需要上传的日志文件移动到待上传目录 , 将待上传的日志文件逐个上传到HDFS , 并移动到备份目录 , 定期清理过期备份日志 .

1.首先生产日志文件:

/**
 * 生产日志
 * @author Administrator
 *
 */
public class MoreLog {
	public static void main(String[] args) throws Exception {
		Logger logger = LogManager.getLogger(MoreLog.class);
		System.out.println("开始打印日志了");
		while(true){
			logger.info("张俊杰zhangjunjie张俊杰zhangjunjie张俊杰zhangjunjie");
			Thread.sleep(1);
		}
	}
}

2.启动日志采集和日志清理

/**
 * 启动日志采集
 * 和日志清理
 * @author Administrator
 *
 */

public class StartUp {
	public static void main(String[] args) {
		Timer timer = new Timer();
		//上传
		timer.schedule(new ClollactionTask(), 0, 2*60*1000l);
		
		//清理文件
		timer.schedule(new ClearTask(), 0, 24*60*1000);
	}
}

3.日志采集并上传HDFS

import java.io.File;
import java.io.FilenameFilter;
import java.net.InetAddress;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimerTask;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
 * 日志采集的:
 * 1.从日志目录里面查看哪些需要上传(1.2.3.....)
 * 2.把需要上传的文件移动到待上传文件
 * 3.上传到hdfs上(/log/2018-8-20/xxx.log)
 * 4.将上传完成的日志文件移动到备份目录
 * @author Administrator
 *
 */

public class ClollactionTask extends TimerTask{

	@Override
	public void run() {
		System.out.println("开始执行任务");
		try {
			SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
			String datetime = format.format(new Date());
			
			// 1:查看哪些文件是需要上传的
			File logDir = new File("e:/testlog/");
			File[] listFiles = logDir.listFiles(new FilenameFilter() {
				// FileNameFilter 是过哪些文件能够获取的
				@Override
				public boolean accept(File dir, String name) {
					return name.startsWith("test.log.");
				}
			});

			// 2:将文件移动到待上传目录
			for (File file : listFiles) {
				// file.renameTo(new File(""));
				FileUtils.moveFileToDirectory(file, new File("e:/waitUpLoad"), true);
			}
			
			//3:将待上传的文件逐个上传到hdfs上，并移动到备份目录
			FileSystem fs = FileSystem.get(new URI("hdfs://hadoop01:9000"), new Configuration(), "root");
			Path detPath = new Path("/log/"+datetime.substring(0, 10));
			//判断待上传的目录是否存在，不存在则创建一个（hdfs上的文件夹）
			boolean exists = fs.exists(detPath);
			if(!exists){
				fs.mkdirs(detPath);
			}
			
			//判断备份目录是否存在
			File backDir = new File("e:/backDir/"+datetime);
			boolean exists2 = backDir.exists();
			if(!exists2){
				backDir.mkdirs();
			}
			
			//得到上传的是哪一个服务上的日志文件
			String hostName = InetAddress.getLocalHost().getHostName();
			
			//4:遍历待上传的目录
			File file = new File("e:/waitUpLoad");
			File[] listFile = file.listFiles();
			for (File f : listFile) {
				//上传到hdfs上
				fs.copyFromLocalFile(new Path(f.getPath()), new Path(detPath,hostName+"_"+f.getName()+"_"+System.currentTimeMillis()));
				
				//cp到备份目录
				FileUtils.moveFileToDirectory(f, backDir, true);
			}
			fs.close();
			
		} catch (Exception e) {
			e.getStackTrace();
		}

	}
	
}

4.清理过期日志

import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimerTask;

import org.apache.commons.io.FileUtils;
/**
 * 清理过期日志
 * @author Administrator
 *
 */
public class ClearTask extends TimerTask {

	@Override
	public void run() {
		try {		
			SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm");
			Date nowdate = new Date();
			
			File file = new File("E:/backDir");
			File[] listFiles = file.listFiles();
			
			for (File dir : listFiles) {
				String name = dir.getName();
				Date dirdate = format.parse(name);
				if(nowdate.getTime() - dirdate.getTime() > 24*60*60*1000) {
					FileUtils.deleteDirectory(dir);
				}				
			}
		} catch (Exception e) {
			e.printStackTrace();
		} 
	}
}

附:log4j配置文件

log4j.rootLogger =ALL,systemOut,logRollingFile
#log4j.rootLogger =ALL,systemOut,logFile,logDailyFile,logRollingFile,logMail,logDB 
#\u8F93\u51FA\u5230\u63A7\u5236\u53F0 
log4j.appender.systemOut = org.apache.log4j.ConsoleAppender 
log4j.appender.systemOut.layout = org.apache.log4j.PatternLayout 
log4j.appender.systemOut.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n 
log4j.appender.systemOut.Threshold = DEBUG 
log4j.appender.systemOut.ImmediateFlush = TRUE 
log4j.appender.systemOut.Target = System.out 

#\u8F93\u51FA\u5230\u6587\u4EF6 
log4j.appender.logFile = org.apache.log4j.FileAppender 
log4j.appender.logFile.layout = org.apache.log4j.PatternLayout 
log4j.appender.logFile.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n 
log4j.appender.logFile.Threshold = WARN 
log4j.appender.logFile.ImmediateFlush = TRUE 
log4j.appender.logFile.Append = TRUE 
log4j.appender.logFile.File = ../Struts2/WebRoot/log/File/log4j_Struts.log 
log4j.appender.logFile.Encoding = UTF-8 

#\u6309DatePattern\u8F93\u51FA\u5230\u6587\u4EF6 
log4j.appender.logDailyFile = org.apache.log4j.DailyRollingFileAppender 
log4j.appender.logDailyFile.layout = org.apache.log4j.PatternLayout 
log4j.appender.logDailyFile.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n 
log4j.appender.logDailyFile.Threshold = DEBUG 
log4j.appender.logDailyFile.ImmediateFlush = TRUE 
log4j.appender.logDailyFile.Append = TRUE 
log4j.appender.logDailyFile.File = ../Struts2/WebRoot/log/DailyFile/log4j_Struts 
log4j.appender.logDailyFile.DatePattern = '.'yyyy-MM-dd-HH-mm'.log' 
log4j.appender.logDailyFile.Encoding = UTF-8 

#\u8BBE\u5B9A\u6587\u4EF6\u5927\u5C0F\u8F93\u51FA\u5230\u6587\u4EF6 
log4j.appender.logRollingFile = org.apache.log4j.RollingFileAppender 
log4j.appender.logRollingFile.layout = org.apache.log4j.PatternLayout 
log4j.appender.logRollingFile.layout.ConversionPattern = [%-5p][%-22d{yyyy/MM/dd HH:mm:ssS}][%l]%n%m%n 
log4j.appender.logRollingFile.Threshold = DEBUG 
log4j.appender.logRollingFile.ImmediateFlush = TRUE 
log4j.appender.logRollingFile.Append = TRUE 
log4j.appender.logRollingFile.File = e:/testlog/test.log
log4j.appender.logRollingFile.MaxFileSize = 1MB 
log4j.appender.logRollingFile.MaxBackupIndex = 50 
log4j.appender.logRollingFile.Encoding = UTF-8 

#\u7528Email\u53D1\u9001\u65E5\u5FD7 
log4j.appender.logMail = org.apache.log4j.net.SMTPAppender 
log4j.appender.logMail.layout = org.apache.log4j.HTMLLayout 
log4j.appender.logMail.layout.LocationInfo = TRUE 
log4j.appender.logMail.layout.Title = Struts2 Mail LogFile 
log4j.appender.logMail.Threshold = DEBUG 
log4j.appender.logMail.SMTPDebug = FALSE 
log4j.appender.logMail.SMTPHost = SMTP.163.com 
log4j.appender.logMail.From = xly3000@163.com 
log4j.appender.logMail.To = xly3000@gmail.com 
#log4j.appender.logMail.Cc = xly3000@gmail.com 
#log4j.appender.logMail.Bcc = xly3000@gmail.com 
log4j.appender.logMail.SMTPUsername = xly3000 
log4j.appender.logMail.SMTPPassword = 1234567 
log4j.appender.logMail.Subject = Log4j Log Messages 
#log4j.appender.logMail.BufferSize = 1024 
#log4j.appender.logMail.SMTPAuth = TRUE 

#\u5C06\u65E5\u5FD7\u767B\u5F55\u5230MySQL\u6570\u636E\u5E93 
log4j.appender.logDB = org.apache.log4j.jdbc.JDBCAppender 
log4j.appender.logDB.layout = org.apache.log4j.PatternLayout 
log4j.appender.logDB.Driver = com.mysql.jdbc.Driver 
log4j.appender.logDB.URL = jdbc:mysql://127.0.0.1:3306/xly 
log4j.appender.logDB.User = root 
log4j.appender.logDB.Password = 123456 
log4j.appender.logDB.Sql = INSERT INTOT_log4j(project_name,create_date,level,category,file_name,thread_name,line,all_category,message)values('Struts2','%d{yyyy-MM-ddHH:mm:ss}','%p','%c','%F','%t','%L','%l','%m')