storm实时去读实时日志

最新推荐文章于 2025-12-17 20:09:02 发布

转载最新推荐文章于 2025-12-17 20:09:02 发布 · 130 阅读

0 ·

CC 4.0 BY-SA版权

原文链接：http://www.cnblogs.com/atwanli/articles/5726355.html

文章标签：

#大数据 #java

本文介绍了一种智能日志读取系统，该系统能够实时读取并处理日志文件，即使在日志文件名变更时也能无缝切换，避免读写冲突。系统采用行读取方式，每读取一行即更新文件大小值，防止半行数据读取。同时，系统在日志文件重命名后会自动暂停读取，确保数据完整性和准确性。

实时读取日志文件信息，在24点时日志文件名会被重命名导致，若此时读取文件则发送读写冲突，所以采用暂停读取，文件被重命名后再读取

日志文件读取方式：采用行读取，内读取一行就调整一下读取文件大小值；防止出现读取半行数据

package com.mysoft.storm.spout;

import java.io.File;
import java.io.RandomAccessFile;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Map;

import org.apache.log4j.Logger;

import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.topology.base.BaseRichSpout;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;


/**  
* @Title: LogReader.java
* @Package cn.itcast.storm.spout
* @Description: TODO(读取日志)
* @author wwl
* @date 2016年7月18日 下午6:05:58
* @version V1.0  
*/
public class LogReader extends BaseRichSpout {
	private static final Logger logger = Logger.getLogger("order");

	private static final long serialVersionUID = 2197521792014017918L;
	
	private SpoutOutputCollector collector;
	private long lastTimeFileSize = 0;
	private String fileName;
	private File file;
	private SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
	private Date startDate;

	@Override
	@SuppressWarnings({ "rawtypes", "deprecation" })
	public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
		this.collector = collector;
		fileName = (String) conf.get("FILENAME");
		file = new File(fileName);
		Date now=new Date();
		try {
			String startTime="20160101000000";
			startDate=format.parse(startTime);
			startDate.setDate(now.getDate()+1);
			startDate.setMonth(now.getMonth());
			startDate.setYear(now.getYear());
			logger.info("日志日切时间："+format.format(startDate));
		} catch (ParseException e) {
			logger.error("时间格式转化错误，录入数据格式错误");
			e.printStackTrace();
		}
	}

	@SuppressWarnings("deprecation")
	@Override
	public void nextTuple() {
		Date now=new Date();
		if(startDate.getTime()-now.getTime()>0&&startDate.getTime()-now.getTime()<180000){
			try {
				SimpleDateFormat sf = new SimpleDateFormat(".yyyyMMdd");
				String tags=sf.format(now);
				Thread.sleep(180000);
				workPreFile(tags);
				startDate.setDate(startDate.getDate()+1);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}  
		}else{
			work();
		}
	}
	
	/**
	 * 读取当天日志
	 */
	public void work() {

		RandomAccessFile randomFile = null;
		logger.info("size " + lastTimeFileSize);
		try {
			randomFile = new RandomAccessFile(file, "r");
			randomFile.seek(lastTimeFileSize);
			String line = null;
			StringBuffer sb=new StringBuffer();
			while ((line = randomFile.readLine()) != null) {
				if(line.contains("END")){//航结束标识，排除读取前半行数据情况
					collector.emit(new Values(line));
					sb.append(line);
					lastTimeFileSize=lastTimeFileSize+line.getBytes().length;
					logger.info("readline " + line+"  "+lastTimeFileSize);
				}
			}
			randomFile.close();
		} catch (Exception e) {
			logger.error("work()", e);
			e.printStackTrace();
		}

	}
	
	/**
	 * 读取前一天的日志文件
	 */
	public void workPreFile(String tags) {
		logger.info("workPreFile(String) - start");

		RandomAccessFile randomFile = null;
		try {
			File preFile=new File(fileName+tags);
			randomFile = new RandomAccessFile(preFile, "r");
			randomFile.seek(lastTimeFileSize);
			String line = null;
			while ((line = randomFile.readLine()) != null) {
				collector.emit(new Values(line));
				logger.info("readline " + line);
			}
			randomFile.close();
			lastTimeFileSize=0;
		} catch (Exception e) {
			logger.error("workPreFile(String)", e);
			e.printStackTrace();
		}

		logger.info("workPreFile(String) - end");
	}
	
	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("line"));
	}

}

转载于:https://www.cnblogs.com/atwanli/articles/5726355.html