根据web服务器log进行简单的流量分析

最近做的一个网站上线，要做流量统计。网上有一些现成的将js添加到页面上的统计工具，但从后台客户注册数据看，有些没有统计到。做网站的时候比较仓促，自己并未在服务器端进行IP跟踪和计数，幸好服务器提供商默认log了访问者，就依次做做分析。

主要运用了以下几点知识。

1.ftp登录并获取文件；

2.读写文件；

3.对log文件进行分析；

4.容器；

5.IP查询。

 

自我感觉有三个地方如果能改良就更好了：

1.增加图形界面，产品化。

2.IP地址与实际地址的对应关系，未知哪里有比较权威的参考？

3.log文件的分析，这个demo里只是从log的内容进行了简单的归纳分类，并未从服务器提供商处获取log的规律。

 

附上代码：IPAnalyze.java

/* @(#)IPAnalyze.java 2010-7-13上午11:27:21 * * Copyright 2003-2010 TONECON Hearing Equipment co,ltd.All right reserved. */ package com.tonecon; import java.io.BufferedReader; import java.io.DataInputStream; import java.io.File; import java.io.FileOutputStream; import java.io.FileReader; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintStream; import java.io.RandomAccessFile; import java.io.Reader; import java.net.URL; import java.net.URLConnection; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import sun.net.TelnetInputStream; import sun.net.ftp.FtpClient; /** * * @author * @create-time 2010-7-13上午11:27:21 * @revision 3.0 */ public class IPAnalyze { private static final String ORIGINAL_PATTERN = "dd/MMM/yyyy:HH:mm:ss Z"; private static final String DATETIME_PATTERN = "yyyy-MM-dd HH:mm:ss"; private static final String DATE_PATTERN = "yyyy-MM-dd"; /** * @param args * @throws IOException * @throws ParseException */ public static void main(String[] args) throws IOException, ParseException { IPAnalyze ip = new IPAnalyze(); // String[] names = new String[2]; // names[0] = "2010-07-12"; // names[1] = "2010-07-13"; // File[] files = ip.getFtpFiles(names); // Map<String, Set<String>> ipMap = ip.readLogToMap(files); // ip.outPutReport(ipMap); String name = "2010-07-14"; File files = ip.getFtpFile(name); Map<String, Set<String>> ipMap = ip.readLogToMap(files); ip.outPutReport(ipMap); } public File getFtpFile(String name) throws IOException { // 连接ftp服务器并读取log文件 FtpClient fc = new FtpClient("ftp address", 21); fc.login("username", "password"); fc.cd("tomcatlog"); TelnetInputStream tis = fc.get("tomcat_access_log." + name + ".txt"); // 写log文件到本地 File fGetted = new File("D://tomcat_access_log." + name + ".txt"); RandomAccessFile fGet = new RandomAccessFile(fGetted, "rw"); fGet.seek(0); DataInputStream puts = new DataInputStream(tis); int ch; while ((ch = puts.read()) >= 0) { fGet.write(ch); } puts.close(); fGet.close(); tis.close(); fc.closeServer(); return fGetted; } public File[] getFtpFiles(String[] names) throws IOException { // 连接ftp服务器并读取log文件 FtpClient fc = new FtpClient("121.199.124.114", 21); fc.login("zky1f1", "10nian7yueftp"); fc.cd("tomcatlog"); File[] files = new File[names.length]; TelnetInputStream tis = null; File fGetted = null; RandomAccessFile fGet = null; DataInputStream puts = null; for (int i = 0; i < names.length; i++) { tis = fc.get("tomcat_access_log." + names[i] + ".txt"); // 写log文件到本地 fGetted = new File("D://tomcat_access_log." + names[i] + ".txt"); fGet = new RandomAccessFile(fGetted, "rw"); fGet.seek(0); puts = new DataInputStream(tis); int ch; while ((ch = puts.read()) >= 0) { fGet.write(ch); } puts.close(); fGet.close(); tis.close(); files[i] = fGetted; } fc.closeServer(); return files; } public Map<String, Set<String>> readLogToMap(File fGetted) throws IOException, ParseException { SimpleDateFormat sdf1 = new SimpleDateFormat(ORIGINAL_PATTERN, Locale.US); SimpleDateFormat sdf2 = new SimpleDateFormat(DATETIME_PATTERN); // 取log文件中有用信息到内存 Map<String, Set<String>> ipMap = new TreeMap<String, Set<String>>(); Reader is = new FileReader(fGetted); BufferedReader br = new BufferedReader(is); String s1 = null; while ((s1 = br.readLine()) != null) { String[] segs = s1.split(" - - "); if (segs[1].contains(".jsp")) { String dStr = segs[1].substring(1, 27); String ip = segs[0]; if (ipMap.containsKey(ip)) { ipMap.get(ip).add(sdf2.format(sdf1.parse(dStr))); } else { Set<String> dateSet = new TreeSet<String>(); dateSet.add(sdf2.format(sdf1.parse(dStr))); ipMap.put(ip, dateSet); } } } br.close(); is.close(); return ipMap; } /** * * @param fsGetted * @return * @throws IOException * @throws ParseException */ public Map<String, Set<String>> readLogToMap(File[] fsGetted) throws IOException, ParseException { SimpleDateFormat sdf1 = new SimpleDateFormat(ORIGINAL_PATTERN, Locale.US); SimpleDateFormat sdf2 = new SimpleDateFormat(DATETIME_PATTERN); // 取log文件中有用信息到内存 Map<String, Set<String>> ipMap = new TreeMap<String, Set<String>>(); for (File fGetted : fsGetted) { Reader fr = new FileReader(fGetted); BufferedReader br = new BufferedReader(fr); String s1 = null; while ((s1 = br.readLine()) != null) { String[] segs = s1.split(" - - "); if (segs[1].contains(".jsp")) { String dStr = segs[1].substring(1, 27); String ip = segs[0]; if (ipMap.containsKey(ip)) { ipMap.get(ip).add(sdf2.format(sdf1.parse(dStr))); } else { Set<String> dateSet = new TreeSet<String>(); dateSet.add(sdf2.format(sdf1.parse(dStr))); ipMap.put(ip, dateSet); } } } br.close(); fr.close(); } return ipMap; } public void outPutReport(Map<String, Set<String>> ipMap) throws IOException { SimpleDateFormat sdf3 = new SimpleDateFormat(DATE_PATTERN); // 输出结果报告 FileOutputStream fout = new FileOutputStream("D://ipReport." + sdf3.format(new Date()) + ".txt"); PrintStream ps = new PrintStream(fout); URL realUrl = null; URLConnection conn = null; BufferedReader br = null; for (String str : ipMap.keySet()) { ps.println("IP地址：" + str); // 使用第三方IP查询接口 realUrl = new URL("http://www.5luyu.cn/get.aspx?ip=" + str); // URL realUrl = new URL( // "http://www.youdao.com/smartresult-xml/search.s?type=ip&q=" // + str); conn = realUrl.openConnection(); conn.connect(); br = new BufferedReader(new InputStreamReader( conn.getInputStream(), "gbk")); String line; ps.print("所在地方："); while ((line = br.readLine()) != null) { ps.println(line); } ps.println("于以下时间访问本站："); Set<String> dateSet = ipMap.get(str); for (String date : dateSet) { ps.println(date); } ps.println("----------------------------------------------"); } ps.println("共计:" + ipMap.size()); fout.close(); } }