一个简单的网页爬虫程序:
/**
* 网页爬虫 :
* 得到网页上的邮箱地址
* 得到网页上的时间戳
*/
public class RegexDemo
{
public static void main(String[] args) throws IOException
{
String url_1 = "http://blog.sina.com.cn/s/blog_515617e60101e151.html";
String url_2 = "http://bbs.tianya.cn/post-934-163735-1.shtml";
//邮箱地址
String reg_1 = "\\w+@\\w+(\\.\\w+)+";
//格式为这样的时间戳:2017-10-25 08:41:27
String reg_2 = "(\\d+-){2}\\d+ (\\d+:){2}\\d+";
webCrawler(url_1, reg_1);//得到网页上的邮箱地址
webCrawler(url_2,reg_2); //得到网页上的时间戳
}
/**
*
* @param str :网址
* @param regex :正则表达式
* @throws IOException
*/
public static void webCrawler(String str,String regex) throws IOException
{
URL url = new URL(str);
URLConnection conn = url.openConnection();
BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String line = null;
Pattern pattern = Pattern.compile(regex);
while((line = br.readLine()) != null)
{
Matcher matcher = pattern.matcher(line);
while(matcher.find())
{
System.out.println(matcher.group());
}
}
br.close();
}
}