crawler4j 爬虫网页数据并保存到数据库中

package hotkidclub.controller;


import hotkidclub.model.Politic;
import java.io.IOException;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.regex.Pattern;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;


public class MyCrawler extends WebCrawler{

private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|gif|jpg"
             + "|png|mp3|mp3|zip|gz))$");

private final static String URL = "http://politics.people.com.cn";

/**
* 这方法决定了要抓取的URL及其内容
*/
     @Override
     public boolean shouldVisit(Page referringPage, WebURL url) {
         String href = url.getURL().toLowerCase();
         return !FILTERS.matcher(href).matches()
                && href.contains(URL+"/n1");
     } 
    
    /**
     * 当URL下载完成会调用这个方法，可以获取下载页面的URL，文本，链接，HTML和唯一ID等内容
     */
@Override
     public void visit(Page page) {
    String url = page.getWebURL().getURL();
         System.out.println("URL: " + url);
         if (page.getParseData() instanceof HtmlParseData) {
              HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
              String html = htmlParseData.getHtml();
              Document doc = Jsoup.parse(html);
              List<String> list = new ArrayList<String>();
              Element element1 = doc.select("h1.clear,red").first();
              Elements elements1 = element1.getElementsByAttribute("href");
              list.add(elements1.attr("href"));//获取链接属性的值
              Element element2 = doc.select("div.center,fr").first();
              Elements elements2 = element2.getElementsByAttribute("href");
              for (int i =0;i<9;i++) {
             list.add(elements2.get(i).attr("href"));//获取链接属性的值
              }
              try {
             for (int j = 0;j<list.size();j++) {
            Politic politic = new Politic();
    Document document = Jsoup.connect(URL+list.get(j)).get();//爬取此时的URL数据
    Elements elements3 = document.select("#rwb_zw");
                politic.setContent(elements3.select("p").text());//内容
                Elements elements4 = document.select("div.clearfix,w1000_320,text_title");
                politic.setTitle(elements4.select("h1").text());//标题
                String str = elements4.select("div.box01 .fl").text();
                //String publishedAt = str.substring(0,4)+"-"+str.substring(5,7)+"-"+str.substring(8,10)+" "+str.substring(11,16);//时间格式
                String publishedAt = str.substring(0, 17);
                SimpleDateFormat sf = new SimpleDateFormat("yyyy年MM月dd日HH:mm"); //定义时间格式
                Date date = sf.parse(publishedAt);  //转换成date类型
                politic.setPublishedAt(new Timestamp(date.getTime()));//时间 ：//date类型转换成Timestamp类型
                politic.setSource(str.substring(21));//来源
                if(ConnectionUtil.select(politic.getTitle()) == 0){//判断title是否重复:若为0(不重复)则插入数据
               int  i = ConnectionUtil.insert(politic);
                    if(i == 0){//判断是否插入成功
                   throw new Exception("新增时政要闻资料失败！"); 
                    }
                }else{
               System.out.println("资料已经存在！");
                }
}
}
             catch (IOException e) {
e.printStackTrace();
}catch (ParseException e) {  
                e.printStackTrace();  
            } catch (Exception e) {
e.printStackTrace();
}     
         } 
       }

    }

*************************************************************实体类***************************************************************************

package hotkidclub.model;


import java.sql.Timestamp;




/**
 * 政治   实体pojo
 * @author ZhenhuaYuan
 *
 */
public class Politic {
private Integer key;//主键
private Timestamp publishedAt;//时间
private String source;//来源
private String title;//标题
private String content;//内容


public Integer getKey() {
return key;
}




public void setKey(Integer key) {
this.key = key;
}




public Timestamp getPublishedAt() {
return publishedAt;
}




public void setPublishedAt(Timestamp publishedAt) {
this.publishedAt = publishedAt;
}




public String getSource() {
return source;
}




public void setSource(String source) {
this.source = source;
}




public String getTitle() {
return title;
}




public void setTitle(String title) {
this.title = title;
}




public String getContent() {
return content;
}




public void setContent(String content) {
this.content = content;
}


}

******************************************************************MySQL jdbc数据库连接****************************************************************************************

package hotkidclub.controller;


import hotkidclub.model.Politic;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import com.mysql.jdbc.Connection;
import com.mysql.jdbc.PreparedStatement;


public class ConnectionUtil {
    public static Connection getConn() {
   String driver = "com.mysql.jdbc.Driver";
   String url = "jdbc:mysql://localhost：8080/news";
   String username = "root";
   String password = "123456";
   Connection conn = null;
   try {
       Class.forName(driver); //classLoader,加载对应驱动
       conn = (Connection) DriverManager.getConnection(url, username, password);
   } catch (ClassNotFoundException e) {
       e.printStackTrace();
   } catch (SQLException e) {
       e.printStackTrace();
   }
   return conn;
}
    
    public static int insert(Politic politic) {
   Connection conn = getConn();
   int i = 0;
   String sql = "insert into politic(publishedAt,source,title,content) values(?,?,?,?)";
   PreparedStatement pstmt;
   try {
       pstmt = (PreparedStatement) conn.prepareStatement(sql);
       pstmt.setObject(1, politic.getPublishedAt());//动态传参数
       pstmt.setObject(2, politic.getSource());
       pstmt.setObject(3, politic.getTitle());
       pstmt.setObject(4, politic.getContent()); 
       i = pstmt.executeUpdate();
       pstmt.close();
       conn.close();
   } catch (SQLException e) {
       e.printStackTrace();
   }
   return i;
}
    public static int select(String title) {
   Connection conn = getConn();
   int i = 0;
   String sql = "SELECT count(*) record FROM politic WHERE title=?";
   PreparedStatement pstmt;
   ResultSet rs;
   try {
       pstmt = (PreparedStatement) conn.prepareStatement(sql);
       pstmt.setObject(1, title);
       rs = pstmt.executeQuery();
       while(rs.next()){
        i = rs.getInt("record");
       }
       rs.close();
       pstmt.close();
       conn.close();
   } catch (SQLException e) {
       e.printStackTrace();
   }
   return i;
}
}

**************************************************************************测试类TestCrawler ****************************************************************************************

package hotkidclub.controller;




import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;


public class TestCrawler {


public static void main(String[] args) throws Exception {                            
  String crawlStorageFolder = "/data/crawl/root";//保存爬取数据的目录
           int numberOfCrawlers = 5; //同时开启线程数(并发线程数)
           CrawlConfig config = new CrawlConfig(); //在start之前进行一些配置，爬取速度优化
           config.setCrawlStorageFolder(crawlStorageFolder);
           config.setMaxDepthOfCrawling(1); //种子爬取深度
           config.setMaxPagesToFetch(1); //最多能爬取的页面
           PageFetcher pageFetcher = new PageFetcher(config);//从网上提取内容
           RobotstxtConfig robotstxtConfig = new RobotstxtConfig();//robots协议类
           RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);//读取robots协议，获取是否可以抓取的信息
           CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);//爬行控制器   监听各个爬虫的运行状态并向URL队列添加新的URL
           controller.addSeed("http://politics.people.com.cn/");//添加种子URL的方法
           controller.start(MyCrawler.class, numberOfCrawlers);//开启爬虫的方法

        }  

}