java 网络爬虫项目(8)

最新推荐文章于 2021-02-12 23:51:56 发布

原创最新推荐文章于 2021-02-12 23:51:56 发布 · 380 阅读

0 ·

CC 4.0 BY-SA版权

java 同时被 3 个专栏收录

50 篇文章

订阅专栏

project

14 篇文章

订阅专栏

数据库

10 篇文章

订阅专栏

本文介绍了一个用于解析优快云博客的Java类，包括解析边栏模块的URL列表，以及对数据库进行插入和查询操作的方法。通过使用HtmlCleaner和XPath，实现了动态加载的文章URL抓取。

解析具体模块列表页并优化解析实现类

类如csdn主页的边栏 5g 模块

给page.类添加url列表属性

//存储模块url(包含列表页url和详情页url)
private List<String> urlList= new ArrayList<String>();

连接数据库模块并提供对数据库的各种操作接口

package work.spider.util;

import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;

/*
 * 数据库操作实现类
 * 2020/3/20
 */
public class DatabaseUtil {
	 
	
	public static int insert(String dbname,String command)  {  //更新数据库功能
		
		Connection con = null;    //连接
		 Statement sql = null;		//sql语句
		 ResultSet rs= null;			//返回的结果
		 int result = 0;
		
		try {
			Class.forName("com.mysql.cj.jdbc.Driver");
			
		}
		catch(Exception e) {
			System.out.println("forNameError:"+e);
		}
		String url="jdbc:mysql://127.0.0.1:3306/"+dbname+"?serverTimezone=UTC";
		String user="root";
		String password="root";
		
		try {
			con=DriverManager.getConnection(url,user,password);
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		
			try {
				sql = con.createStatement();
			    result= sql.executeUpdate(command);
				con.close();
				
				
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}

			return result;
	}
	
	public static boolean isfind(String dbname,String URL) {  //查询数据库是否已经存在相关字段的数据
		boolean flag=false;
		Connection con = null;    //连接
		 Statement sql = null;		//sql语句
		 ResultSet rs= null;			//返回的结果
		 int result = 0;
		
		try {
			Class.forName("com.mysql.cj.jdbc.Driver");
			
		}
		catch(Exception e) {
			System.out.println("forNameError:"+e);
		}
		String url="jdbc:mysql://127.0.0.1:3306/"+dbname+"?serverTimezone=UTC";
		String user="root";
		String password="root";
		
		try {
			con=DriverManager.getConnection(url,user,password);
		} catch (SQLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		
			try {
				sql = con.createStatement();
			    rs= sql.executeQuery("SELECT * FROM csdn WHERE url='"+URL+"'");
			    while(rs.next()) {
			    	result++;
			    }
			    
				con.close();
				
				
			} catch (SQLException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			if(result!=0) flag=true;
			return flag;
				
	}
/* rs=sql.executeQuery("SELECT * FROM mess"); //查询语句
while(rs.next()) {
	String number = rs.getString(1);
	String name= rs.getString(2);
	Date date=rs.getDate(3);
	float height = rs.getFloat(4);*/

}

对解析类的优化、保存一个列表中每一篇文章的url

package work.spider.service.impl;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPatherException;

import work.spider.entity.Page;
import work.spider.service.IProcessService;
import work.spider.util.HtmlUtil;
import work.spider.util.LoadPropertyUtil;
import work.spider.util.RegexUtil;
/**
 * 
 *优快云页面解析实现类
 * @auther lwr
 * create by 2020-03-13
 * */
public class 优快云ProcessService implements IProcessService {
	
	public void process(Page page) {
		// TODO Auto-generated method stub
		String content =page.getContent();
		HtmlCleaner htmlCleaner = new HtmlCleaner();
        TagNode rootNode =htmlCleaner.clean(content);
        
        
        if(page.getUrl().startsWith("https://blog.youkuaiyun.com/")) {
        	//解析博客详情页
        	parseDetail(page,rootNode);
        }
        else {        	
        	//获取详情页的url
        	try {
        		//未加载前的页面
				Object[] evaluateXpath=rootNode.evaluateXPath(LoadPropertyUtil.get优快云("hrefPath1"));
				if(evaluateXpath.length>0) {
					for(Object object:evaluateXpath) {
						TagNode tagNode = (TagNode) object;
						String detailurl = tagNode.getAttributeByName("href");
						page.addUrl(detailurl);
					}
				}
				//加载之后的页面
				evaluateXpath = rootNode.evaluateXPath(LoadPropertyUtil.get优快云("hrefPath2"));
				if(evaluateXpath.length>0) {
					for(Object object:evaluateXpath) {
						TagNode tagNode = (TagNode) object;
						String detailurl = tagNode.getAttributeByName("href");
						page.addUrl(detailurl);
						System.out.println("detailurl:"+detailurl);
					}
				}
				
				
        	} catch (XPatherException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
        }
   
	}
	/*
	 * 
	 * 解析模块详情页
	 */
	public void parseDetail(Page page,TagNode rootNode) {
		//获取总阅读数

	      page.setAllnumber(HtmlUtil.getFieldByRegex(rootNode,LoadPropertyUtil.get优快云("readNumberPath"),LoadPropertyUtil.get优快云("allnumberRegex")));   	  
	//获取评论数
	      
	      page.setCommentNuber(HtmlUtil.getFieldByRegex(rootNode, LoadPropertyUtil.get优快云("commentNumberPath"),LoadPropertyUtil.get优快云("commentnumberRegex")));
	//获取发文时间
	      
	      page.setPublishtime(HtmlUtil.getFieldByRegex(rootNode,LoadPropertyUtil.get优快云("publishTimePath") ,LoadPropertyUtil.get优快云("publishTime")));
  
	//获取点赞数
	      page.setSupportNumber(HtmlUtil.getFieldByRegex(rootNode,LoadPropertyUtil.get优快云("supportNumberPath") ,LoadPropertyUtil.get优快云("supportNumberRegex")));
	 //获取博主id
	      page.setBlogger(HtmlUtil.getFieldByRegex(rootNode,LoadPropertyUtil.get优快云("bloggerPath") ,LoadPropertyUtil.get优快云("bloggerRegex")));
	      
	 //获取博客标题
	      page.setBlogTitle(HtmlUtil.getFieldByRegex(rootNode,LoadPropertyUtil.get优快云("blogTitlePath") ,LoadPropertyUtil.get优快云("blogTitleRegex")));
	      
	}

}

当前遇到一个问题就是关于 csdn博客的下拉动态加载问题，怎样获取加载出的新的文章的url