掠夺者—自己写新闻采集器（3）

六、GetWebNews 采集WEB新闻，这是整个掠夺者的核心了，也是比较麻烦的地方。

这里给出的是采集**网新闻的例子，采集后可以将所有与该网站相关的文字、注释等过滤掉，仅留新闻正文和图片，用到了大量的正则表达式，若日后该网站改版则需要更改程序的相应部分。

import java.awt.image.BufferedImage; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.htmlparser.Node; import org.htmlparser.NodeFilter; import org.htmlparser.Parser; import org.htmlparser.filters.NodeClassFilter; import org.htmlparser.filters.OrFilter; import org.htmlparser.nodes.TextNode; import org.htmlparser.tags.LinkTag; import org.htmlparser.util.NodeList; import org.htmlparser.util.ParserException; //下面需要调用上文发布的bean和几个公共类 import cn.**.bean.Article; import cn.**.Common; import cn.**.SqlDao; public class GetWebNews { /** * 一个空构造函数 */ public GetWebNews() { //nothing } /** * 构造函数 * @param classId 要添加的频道ID * @param resourceUrl 从这个地址获得（新闻）信息 * @param code 编码 * @throws Exception */ public GetWebNews(String classId, String resourceUrl, String code) throws Exception { this.releaseArticles(classId, resourceUrl, code); } public int releaseArticles(String classId, String resourceUrl, String code) { SqlDao sdao = new SqlDao(); String dateStr = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()); List<Article> articleList = new ArrayList<Article>(); articleList = getArticleList(resourceUrl, code); if(articleList == null) return 0; String sqlPrefix = "INSERT INTO tboperation (ClsSort, ClsContent, Sort, Zt, usrCode, Img, IsShopTop, ShowTime) "; String sql = ""; Article article = new Article(); for(int i = articleList.size() - 1; i > -1; i--) { article = articleList.get(i); int isShopTop = 1; if(article.getPictureUrl() == "") isShopTop = 0; //写入新闻的SQL句子 sql = sqlPrefix + "values ('" + article.getTitle() + "', '" + article.getContent() + "', '新闻', '" + classId + "', '作者', '" + article.getPictureUrl() + "', " + isShopTop + ", '"+ dateStr +"')"; System.out.println(sql); if(sdao.executeUpdate(sql) != 1) continue; } sdao.closeDB(); return articleList.size(); } /** * 检查文章是否已被采集过，每个被采集过的地址存放到一个表中，日后采集的时候只要判断一下要采集的地址在不在这个表中就可以了 * @param articleUrl 本次采集到的文章地址 * @return * @throws Exception */ public boolean hasRepeatArticle(String articleUrl) { SqlDao sdao = new SqlDao(); String sql = "select id from nArticleUrlOfResource where isWap = 0 and articleUrlOfResource = '" + articleUrl + "'"; if (sdao.rsNext(sql)) { sdao.closeDB(); return true; } String sqlgeted = "insert into nArticleUrlOfResource (articleUrlOfResource, isWap) values ('" + articleUrl + "', 0)"; sdao.executeUpdate(sqlgeted); sdao.closeDB(); return false; } /** * 获取并解析新闻 * @param resourceURL 要采集的频道新闻主页（带新闻列表）如体育、娱乐、国际、国内等频道主页 * @param code 编码 * @throws Exception */ public List<Article> getArticleList(String resourceUrl, String code) { Parser myParser = null; try { myParser = new Parser(resourceUrl); myParser.setEncoding(code); } catch (ParserException e) { // TODO Auto-generated catch block System.out.println("WEB新闻，初始化Parser失败"); e.printStackTrace(); return null; } NodeList nodeList = null; NodeFilter textFilter = new NodeClassFilter(TextNode.class); NodeFilter linkFilter = new NodeClassFilter(LinkTag.class); OrFilter lastFilter = new OrFilter(); lastFilter.setPredicates(new NodeFilter[] { textFilter, linkFilter }); try { nodeList = myParser.parse(lastFilter); } catch (ParserException e) { // TODO Auto-generated catch block System.out.println("WEB新闻，nodeList赋值失败"); e.printStackTrace(); return null; } Node[] nodes = nodeList.toNodeArray(); List<Article> articleList = new ArrayList<Article>(); Article article = new Article(); for (int i = 0; i < nodes.length; i++) { Node anode = (Node) nodes[i]; String articleUrl = ""; if (anode instanceof LinkTag) { LinkTag linknode = (LinkTag) anode; articleUrl = linknode.getLink(); if (Common.isTrimEmpty(articleUrl)) continue; //只要今天的 String todayKey = Common.getFormatNowTime("yyyy-MM/dd"); if(articleUrl.indexOf(todayKey) < 0) continue; //检查该文章是否已经被采集过 if(hasRepeatArticle(articleUrl)) continue; article = getArticleFormUrl(articleUrl, code); if(article == null) continue; articleList.add(article); if(articleList.size() >= 20) break; //System.out.println(articleUrl); } } return articleList; } /** * 从地址解析新闻内容 * @param articleUrl * @param code * @return * @throws Exception */ public Article getArticleFormUrl(String articleUrl, String code) { System.out.println(articleUrl); StringBuffer sb = new StringBuffer(); try { java.net.URL url = new java.net.URL(articleUrl); BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), code)); String line; while((line = in.readLine()) != null) { sb.append(line); } in.close(); }catch(Exception e) { System.out.println("WEB新闻，找不到指定的URL，可能是网络链接失败！"); e.printStackTrace(); return null; } String pageContent = sb.toString(); //System.out.println(pageContent); //网页中几个重要分割点 String titleStart = "<div id=/"Title/">"; String titleEnd = "</div>"; String contentStart = "<div id=/"Content/">"; String contentEnd = "</div>"; //抛掉标题以前的内容 if(pageContent.indexOf(titleStart) >= 0) pageContent = pageContent.substring(pageContent.indexOf(titleStart) + titleStart.length()); else return null; //解析标题 String title = ""; if(pageContent.indexOf(titleEnd) >= 0) title = pageContent.substring(0, pageContent.indexOf(titleEnd)); else return null; if(Common.hasKeyWord(title)) return null; if(title.indexOf("快讯：") >= 0) return null; if(title.indexOf("快讯:") >= 0) return null; if(title.indexOf("组图") >= 0) return null; title = title.replaceAll("<//s*([^>]*)//s*>", "").replaceAll("( )+", "　　").replace(" ", "").replace("'", "’").trim(); //抛掉正文以前的内容 if(pageContent.indexOf(contentStart) >= 0) pageContent = pageContent.substring(pageContent.indexOf(contentStart) + contentStart.length()); else return null; if(pageContent.indexOf("快讯：") >= 0) return null; if(pageContent.indexOf("class=/"pagelink/">") >= 0) return null; //解析正文 String content = ""; if(pageContent.indexOf(contentEnd) >= 0) content = pageContent.substring(0, pageContent.indexOf(contentEnd) + 6); else return null; //干掉新闻链接、延伸阅读等垃圾 content = content.replaceAll("<P([^>]*)//s*>//s*<IMG//s+([^>]*)//s*>(//s*)(</P>)*(<P([^>]*)//s*>( )+//s*<A//s+([^>]*)//s*>.*</A></P>)*//s*</font></div>", ""); if(content.indexOf("alt=新闻链接") >= 0) { content = content.substring(0, content.indexOf("alt=新闻链接")); //砍掉尾巴 content = content.substring(0, content.lastIndexOf("<IMG")); } if(content.indexOf("alt=延伸阅读") >= 0) { content = content.substring(0, content.indexOf("alt=延伸阅读")); content = content.substring(0, content.lastIndexOf("<IMG")); } if(content.indexOf("相关报道：") >= 0) { content = content.substring(0, content.indexOf("相关报道：")); content = content.substring(0, content.lastIndexOf("<P>")); } if(content.indexOf("延伸阅读：") >= 0) { content = content.substring(0, content.indexOf("延伸阅读：")); content = content.substring(0, content.lastIndexOf("<P>")); } if(content.indexOf("新闻回顾") >= 0) { content = content.substring(0, content.indexOf("新闻回顾")); content = content.substring(0, content.lastIndexOf("<P")); } //过滤除分段和图片以外的所有HTML标记 content = content.replaceAll("<UL>.*</UL>", ""); content = content.replaceAll("<LI>.*</LI>", ""); content = content.replaceAll("<BLOCKQUOTE.*</BLOCKQUOTE>", ""); //过滤文章底部附带的新闻链接 content = content.replaceAll("<P><IMG//s+([^>]*)>(<P><A//s+([^>]*)><FONT color=#.{6}><STRONG>.*</STRONG></FONT></A></P>)+", ""); content = content.replaceAll("<P><IMG//s+([^>]*)>(<P><A//s+([^>]*)><STRONG><FONT color=#.{6}>.*</FONT></STRONG></A></P>)+", ""); content = content.replaceAll("<A//s+([^>]*)><STRONG><FONT color=#.{6}>.*</FONT></STRONG></A>", ""); content = content.replaceAll("<A//s+([^>]*)><FONT color=#.{6}><STRONG>.*</STRONG></FONT></A>", ""); //干掉今日聚焦 content = content.replaceAll("<TABLE//s+.{1,}今日国内聚焦.{1,}</TABLE>", ""); content = content.replaceAll("<//s*([^(I)^P])([^>]*)//s*>", "").replaceAll("<//s*(IFRAME)([^>]*)//s*>", "").trim(); content += "</P>"; //干掉点击图片进入专题 content = content.replaceAll("<IMG//s+([^>]*)//s*>点击图片进入专题", ""); content = content.replaceAll("<IMG//s+([^>]*)//s*>点击进入专题(>)*", ""); content = content.replaceAll("<IMG//s+([^>]*)//s*>[（|//(]*点击进入专题[）|//)]*", ""); content = content.replaceAll("<IMG//s+([^>]*)//s*><P align=center>点击上图进入专题</P>", ""); content = content.replaceAll("<IMG//s+([^>]*)//s*>(<P//s+([^>]*)//s*>)*点击.{0,5}进入专题(>)*(</P>)*", ""); //干掉点击详细 content = content.replace(">>>点击详细", ""); if(Common.hasKeyWord(content)) return null; //解析所有图片 List<String> pictureUrlList = new ArrayList<String>(); String pictureUrl = ""; String pictureOldName = ""; String pictureNewName = ""; String pictureUrlRoot = articleUrl.substring(0, articleUrl.lastIndexOf("/") + 1); int imgSite = content.indexOf("<IMG"); if(imgSite > 0) { String patternStr = "//s+src=/"//w*_//d*//.//w{3}/""; Pattern pattern = Pattern.compile(patternStr, Pattern.MULTILINE); Matcher matcher = pattern.matcher(content); while (matcher.find()) { pictureOldName = matcher.group().replaceAll("//s+src=", "").replaceAll("/"", ""); pictureUrl = pictureUrlRoot + pictureOldName; pictureNewName = Common.getPictureFromUrl(pictureUrl, "D://UpLoadIMG"); if(pictureNewName != null) { pictureUrl = "/UpLoadIMG/" + pictureNewName; //检查图片长宽是否符合在首页显示缩略图的要求 File imgFile = new File("D://UpLoadIMG//" + pictureNewName); //读入图片文件 BufferedImage image; try { image = javax.imageio.ImageIO.read(imgFile); //构造Image对象 int width = image.getWidth(); //得到源图宽 int height = image.getHeight(); //得到源图长 //System.out.println(width + "_" + height); //宽大于等于高，宽小于等于高的2倍，宽大于150 if(width >= height && width <= 2 * height && width > 150) { pictureUrlList.add(pictureUrl); } else { if(width > 5 * height && width > 500) { return null; //content = content.replaceAll("<IMG//s+.*//s+src=/"" + pictureOldName + "/"//s+([^>]*)//s*>.*", ""); } } } catch (IOException e) { // TODO Auto-generated catch block System.out.println("WEB新闻，构造图片对象失败！"); e.printStackTrace(); } content = content.replaceAll(pictureOldName, pictureUrl); } else { break; } //System.out.println(pictureUrl); } } //干掉各种尾巴，烦死了 content = content.replaceAll("(>)*更多内容请关注国际频道(<)*", ""); content = content.replaceAll("(<P align=center>)+", "<P align=center>"); content = content.replace("<P align=center></P>", ""); content = content.replace("▲", "").replace("■", ""); //干掉**网***电 content = content.replaceAll("**网.{2,10}[０-９]{1,2}月[０-９]{1,2}日电", ""); content = content.replace("**发", "").replace("（**电）", ""); content = content.replaceAll("( ){2,}", "　　").replace("'", "’").trim(); //干掉记者编辑通讯员，对不起了 content = content.replaceAll("[（|//(][(首席)|(见习)|(实习)]*记者([^）|^//)]*)[）|//)]", ""); content = content.replaceAll("[（|//(]本报驻.{2,15}记者([^）|^//)]*)[）|//)]", " "); content = content.replaceAll("[（|//(]文/表 记者([^）|^//)]*)[）|//)]", ""); content = content.replaceAll("[（|//(]编辑[：|:|//s+]([^）|^//)]*)[）|//)]", ""); content = content.replaceAll("[（|//(]通讯员([^）|^//)]*)[）|//)]", " "); content = content.replaceAll("[（|//(]来源[：|:|//s+]([^）|^//)]*)[）|//)]", ""); content = content.replace("**社/**", "").replace("新华社/法新", "").replace(">>详细", ""); content = content.replaceAll("***记者//s*.{2,5}//s*摄", ""); content = content.replace("　 ", "　"); content = content.replaceAll(" ", " ").trim(); System.out.println(title); System.out.println(content); Article article = new Article(); article.setTitle(title); if(pictureUrlList.size() > 0) { article.setPictureUrl(pictureUrlList.get(0)); } else article.setPictureUrl(""); article.setContent(content); article.setArticleUrlOfResource(articleUrl); return article; } }

其它的采集天气预报和WAP新闻的类似再次就不详细贴出来了，关键是要找准几个分割点，利用正则表达式过滤掉不需要的内容。

七、StaticIndex 静态化主页

import java.io.BufferedReader; import java.io.File; import java.io.InputStreamReader; import java.util.TimerTask; import cn.libodls.common.MyTime; import cn.libodls.files.FileIO; import cn.libodls.string.StringCommon; public class StaticIndex extends TimerTask { private File rootPath; private String indexUrl; private FileIO fileIO; private String code; /** * 空构造函数 */ public StaticIndex() { } public StaticIndex(String content, String fileName,String rootPath, String indexUrl, String code) { this.rootPath = new File(rootPath); if (!this.rootPath.exists()) { this.rootPath.mkdirs(); } this.indexUrl = indexUrl; this.fileIO = new FileIO(rootPath); this.code = code; createStaticPage(content, fileName); } public String buildStaticCode() { StringBuffer sb = new StringBuffer(); boolean success = false; int i = 1; while(success == false && i <= 100) { try { java.net.URL url = new java.net.URL(indexUrl); BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream(), code)); String line; while((line = in.readLine()) != null) { sb.append(line + "/n"); } in.close(); success = true; }catch(Exception e) { System.out.println("静态化主页，找不到指定的URL，可能是网络链接失败，重试第" + i + "次！"); i++; success = false; e.printStackTrace(); } } if(success == false) { System.out.println("静态化主页，找不到指定的URL，已经重试了100次，可能是网络链接失败，撤销本次静态化任务！"); return null; } //System.out.println(pageContent); return pageContent; } public void createStaticPage(String pageContent, String fileName) { if(StringCommon.isNullOrEmpty(pageContent)) pageContent = buildStaticCode(); if(pageContent != null) fileIO.write(pageContent, fileName, code); else System.out.println("静态化主页 失败！"); } @Override public void run() { System.out.println("/n静态化主页！"); String fileName = "default.aspx"; //静态化之后页面的名字 String rootPath = "D://WWWROOT"; //静态化后页面存放的目录 String indexUrl = "www.baidu.com"; //要静态化的页面地址 String code = "GBK"; new StaticIndex(null, fileName, rootPath, indexUrl, code); //将执行时间写入日志（供报警使用） FileIO fileIO = new FileIO("D://WWWROOT//log"); fileIO.write(MyTime.getTimeStamp("yyyy-MM-dd HH:mm:ss"), "LogForStaticIndexAlarm", "utf-8"); System.out.println("静态化主页 完成"); } }

注意：上述程序仅作本人研究和分享之用，由此带来的一切版权问题本人概不负责。