Android 网页抓取（实现新闻客户端）

最新推荐文章于 2022-07-13 11:52:26 发布

000wen_Android

最新推荐文章于 2022-07-13 11:52:26 发布

阅读量1.6k

点赞数

本文介绍如何使用Android客户端抓取并解析网页，实现新闻客户端的功能，包括使用jsoup和htmlparser两种方法进行新闻数据的抓取和解析。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

Android 网页抓取（实现新闻客户端）

文章摘自：http://gqdy365.iteye.com/blog/2104046

如何实现从各大网抓取新闻并经过格式处理现实到我们的新闻客户端呢？
Android客户端抓取并解析网页的方法我用到的有两种：
一、用jsoup
没仔细研究，网上有类似的，可以参考这两位兄弟的：
http://decentway.iteye.com/blog/1333127
http://blog.youkuaiyun.com/hellohaifei/article/details/9352069

二、用htmlparser
我项目中就用htmlparser，抓紧并解析腾讯新闻，代码如下：

Java代码

public class NetUtil {
public static List<NewsBrief> DATALIST = new ArrayList<NewsBrief>();
public static String[][] CHANNEL_URL = new String[][] {
new String[]{"http://news.qq.com/world_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/society_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
new String[]{"http://news.qq.com/china_index.shtml","http://news.qq.com"},
};
public static int getTechNews(List<NewsBrief> techData, int cId) {
int result = 0;
try {
NodeFilter filter = new AndFilter(new TagNameFilter("div"),
new HasAttributeFilter("id", "listZone"));
Parser parser = new Parser();
parser.setURL(CHANNEL_URL[cId][0]);
parser.setEncoding(parser.getEncoding());
NodeList list = parser.extractAllNodesThatMatch(filter);
for (int i = 0; i < list.size(); i++) {
Tag node = (Tag) list.elementAt(i);
for (int j = 0; j < node.getChildren().size(); j++) {
try {
String textstr = node.getChildren().elementAt(j).toHtml();
if (textstr.trim().length() > 0) {
NodeFilter subFilter = new TagNameFilter("p");
Parser subParser = new Parser();
subParser.setResource(textstr);
NodeList subList = subParser.extractAllNodesThatMatch(subFilter);
NodeFilter titleStrFilter = new AndFilter(new TagNameFilter("a"),
new HasAttributeFilter("class", "linkto"));
Parser titleStrParser = new Parser();
titleStrParser.setResource(textstr);
NodeList titleStrList = titleStrParser.extractAllNodesThatMatch(titleStrFilter);
int linkstart = titleStrList.toHtml().indexOf("href=\"");
int linkend = titleStrList.toHtml().indexOf("\">");
int titleend = titleStrList.toHtml().indexOf("</a>");
String link = CHANNEL_URL[cId][1]+titleStrList.toHtml().substring(linkstart+6, linkend);
String title = titleStrList.toHtml().substring(linkend+2, titleend);
NewsBrief newsBrief = new NewsBrief();
newsBrief.setTitle(title);
newsBrief.setUrl(link);
newsBrief.setSummary(subList.asString());
techData.add(newsBrief);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
} catch (Exception e) {
result = 1;
e.printStackTrace();
}
return result;
}
public static int getTechNews2(List<NewsBrief> techData, int cId) {
int result = 0;
try {
// 查询http://tech.qq.com/tech_yejie.htm 页面滚动新闻的标签以及ID
NodeFilter filter = new AndFilter(new TagNameFilter("div"),
new HasAttributeFilter("id", "listZone"));
Parser parser = new Parser();
parser.setURL(CHANNEL_URL[cId][0]);
parser.setEncoding(parser.getEncoding());
// 获取匹配的fileter的节点
NodeList list = parser.extractAllNodesThatMatch(filter);
StringBuilder NewsStr = new StringBuilder("<table>");// 新闻表格字符串
for (int i = 0; i < list.size(); i++) {
Tag node = (Tag) list.elementAt(i);
for (int j = 0; j < node.getChildren().size(); j++) {
String textstr = node.getChildren().elementAt(j).toHtml()
.trim();
if (textstr.length() > 0) {
int linkbegin = 0, linkend = 0, titlebegin = 0, titleend = 0;
while (true) {
linkbegin = textstr.indexOf("href=", titleend);// 截取链接字符串起始位置
// 如果不存在 href了也就结束了
if (linkbegin < 0)
break;
linkend = textstr.indexOf("\">", linkbegin);// 截取链接字符串结束位置
String sublink = textstr.substring(linkbegin + 6,linkend);
String link = CHANNEL_URL[cId][1] + sublink;
titlebegin = textstr.indexOf("\">", linkend);
titleend = textstr.indexOf("</a>", titlebegin);
String title = textstr.substring(titlebegin + 2,titleend);
NewsStr.append("\r\n<tr>\r\n\t<td><a target=\"_blank\" href=\""
+ link + "\">");
NewsStr.append(title);
NewsStr.append("</a></td></tr>");
NewsBrief newsBrief = new NewsBrief();
newsBrief.setTitle(title);
newsBrief.setUrl(link);
techData.add(newsBrief);
}
}
}
}
} catch (Exception e) {
result = 1;
e.printStackTrace();
}
return result;
}
public static int parserURL(String url,NewsBrief newsBrief) {
int result = 0;
try {
Parser parser = new Parser(url);
NodeFilter contentFilter = new AndFilter(
new TagNameFilter("div"),
new HasAttributeFilter("id","Cnt-Main-Article-QQ"));
NodeFilter newsdateFilter = new AndFilter(
new TagNameFilter("span"),
new HasAttributeFilter("class",
"article-time"));
NodeFilter newsauthorFilter = new AndFilter(
new TagNameFilter("span"),
new HasAttributeFilter("class",
"color-a-1"));
NodeFilter imgUrlFilter = new TagNameFilter("IMG");
newsBrief.setContent(parserContent(contentFilter,parser));
parser.reset(); // 记得每次用完parser后，要重置一次parser。要不然就得不到我们想要的内容了。
newsBrief.setPubDate(parserDate(newsdateFilter,parser));
parser.reset();
newsBrief.setSource(parserAuthor(newsauthorFilter, parser));
parser.reset();
newsBrief.setImgUrl(parserImgUrl(contentFilter,imgUrlFilter, parser));
} catch (Exception e) {
result=1;
e.printStackTrace();
}
return result;
}
private static String parserContent(NodeFilter filter, Parser parser) {
String reslut = "";
try {
NodeList contentList = (NodeList) parser.parse(filter);
// 将DIV中的标签都去掉只留正文
reslut = contentList.asString();
} catch (Exception e) {
e.printStackTrace();
}
return reslut;
}
private static String parserDate(NodeFilter filter, Parser parser) {
String reslut = "";
try {
NodeList datetList = (NodeList) parser.parse(filter);
// 将DIV中的标签都去掉只留正文
reslut = datetList.asString();
} catch (Exception e) {
e.printStackTrace();
}
return reslut;
}
private static String parserAuthor(NodeFilter filter, Parser parser) {
String reslut = "";
try {
NodeList authorList = (NodeList) parser.parse(filter);
// 将DIV中的标签都去掉只留正文
reslut = authorList.asString();
} catch (Exception e) {
e.printStackTrace();
}
return reslut;
}
private static List<String> parserImgUrl(NodeFilter bodyfilter,NodeFilter filter, Parser parser) {
List<String> reslut = new ArrayList<String>();
try {
NodeList bodyList = (NodeList) parser.parse(bodyfilter);
Parser imgParser = new Parser();
imgParser.setResource(bodyList.toHtml());
NodeList imgList = imgParser.extractAllNodesThatMatch(filter);
String bodyString = imgList.toHtml();
//正文包含图片
if (bodyString.contains("<IMG")
&& bodyString.contains("src=")) {
if(imgList.size()>0){
for(int i=0;i<imgList.size();i++){
String imgString = imgList.elementAt(i).toHtml();
int imglinkstart = imgString.indexOf("src=\"");
int imglinkend = imgString.indexOf(">");
if(imgString.contains("\" alt=")){
imglinkend = imgString.indexOf("\" alt=");
}
if(imgString.contains("_fck")){
imglinkend = imgString.indexOf("_fck");// 截取链接字符串结束位置
}
reslut.add(imgString.substring(imglinkstart + 5, imglinkend));
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return reslut;
}
}