java jsoup 网络爬虫 学习例子（六）京东和当当商品比价
package com.iteye.injavawetrust.jdvsdd;
import java.util.List;
/**
*
* @author InJavaWeTrust
*
*/
public interface ProductList {
/**
* 爬取商品列表
* @return
*/
public List<ProductInfo> getProductList();
}
package com.iteye.injavawetrust.jdvsdd;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
*
* @author InJavaWeTrust
*
*/
public class JDProductList implements ProductList{
private String jdUrl;
private String productName;
private static PriceCheckUtil pcu = PriceCheckUtil.getInstance();
public JDProductList(String jdUrl, String productName){
this.jdUrl = jdUrl;
this.productName = productName;
}
@Override
public List<ProductInfo> getProductList() {
List<ProductInfo> jdProductList = new ArrayList<ProductInfo>();
ProductInfo productInfo = null;
String url = "";
for(int i = 0; i < 10; i++){
try {
System.out.println("JD Product 第[" + (i + 1) + "]页");
if(i == 0) {
url = jdUrl;
}else{
url = Constants.JDURL + pcu.getGbk(productName) + Constants.JDENC + Constants.JDPAGE + (i + 1);
}
System.out.println(url);
Document document = Jsoup.connect(url).timeout(5000).get();
Elements uls = document.select("ul[class=gl-warp clearfix]");
Iterator<Element> ulIter = uls.iterator();
while(ulIter.hasNext()) {
Element ul = ulIter.next();
Elements lis = ul.select("li[data-sku]");
Iterator<Element> liIter = lis.iterator();
while(liIter.hasNext()) {
Element li = liIter.next();
Element div = li.select("div[class=gl-i-wrap]").first();
Elements title = div.select("div[class=p-name p-name-type-2]>a");
String productName = title.attr("title"); //得到商品名称
Elements price = div.select(".p-price>strong");
String productPrice =price.attr("data-price"); //得到商品价格
productInfo = new ProductInfo();
productInfo.setProductName(productName);
productInfo.setProductPrice(productPrice);
jdProductList.add(productInfo);
}
}
} catch(Exception e) {
System.out.println("Get JD product has error [" + url + "]");
System.out.println(e.getMessage());
}
}
return jdProductList;
}
public static void main(String[] args) {
try {
String productName = "书包";
String jdUrl = Constants.JDURL + pcu.getGbk(productName) + Constants.JDENC;
List<ProductInfo> list = new JDProductList(jdUrl, productName).getProductList();
System.out.println(list.size());
for(ProductInfo pi : list){
System.out.println(pi.getProductName() + " " + pi.getProductPrice());
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
package com.iteye.injavawetrust.jdvsdd;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
*
* @author InJavaWeTrust
*
*/
public class DDProductList implements ProductList{
private String ddUrl;
private String productName;
public DDProductList(String ddUrl, String productName) {
this.ddUrl = ddUrl;
this.productName = productName;
}
@Override
public List<ProductInfo> getProductList() {
List<ProductInfo> ddProductList = new ArrayList<ProductInfo>();
ProductInfo productInfo = null;
String url = "";
for(int i = 0; i < 10; i++){
try{
System.out.println("DD Product 第[" + (i + 1) + "]页");
if(i == 0){
url = ddUrl;
}else{
url = Constants.DDURL + productName + Constants.ACT + Constants.DDPAGE + (i + 1);
}
System.out.println(url);
Document document = Jsoup.connect(url).timeout(5000).get();
Elements uls = document.select("ul[class=bigimg cloth_shoplist]");
Iterator<Element> ulIter = uls.iterator();
while(ulIter.hasNext()){
Element ul = ulIter.next();
Elements lis = ul.select("li");
Iterator<Element> liIter = lis.iterator();
while(liIter.hasNext()){
Element li = liIter.next();
Elements price = li.select("p[class=price]>span");
String productPrice = price.html().replaceAll("¥", "");
Elements title = li.select("p[class=name]>a");
String productName = title.attr("title");
productInfo = new ProductInfo();
productInfo.setProductName(productName);
productInfo.setProductPrice(productPrice);
ddProductList.add(productInfo);
}
}
} catch(Exception e){
System.out.println("Get DD product has error [" + url + "]");
System.out.println(e.getMessage());
}
}
return ddProductList;
}
public static void main(String[] args) {
String productName = "学生铅笔";
String ddUrl = Constants.DDURL + productName;
List<ProductInfo> list = new DDProductList(ddUrl, productName).getProductList();
System.out.println(list.size());
for(ProductInfo pi : list){
System.out.println(pi.getProductName() + " " + pi.getProductPrice());
}
}
}
package com.iteye.injavawetrust.jdvsdd;
import java.io.Serializable;
import java.util.Date;
/**
*
* @author InJavaWeTrust
*
*/
public class ProductInfo implements Serializable{
private static final long serialVersionUID = 8179244535272774089L;
/**
* 商品ID
*/
private String productid;
/**
* 商品名称
*/
private String productName;
/**
* 商品价格
*/
private String productPrice;
/**
* 月销售笔数
*/
private String tradeNum;
/**
* 商品URL
*/
private String productUrl;
/**
* 商品网店名称
*/
private String shopName;
/**
* 电商名称
*/
private String ecName;
/**
* 爬取入库日期
*/
private Date date;
public String getProductid() {
return productid;
}
public void setProductid(String productid) {
this.productid = productid;
}
public String getProductName() {
return productName;
}
public void setProductName(String productName) {
this.productName = productName;
}
public String getProductPrice() {
return productPrice;
}
public void setProductPrice(String productPrice) {
this.productPrice = productPrice;
}
public String getTradeNum() {
return tradeNum;
}
public void setTradeNum(String tradeNum) {
this.tradeNum = tradeNum;
}
public String getProductUrl() {
return productUrl;
}
public void setProductUrl(String productUrl) {
this.productUrl = productUrl;
}
public String getShopName() {
return shopName;
}
public void setShopName(String shopName) {
this.shopName = shopName;
}
public String getEcName() {
return ecName;
}
public void setEcName(String ecName) {
this.ecName = ecName;
}
public Date getDate() {
return date;
}
public void setDate(Date date) {
this.date = date;
}
}
package com.iteye.injavawetrust.jdvsdd;
/**
*
* @author InJavaWeTrust
*
*/
public class Constants {
/**
* JDURL
*/
public static String JDURL = "http://search.jd.com/Search?keyword=";
/**
* JD汉字编码格式
*/
public static String JDENC = "&enc=utf-8";
/**
* JD分页
*/
public static String JDPAGE ="&page=";
/**
* 当当URL
*/
public static String DDURL = "http://search.dangdang.com/?key=";
/**
* 当当ACT
*/
public static String ACT = "&act=input";
/**
* 当当分页
*/
public static String DDPAGE = "&page_index=";
}
package com.iteye.injavawetrust.jdvsdd;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.TimeZone;
/**
*
* @author InJavaWeTrust
*
*/
public class PriceCheckUtil {
private PriceCheckUtil() {
}
private static final PriceCheckUtil instance = new PriceCheckUtil();
public static PriceCheckUtil getInstance() {
return instance;
}
/**
* 商品汉字转码
* @param productName 商品名称
* @return
*/
public String getGbk(String productName){
String retGbk = "";
try {
retGbk = new String(productName.getBytes("UTF-8"), "GBK");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
return retGbk;
}
/**
* 从列表list中找到与productName相似度最高的ProductInfo
*
* @param productName
* @param list
* @return 相似度最高的productName
*/
public ProductInfo getSimilarity(String productName, List<ProductInfo> list) {
ProductInfo productInfo = null;
/**
* 找到list中所有的productName与字符串productName的相似度，保存在lens数组中
*/
double lens[] = new double[list.size()];
for (int i = 0; i < list.size() - 1; i++) {
lens[i] = sim(productName, list.get(i).getProductName());
}
/**
* 遍历出最大的相似度maxLen
*/
double maxLen = 0.0;
for (int i = 0; i < lens.length; i++) {
if (maxLen < lens[i]) {
maxLen = lens[i];
}
}
/**
* 遍历出最大的相似度的索引maxLenIndex
*/
int maxLenIndex = 0;
for (int i = 0; i < lens.length; i++) {
if (maxLen == lens[i]) {
maxLenIndex = i;
}
}
productInfo = list.get(maxLenIndex);
return productInfo;
}
/**
* 求三个数中最小的一个
* @param one
* @param two
* @param three
* @return
*/
public int min(int one, int two, int three) {
int min = one;
if(two < min) {
min = two;
}
if(three < min) {
min = three;
}
return min;
}
/**
* 计算矢量距离
* Levenshtein Distance(LD)
* @param str1
* @param str2
* @return
*/
public int ld(String str1, String str2) {
int d[][]; //矩阵
int n = str1.length();
int m = str2.length();
int i; //遍历str1的
int j; //遍历str2的
char ch1; //str1的
char ch2; //str2的
int temp; //记录相同字符,在某个矩阵位置值的增量,不是0就是1
if(n == 0) {
return m;
}
if(m == 0) {
return n;
}
d = new int[n+1][m+1];
for(i=0; i<=n; i++) { //初始化第一列
d[i][0] = i;
}
for(j=0; j<=m; j++) { //初始化第一行
d[0][j] = j;
}
for(i=1; i<=n; i++) { //遍历str1
ch1 = str1.charAt(i-1);
//去匹配str2
for(j=1; j<=m; j++) {
ch2 = str2.charAt(j-1);
if(ch1 == ch2) {
temp = 0;
} else {
temp = 1;
}
//左边+1,上边+1, 左上角+temp取最小
d[i][j] = min(d[i-1][j]+1, d[i][j-1]+1, d[i-1][j-1]+temp);
}
}
return d[n][m];
}
/**
* 计算相似度
* @param str1
* @param str2
* @return
*/
public double sim(String str1, String str2) {
int ld = ld(str1, str2);
return 1 - (double) ld / Math.max(str1.length(), str2.length());
}
/**
* 毫秒转换成hhmmss
* @param ms 毫秒
* @return hh:mm:ss
*/
public String msToss(long ms) {
SimpleDateFormat formatter = new SimpleDateFormat("HH:mm:ss");
formatter.setTimeZone(TimeZone.getTimeZone("GMT+00:00"));
String ss = formatter.format(ms);
return ss;
}
}
package com.iteye.injavawetrust.jdvsdd;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
/**
*
* @author InJavaWeTrust
*
*/
public class PriceCheckMain {
private static PriceCheckUtil pcu = PriceCheckUtil.getInstance();
public List<Map<String, ProductInfo>> getProductList(String productName) {
String jdUrl = Constants.JDURL + pcu.getGbk(productName) + Constants.JDENC;
String ddUrl = Constants.DDURL + productName;
return getProductFromUrls(jdUrl, ddUrl, productName);
}
public List<Map<String, ProductInfo>> getProductFromUrls(String jdUrl, String ddUrl, String productName) {
List<Map<String, ProductInfo>> retListMap = new ArrayList<Map<String,ProductInfo>>();
List<ProductInfo> jdProductList = new JDProductList(jdUrl, productName).getProductList();
List<ProductInfo> ddProductList = new DDProductList(ddUrl, productName).getProductList();
for(int i = 0; i < jdProductList.size(); i++){
String jdProductName = jdProductList.get(i).getProductName();
Map<String, ProductInfo> map = new HashMap<String, ProductInfo>();
map.put("JD", jdProductList.get(i));
ProductInfo ddProduct = pcu.getSimilarity(jdProductName, ddProductList);
map.put("DD", ddProduct);
retListMap.add(map);
}
return retListMap;
}
public static void main(String[] args) {
System.out.println("输入商品名称：");
Scanner scanner = new Scanner(System.in);
String productName = scanner.next();
scanner.close();
System.out.println("京东和当当[" + productName + "]商品比价开始。。。。。。");
try{
long starTime = System.currentTimeMillis();
List<Map<String, ProductInfo>> list = new PriceCheckMain().getProductList(productName);
for(Map<String, ProductInfo> map : list) {
String jdName = map.get("JD").getProductName();
String jdPrice = map.get("JD").getProductPrice();
String ddName = map.get("DD").getProductName();
String ddPrice = map.get("DD").getProductPrice();
System.out.println("[" + jdName + "] [" + ddName + "]");
System.out.println("[" + jdPrice + "] [" + ddPrice + "]");
System.out.println("-----------------------------------------------------------");
}
long endTime = System.currentTimeMillis();
System.out.println("用时 [" + pcu.msToss(endTime - starTime) + "]");
}catch(Exception e){
System.out.println("error");
System.out.println(e.getMessage());
}
}
}
运行结果：
输入商品名称：
铅笔
京东和当当[铅笔]商品比价开始。。。。。。
JD Product 第[1]页
http://search.jd.com/Search?keyword=閾呯瑪&enc=utf-8
JD Product 第[2]页
http://search.jd.com/Search?keyword=閾呯瑪&enc=utf-8&page=2
..............................
DD Product 第[1]页
http://search.dangdang.com/?key=铅笔
DD Product 第[2]页
http://search.dangdang.com/?key=铅笔&act=input&page_index=2
DD Product 第[3]页
...................................
[得力（deli） S908 木世界系列六角笔杆原木HB铅笔/素描绘图学生铅笔 50支/桶] [ 【开学必备文具】正品 得力文具S907/s908原木HB/2B铅笔绘图素描儿童学生铅笔50支装 ]
[18.00] [16.80]
-----------------------------------------------------------
[得力（deli） 7084 安全考试专用填涂答题卡2B木质铅笔/学生铅笔 12支/盒] [ 得力文具（deli） 0641 削笔器 削笔刀 卷笔刀 削笔机学习用品手摇转笔刀笔刨文具 ]
[10.00] [13.50]
-----------------------------------------------------------
[得力(deli)7083 安全石墨铅芯素描 绘图HB铅笔/学生铅笔 12支/盒] [ 六一儿童节礼物！三菱 绘图HB/2B铅笔12支装 ]
[9.00] [35.00]
-----------------------------------------------------------
[辉柏嘉（Faber-castell）114468 水溶性彩色铅笔 水溶彩铅 48色套装（赠毛笔+笔刨）] [ 德国Faber-castell辉柏嘉三角杆学生铅笔 儿童铅笔 HB 2H 2B 12支 ]
[109.00] [12.00]
................................................
[中华6725桶装彩色铅笔36色24色18色12色原木三角杆彩色铅笔 美术绘画涂鸦涂色彩铅 12色] [ 晨光彩色铅笔36色 24色绘图涂鸦 桶装绘画彩铅彩笔 儿童绘画 ]
[8.80] [12.00]
-----------------------------------------------------------
用时 [00:00:10]
6666
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
