爬虫客户端类

package fdl;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.util.EntityUtils;

public class HttpClientTest {
public static void main(String[] args) throws Exception {
// String url =
// "http://www.google.com.hk/search?hl=zh-CN&source=hp&q=nihao+&aq=f&aqi=&aql=&oq=&gs_rfai=";
String url = "http://www.baidu.com/";
String txt = simpleHttpClient(url);
System.out.println(txt);

}

public HttpResponse init(String url) throws Exception {
// 初始化，此处构造函数就与3.1中不同
HttpClient httpclient = new DefaultHttpClient();

// HttpHost targetHost = new HttpHost("3g.youku.com");
HttpGet httpget = new HttpGet(url);
// HttpGet httpget = new HttpGet("/");

// 查看默认request头部信息
System.out.println("Accept-Charset:"
+ httpget.getFirstHeader("Accept-Charset"));
// 以下这条如果不加会发现无论你设置Accept-Charset为gbk还是utf-8，他都会默认返回gb2312（本例针对google.cn来说）
httpget.setHeader("User-Agent",
// "Mozilla/5.0 (SymbianOS/9.1; U; en-us) AppleWebKit/413 (KHTML, like Gecko) Safari/413");
RquestHeader.Oper.valueOf());
// "Opera/9.80 (Windows NT 5.1; U; zh-cn) Presto/2.6.30 Version/10.70");
// 用逗号分隔显示可以同时接受多种编码
httpget.setHeader("Accept-Language", "zh-cn,zh;q=0.5");
httpget.setHeader("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7");
httpget
.setHeader("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// 验证头部信息设置生效
System.out.println("Accept-Charset:"
+ httpget.getFirstHeader("Accept-Charset").getValue());

// Execute HTTP request
System.out.println("executing request " + httpget.getURI());
// HttpResponse response = httpclient.execute(targetHost, httpget);
HttpResponse response = httpclient.execute(httpget);

System.out.println("----------------------------------------");
System.out.println("Location: " + response.getLastHeader("Location"));
System.out.println(response.getStatusLine().getStatusCode());
System.out.println(response.getLastHeader("Content-Type"));
System.out.println(response.getLastHeader("Content-Length"));
System.out.println("----------------------------------------");

// 判断页面返回状态判断是否进行转向抓取新链接
int statusCode = response.getStatusLine().getStatusCode();
System.out.println("statusCode: " + statusCode);
if ((statusCode == HttpStatus.SC_MOVED_PERMANENTLY)
|| (statusCode == HttpStatus.SC_MOVED_TEMPORARILY)
|| (statusCode == HttpStatus.SC_SEE_OTHER)
|| (statusCode == HttpStatus.SC_TEMPORARY_REDIRECT)) {
// 此处重定向处理 此处还未验证
String newUri = response.getLastHeader("Location").getValue();
System.out.println("newUri: ".concat(newUri));
httpclient = new DefaultHttpClient();
httpget = new HttpGet(newUri);
response = httpclient.execute(httpget);
}

return response;
}

public static String baseHttp(HttpResponse response, String url)
throws Exception {
long start = System.currentTimeMillis();
HttpClient httpclient = new DefaultHttpClient();

// Get hold of the response entity
HttpEntity entity = response.getEntity();

// 查看所有返回头部信息
Header headers[] = response.getAllHeaders();
int ii = 0;
while (ii < headers.length) {
System.out.println(headers[ii].getName() + ": "
+ headers[ii].getValue());
++ii;
}
System.out.println("----------------------------------------");

// If the response does not enclose an entity, there is no need
// to bother about connection release
if (entity != null) {
// 将源码流保存在一个byte数组当中，因为可能需要两次用到该流，
byte[] bytes = EntityUtils.toByteArray(entity);
String charSet = "";

// 如果头部Content-Type中包含了编码信息，那么我们可以直接在此处获取
charSet = EntityUtils.getContentCharSet(entity);

System.out.println("In header: " + charSet);
// 如果头部中没有，那么我们需要 查看页面源码，这个方法虽然不能说完全正确，因为有些粗糙的网页编码者没有在页面中写头部编码信息
if (charSet == "" || charSet == null) {
String regEx_html = "(?=<meta).+?(?<=charset=['/"]?)([//w-]+)(?=['/"//s+])";
String regEx_xml = "(?=<//?xml).+?(?<=encoding=['/"]?)([//w-]+)(?=['/"//s+])";
Pattern p_html = Pattern.compile(regEx_html,
Pattern.CASE_INSENSITIVE);
Pattern p_xml = Pattern.compile(regEx_xml,
Pattern.CASE_INSENSITIVE);
Matcher m_html = p_html.matcher(new String(bytes)); // 默认编码转成字符串，因为我们的匹配中无中文，所以串中可能的乱码对我们没有影响
if (m_html.find())
charSet = m_html.group(1);
else {
Matcher m_xml = p_xml.matcher(new String(bytes)); // 默认编码转成字符串，因为我们的匹配中无中文，所以串中可能的乱码对我们没有影响
if (m_xml.find())
charSet = m_xml.group(1);
else
charSet = "GBK";
}
}
if (charSet.toUpperCase().startsWith("GB")) // 处理汉字编码集过小情况
charSet = "GBK";
System.out.println("Last get: " + charSet);
// 至此，我们可以将原byte数组按照正常编码专成字符串输出（如果找到了编码的话）

String txt = new String(bytes, charSet);
// System.out.println("Encoding string is:/n" + txt);
long end = System.currentTimeMillis();
System.out.println("Cost time is: " + (end - start) / 1000.00
+ " s.");
return txt;
}

httpclient.getConnectionManager().shutdown();
httpclient.getConnectionManager();

return "";
}

public static String simpleHttpClient(String url) throws Exception {
long start = System.currentTimeMillis();
// 初始化，此处构造函数就与3.1中不同
HttpClient httpclient = new DefaultHttpClient();

// 这里的http.socket.timeout相当于SO_TIMEOUT
// httpclient.getParams().setIntParameter("http.socket.timeout", 1);
HttpParams params = httpclient.getParams();
HttpConnectionParams.setConnectionTimeout(params, 5000);
HttpConnectionParams.setSoTimeout(params, 10000);

// HttpHost targetHost = new HttpHost("3g.youku.com");
HttpGet httpget = new HttpGet(url);
// HttpGet httpget = new HttpGet("/");

// 查看默认request头部信息
System.out.println("Accept-Charset:"
+ httpget.getFirstHeader("Accept-Charset"));
// 以下这条如果不加会发现无论你设置Accept-Charset为gbk还是utf-8，他都会默认返回gb2312（本例针对google.cn来说）
httpget.setHeader("User-Agent",
// "Mozilla/5.0 (SymbianOS/9.1; U; en-us) AppleWebKit/413 (KHTML, like Gecko) Safari/413");
RquestHeader.Oper.valueOf());
// "Opera/9.80 (Windows NT 5.1; U; zh-cn) Presto/2.6.30 Version/10.70");
// 用逗号分隔显示可以同时接受多种编码
httpget.setHeader("Accept-Language", "zh-cn,zh;q=0.5");
httpget.setHeader("Accept-Charset", "GB2312,utf-8;q=0.7,*;q=0.7");
httpget
.setHeader("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// 验证头部信息设置生效
System.out.println("Accept-Charset:"
+ httpget.getFirstHeader("Accept-Charset").getValue());

// Execute HTTP request
System.out.println("executing request " + httpget.getURI());
// HttpResponse response = httpclient.execute(targetHost, httpget);
HttpResponse response = httpclient.execute(httpget);

response.setEntity(null);

System.out.println("----------------------------------------");
System.out.println("Location: " + response.getLastHeader("Location"));
System.out.println(response.getStatusLine().getStatusCode());
System.out.println(response.getLastHeader("Content-Type"));
System.out.println(response.getLastHeader("Content-Length"));
System.out.println("----------------------------------------");

// 判断页面返回状态判断是否进行转向抓取新链接
int statusCode = response.getStatusLine().getStatusCode();
System.out.println("statusCode: " + statusCode);
if ((statusCode == HttpStatus.SC_MOVED_PERMANENTLY)
|| (statusCode == HttpStatus.SC_MOVED_TEMPORARILY)
|| (statusCode == HttpStatus.SC_SEE_OTHER)
|| (statusCode == HttpStatus.SC_TEMPORARY_REDIRECT)) {
// 此处重定向处理 此处还未验证
String newUri = response.getLastHeader("Location").getValue();
System.out.println("newUri: ".concat(newUri));
httpclient = new DefaultHttpClient();
httpget = new HttpGet(newUri);
response = httpclient.execute(httpget);
}

// Get hold of the response entity
HttpEntity entity = response.getEntity();

// 查看所有返回头部信息
Header headers[] = response.getAllHeaders();
int ii = 0;
while (ii < headers.length) {
System.out.println(headers[ii].getName() + ": "
+ headers[ii].getValue());
++ii;
}
System.out.println("----------------------------------------");

// If the response does not enclose an entity, there is no need
// to bother about connection release
if (entity != null) {
// 将源码流保存在一个byte数组当中，因为可能需要两次用到该流，
byte[] bytes = EntityUtils.toByteArray(entity);
String charSet = "";

// 如果头部Content-Type中包含了编码信息，那么我们可以直接在此处获取
charSet = EntityUtils.getContentCharSet(entity);

System.out.println("In header: " + charSet);
// 如果头部中没有，那么我们需要 查看页面源码，这个方法虽然不能说完全正确，因为有些粗糙的网页编码者没有在页面中写头部编码信息
if (charSet == "" || charSet == null) {
String regEx_html = "(?=<meta).+?(?<=charset=['/"]?)([//w-]+)(?=['/"//s+])";
String regEx_xml = "(?=<//?xml).+?(?<=encoding=['/"]?)([//w-]+)(?=['/"//s+])";
Pattern p_html = Pattern.compile(regEx_html,
Pattern.CASE_INSENSITIVE);
Pattern p_xml = Pattern.compile(regEx_xml,
Pattern.CASE_INSENSITIVE);
Matcher m_html = p_html.matcher(new String(bytes)); // 默认编码转成字符串，因为我们的匹配中无中文，所以串中可能的乱码对我们没有影响
if (m_html.find())
charSet = m_html.group(1);
else {
Matcher m_xml = p_xml.matcher(new String(bytes)); // 默认编码转成字符串，因为我们的匹配中无中文，所以串中可能的乱码对我们没有影响
if (m_xml.find())
charSet = m_xml.group(1);
else
charSet = "GBK";
}
}
if (charSet.toUpperCase().startsWith("GB")) // 处理汉字编码集过小情况
charSet = "GBK";
System.out.println("Last get: " + charSet);
// 至此，我们可以将原byte数组按照正常编码专成字符串输出（如果找到了编码的话）

String txt = new String(bytes, charSet);
// System.out.println("Encoding string is:/n" + txt);
long end = System.currentTimeMillis();
System.out.println("Cost time is: " + (end - start) / 1000.00
+ " s.");
return txt;
}

// httpclient.getConnectionManager().shutdown();

return "";
}

public static void test1(String url) throws Exception {
// (?<=<a).+?href=["']?(.+?)(?=["' >])

Pattern p = Pattern.compile("<a//s+?href=[/"']?(.*?)[/"'//s >]",
Pattern.CASE_INSENSITIVE);

String txt = simpleHttpClient(url);
long b = System.nanoTime();

Matcher m = p.matcher(txt);
Pattern pt = Pattern.compile("^((javascript|mailto):.*)|([#/])$",
Pattern.CASE_INSENSITIVE);
while (m.find()) {
// System.out.println(m.group(1));
String rs = m.group(1);

Matcher mt = pt.matcher(rs);
if (!mt.matches())
System.out.println("--/t"
.concat(isRelativeAddressToFullUrlAddressNew(rs, url)));

}
long e = System.nanoTime() - b;
System.out.println("Cost time is: " + (e / 1000000000.00) + " s.");

}

/**
*
* 处理带"#"情况的URL
*
* @param link
* 需要转换的链接
* @param base
* 页面的url
* @return 绝对 URL.
*/
public static String isRelativeAddressToFullUrlAddressNew(String link,
String base) {

String dealLinkStr = link.trim();
URL url = null;
try {
if (dealLinkStr.startsWith("#"))
url = new URL(base + dealLinkStr);
else
url = isRelativeAddressToFullUrlAddress(link, base);
return url.toString();
} catch (MalformedURLException e) {
// e.printStackTrace();
// logger.debug("连接不合法..... <url> " + link);
return link;
}
}

/**
* Build a URL from the link and base provided.
*
* @param link
* 需要转换的链接
* @param base
* 页面的url
* @return 绝对 URL.
* @exception MalformedURLException
* If creating the URL fails.
*/
public static URL isRelativeAddressToFullUrlAddress(String link, String base)
throws MalformedURLException {
String path;
boolean modified;
boolean absolute;
boolean strict = haveQM(base);
int index;
URL url; // constructed URL combining relative link and base

// Bug #1461473 Relative links starting with ?
if (!strict && ('?' == link.charAt(0))) { // remove query part of base
// if any
if (-1 != (index = base.lastIndexOf('?')))
base = base.substring(0, index);
url = new URL(base + link);
} else {
url = new URL(new URL(base), link);
}
path = url.getFile();
modified = false;
absolute = link.startsWith("/");
if (!absolute) { // we prefer to fix incorrect relative links
// this doesn't fix them all, just the ones at the start
while (path.startsWith("/.")) {
if (path.startsWith("/../")) {
path = path.substring(3);
modified = true;
} else if (path.startsWith("/./") || path.startsWith("/.")) {
path = path.substring(2);
modified = true;
} else
break;
}
}
// fix backslashes
while (-1 != (index = path.indexOf("///"))) {
path = path.substring(0, index + 1) + path.substring(index + 2);
modified = true;
}
if (modified)
url = new URL(url, path);

return (url);
}

private static boolean haveQM(String urlStr) {
if (urlStr.contains("?")) {
return true;
} else {
return false;
}
}
}

/**
*
*各种请求头类型
*
* @author Jerome
*
*/
enum RquestHeader {
Oper {
@Override
public String valueOf() {
return "Opera/9.80 (Windows NT 5.1; U; zh-cn) Presto/2.6.30 Version/10.70";
}
},

IPhone {

@Override
public String valueOf() {
return "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16";
}

},

IE {

@Override
public String valueOf() {
return "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1)";
}

};
// 获取请求头内容
public abstract String valueOf();
}