2014年抓取Google查询结果的方法

最新推荐文章于 2023-01-04 15:02:44 发布

转载最新推荐文章于 2023-01-04 15:02:44 发布 · 1k 阅读

文章标签：

#抓取 #Google #最新 #2014

JavaEE 同时被 3 个专栏收录

25 篇文章

订阅专栏

信息检索

18 篇文章

订阅专栏

Java通用范例开发金典

14 篇文章

订阅专栏

本文介绍了一种使用Java编程语言从Google搜索结果中抓取网页内容的方法，包括设置URL连接、请求头以及处理输入流等步骤。

import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.net.URLConnection;
import java.util.Scanner;
public class Exec {
public String fetchPageContent(String url, String userAgent)
throws IOException {
String pageContent = "";

// set URL connection
URL urlFetcher = new URL(url);
URLConnection connection = urlFetcher.openConnection();
// close the connection in 10 seconds
connection.setConnectTimeout(10000);
// set the User-Agent
connection.setRequestProperty("User-Agent", userAgent);
if (connection != null) {
InputStream inputStream = connection.getInputStream();
if (inputStream != null) {
Scanner scanner = new Scanner(inputStream);
if (scanner != null) {
scanner.useDelimiter("\\Z");
pageContent = scanner.next();
inputStream.close();
}
}
}
return pageContent;
}
public String searchGoogle(String queryStr) {
String searchResult = "";
try {
queryStr = queryStr.replaceAll(" ", "+");
String url = "http://www.google.com/search?hl=en&&as_epq="
+ queryStr;
int random = (int) Math.random() * 100;
String userAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1"
+ random + ")";
searchResult = fetchPageContent(url, userAgent);
} catch (IOException e) {
e.printStackTrace();
}
return searchResult;
}
public static void main(String[] args) {
Exec e = new Exec();
System.out.println(e.searchGoogle("java"));
}
}