简单爬虫

先引用需要的jar包

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.play</groupId>
	<artifactId>game</artifactId>
	<version>0.0.1-SNAPSHOT</version>

	<dependencies>
		<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
		<dependency>
			<groupId>org.jsoup</groupId>
			<artifactId>jsoup</artifactId>
			<version>1.11.2</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient -->
		<dependency>
			<groupId>org.apache.httpcomponents</groupId>
			<artifactId>httpclient</artifactId>
			<version>4.5.10</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
		<!-- <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> 
			<version>1.7.25</version> <scope>test</scope> </dependency> -->
		<dependency>
			<groupId>com.google.code.gson</groupId>
			<artifactId>gson</artifactId>
			<version>2.8.1</version>
		</dependency>

		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-context</artifactId>
			<version>5.0.6.RELEASE</version>
		</dependency>
		<dependency>
			<groupId>junit</groupId>
			<artifactId>junit</artifactId>
			<version>4.12</version>
		</dependency>
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<version>5.1.47</version>
		</dependency>
		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-jdbc</artifactId>
			<version>5.1.2.RELEASE</version>
		</dependency>
		<dependency>
			<groupId>org.springframework</groupId>
			<artifactId>spring-tx</artifactId>
			<version>5.1.2.RELEASE</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/c3p0/c3p0 -->
		<dependency>
			<groupId>c3p0</groupId>
			<artifactId>c3p0</artifactId>
			<version>0.9.1.2</version>
		</dependency>
		<dependency>
			<groupId>commons-logging</groupId>
			<artifactId>commons-logging</artifactId>
			<version>1.2</version>
		</dependency>
	</dependencies>

</project>

// 如果是post请求，且需要加参数就这样

String url = "url";
	List<NameValuePair> list = new LinkedList<>();
	BasicNameValuePair param1 = new BasicNameValuePair("pageSize", "10");
	BasicNameValuePair param2 = new BasicNameValuePair("pageNum", "1");
	BasicNameValuePair param3 = new BasicNameValuePair("newsCategory", "196");
	list.add(param1);
	list.add(param2);
	list.add(param3);
	UrlEncodedFormEntity entity1 = null;
	entity1 = new UrlEncodedFormEntity(list, "UTF-8");
	HttpPost httpPost = new HttpPost(url);
	httpPost.setEntity(entity1);
	CloseableHttpClient createDefault = HttpClients.createDefault();
	CloseableHttpResponse execute = createDefault.execute(httpPost);

// 这个是get请求

String url = "url";
HttpGet httpget = new HttpGet(url);
CloseableHttpClient createDefault = HttpClients.createDefault();
CloseableHttpResponse execute = createDefault.execute(httpget);
if (200 == execute.getStatusLine().getStatusCode()) {
String string = EntityUtils.toString(execute.getEntity(), Charset.forName("utf-8"));
// 这个是直接转成string查询
Document parse = Jsoup.parse(string);
String productName = parse.select("[class=parameter2 p-parameter-list] li").get(0).text();
// 也可以转成map取值
Gson gson = new Gson();
	ArrayList<Map> fromJson = gson.fromJson(substring, ArrayList.class);
	for (Object map : fromJson) {
		Map<String, Double> map1 = (Map) map;
		Double double1 = (Double) map1.get("id");
		// 此处写逻辑代码
	}