Java 爬虫 +Mysql

最新推荐文章于 2021-01-27 01:51:57 发布

原创最新推荐文章于 2021-01-27 01:51:57 发布 · 267 阅读

0 ·

CC 4.0 BY-SA版权

本文介绍了一个使用Java编写的简单爬虫程序，该程序能够从豆瓣网站抓取日剧推荐列表，并将数据解析为JSON格式。示例中包含了HTTP请求、JSON解析及MySQL数据库操作等关键步骤。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

配置文件：


    <dependency>
      <groupId>net.sf.json-lib</groupId>
      <artifactId>json-lib</artifactId>
      <version>2.4</version>
    </dependency>
    <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
    <dependency>
      <groupId>com.google.code.gson</groupId>
      <artifactId>gson</artifactId>
      <version>2.8.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.httpcomponents</groupId>
      <artifactId>httpclient</artifactId>
      <version>4.5.2</version>
    </dependency>
    <dependency>
      <groupId>mysql</groupId>
      <artifactId>mysql-connector-java</artifactId>
      <version>5.1.44</version>
    </dependency>

代码：

package com.spider;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.mysql.jdbc.Connection;
import com.mysql.jdbc.Statement;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import java.io.IOException;
import java.sql.*;
/**
 * @Author zhaoxin
 * @Email 1272743926@qq.com
 * @Description //TODO
 * @Date 2018/11/29
 **/
public class DB {
    public static void main(String[] args) throws Exception {
        connect();
        String url="https://movie.douban.com/j/search_subjects?type=tv&tag=%E6%97%A5%E5%89%A7&sort=recommend&page_limit=20&page_start=0";
        CloseableHttpResponse indexRes = sendGet(url);
//        获取json内容，将其转换为字符串
        String indexHtml = EntityUtils.toString(indexRes.getEntity(), "UTF-8");
//        截取成json字符串
        System.out.println(indexHtml);

        JSONArray move = JSON.parseObject(indexHtml).getJSONArray("subjects");
        System.out.println(move.size());
//
        for (Object item:move){
            System.out.println(JSON.parseObject(item.toString()).getString("title"));
            System.out.println(JSON.parseObject(item.toString()).getString("url"));
        }
    }
    //发送get请求,获取响应结果
    public static CloseableHttpResponse sendGet(String url) throws IOException {
        //创建httpClient客户端
        CloseableHttpClient httpClient = HttpClients.createDefault();
        //创建请求对象,发送请求
        HttpGet httpGet = new HttpGet(url);
        httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36");
        CloseableHttpResponse response = httpClient.execute(httpGet);
        return response;
    }
    public static void connect()throws Exception{
        String driver="com.mysql.jdbc.Driver";
        String url="jdbc:mysql://localhost:3306/test.db?useSSL=true";
        String user="root";
        String pwd ="root";
        Connection conn = null;
        Statement stmt = null;
        // 注册 JDBC 驱动
        Class.forName("com.mysql.jdbc.Driver");

        // 打开链接
        System.out.println("连接数据库...");
        conn = (Connection) DriverManager.getConnection(url,user,pwd);
        stmt= (Statement) conn.createStatement();
        System.out.println("连接成");
        String sql = "insert into move values (3,'sss','ssss')";
        stmt.execute(sql);
        conn.close();
    }
}