网页数据抓取与存储-优快云博客

本文链接：https://blog.youkuaiyun.com/robinliu2010/article/details/7551985

本文介绍了使用Java和Jsoup库进行网页数据抓取的方法，包括解析URL获取链接及文本内容，并将抓取的数据存储到MySQL数据库中。此外，还展示了如何通过选择器定位特定元素。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

程序一：

import java.io.File;
import java.io.IOException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class crawltest1 {
private crawldata op;
public crawltest1(){
  this.op = new crawldata();
}
public static void main(String[] args) {
  crawltest1 t = new crawltest1();
  t.parseUrl();
}

public void parseString() {
  String url = "http://……";//文章来源网站名称。
  Document doc = Jsoup.parse(url);
  System.out.println(doc);
//  Elements es = doc.body().getAllElements();
//  System.out.println(es.attr("onload"));
//  System.out.println(es.select("p"));
}

public void parseUrl() {
  try {
//   Document doc = Jsoup.connect("http://……/").get();
   String doc_web="http://www.…….cn";
   Document doc = Jsoup.connect("http://www.…….cn/download/do_list.jsp?TYPE=1").get();
//   String title = doc.title();
   Elements hrefs = doc.select("a[href~=www]");
   //   for(Element href : hrefs){
//    op.saveUrl(href.toString());
//   }
//   String doc_link = hrefs.attr("href");
//   String doc_name = hrefs.text();

   for (Element href : hrefs){
   String doc_link = doc_web+href.attr("href");
   String doc_name = href.text();
   System.out.println(" address="+doc_link+" "+" name="+doc_name);
   op.saveUrl(doc_name,doc_link,doc_web);
   }
//   System.out.println(doc_name);

//   System.out.println("------------------");
//   System.out.println(hrefs.select("[href^=http]"));
//   System.out.println(title);
  } catch (IOException e) {
   e.printStackTrace();
  }
}

public void parseFile() {
  try {
   File input = new File("d:\\abc\\input.html");
   Document doc = Jsoup.parse(input, "UTF-8");
   // 提取出所有的编号
   Elements codes = doc.body().select("td[title^=IA] > a[href^=javascript:view]");
   System.out.println(codes);
   System.out.println("------------------");
   System.out.println(codes.html());
  } catch (IOException e) {
   e.printStackTrace();
  }
}
}

程序二

import java.io.File;
import java.io.IOException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class testpage {
private crawldata op;
public testpage(){
  this.op = new crawldata();
}
public static void main(String[] args) {
  crawltest1 t = new crawltest1();
  t.parseUrl();
}

public void parseUrl() {
  try {
//   Document doc = Jsoup.connect("http://www…….cn/fagui/").get();
//   String doc_web="http://www.…….cn";
   Document doc = Jsoup.connect("http://www.……cn/download/do_list.jsp?TYPE=1").get();
//   String title = doc.title();
//   Elements hrefs = doc.select("a[href~=www]");
   Elements href2 = doc.select(":containsOwn(尾页)");
   //   for(Element href : hrefs){
//    op.saveUrl(href.toString());
//   }
//   String doc_link = hrefs.attr("href");
//   String doc_name = hrefs.text();

//   for (Element href : hrefs){
//   String doc_link = doc_web+href.attr("href");
//   String doc_name = href.text();
//   System.out.println(" address="+doc_link+" "+" name="+doc_name);
//   op.saveUrl(doc_name,doc_link,doc_web);
//   }
   System.out.println(href2.toString());
//   System.out.println(doc_name);

//   System.out.println("------------------");
//   System.out.println(hrefs.select("[href^=http]"));
//   System.out.println(title);
  } catch (IOException e) {
   e.printStackTrace();
  }
}

}

程序三：

import java.sql.DriverManager;
import java.sql.Connection;
import java.sql.Statement;
import java.sql.ResultSet;
import java.sql.SQLException;

public class crawldata {

public static Connection getConnection() throws Exception {
    String driver = "com.mysql.jdbc.Driver";
    String url = "jdbc:mysql://localhost/test1";
    String username = "root";
    String password = "********";
    Class.forName(driver);
    Connection conn = DriverManager.getConnection(url, username, password);
    return conn;
}

public void saveUrl(String name,String url,String web) {
    Connection conn = null;
    Statement stmt = null;
    ResultSet rs = null;
    try {
      conn = getConnection();
      String sql = "INSERT INTO docfirst(docname,docaddress,docweb) VALUES('"+name+"','"+url+"','"+web+"')";
      stmt = conn.createStatement();
      stmt.execute(sql);

//      rs = stmt.executeQuery("SELECT * FROM docfirst");
//      while (rs.next()) {
//        Integer id = rs.getInt("id");
//        String address = rs.getString("docaddress");
//        System.out.println("id=" + id + " address=" + address);
//      }

    } catch (SQLException ex) {
      System.err.println("SQLException: " + ex.getMessage());
      System.err.println("SQLState: " + ex.getSQLState());
      System.err.println("Message: " + ex.getMessage());
      System.err.println("Vendor error code: " + ex.getErrorCode());
    } catch (Exception e) {
      System.err.println("Exception: " + e.getMessage());
    } finally {
      try {
        rs.close();
        stmt.close();
        conn.close();
      } catch (Exception ignore) {
      }
    }
}
}

一段MYSQL的插入代码