网页爬虫程序pageSpider

最新推荐文章于 2024-09-04 21:39:25 发布

原创最新推荐文章于 2024-09-04 21:39:25 发布 · 199 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#Java #thread #.net

Search Engine 专栏收录该内容

19 篇文章

订阅专栏

本文介绍了一个简单的Java程序，该程序能够通过指定的URL抓取网页内容并将其打印输出。程序利用了Java内置的HTTPURLConnection类来创建与目标URL的连接，并通过设置GET请求方法读取网页数据。

2009-05-05 19:44

该程序仅对单个URL所对应的page网页信息进行抓取（pageSpider.java）。程序流程图如下：

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.URL;

public class pageSpider implements Runnable {

HttpURLConnection httpUrlConnection;
InputStream inputStream;
BufferedReader bufferedReader;
String url;

public pageSpider() {

try {url="http://www.baidu.com"; } catch (Exception e) {e.printStackTrace();}

    try {
     httpUrlConnection = (HttpURLConnection) new URL(url).openConnection(); //创建连接
    } catch (MalformedURLException e) {
     e.printStackTrace();
    } catch (IOException e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
    }

System.out.println("---------start-----------");

    Thread thread = new Thread(this);
    thread.start();
    try {thread.join();} catch (InterruptedException e) {e.printStackTrace();}

System.out.println("----------end------------");
}

public void run() {
    // TODO Auto-generated method stub
    try {
     httpUrlConnection.setRequestMethod("GET");
    } catch (ProtocolException e) {
     e.printStackTrace();
    }

    try {
     httpUrlConnection.setUseCaches(true); //使用缓存
     httpUrlConnection.connect();           //建立连接
    } catch (IOException e) {
     e.printStackTrace();
    }

    try {
     inputStream = httpUrlConnection.getInputStream(); //读取输入流
     bufferedReader = new BufferedReader(new InputStreamReader(inputStream, "gb2312"));
     String string;
     while ((string = bufferedReader.readLine()) != null) {
        System.out.println(string); //打印输出
     }
    } catch (IOException e) {
     e.printStackTrace();
    } finally {
     try {
      bufferedReader.close();
      inputStream.close();
      httpUrlConnection.disconnect();
     } catch (IOException e) {
      e.printStackTrace();
     }