爬虫抓取pdf、PPT等文件代码

最新推荐文章于 2024-08-23 11:48:36 发布

原创

最新推荐文章于 2024-08-23 11:48:36 发布 · 2k 阅读

2 ·

CC 4.0 BY-SA版权

这个Java程序实现了一个爬虫，利用Apache HttpClient库抓取网页上的PDF和PPT文件。它首先解析HTML内容，找出链接，然后下载以.pdf或.PDF结尾的文件。程序使用多线程下载，并设置了超时参数以处理网络延迟。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

package com;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.params.ConnManagerParams;
import org.apache.http.conn.params.ConnPerRouteBean;
import org.apache.http.conn.scheme.PlainSocketFactory;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;
import org.apache.http.params.BasicHttpParams;
import org.apache.http.params.HttpConnectionParams;
import org.apache.http.params.HttpParams;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.AndFilter;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
/**
* @author ZhangZhenli
* @date 2015-6-2
*/
public class CrawlerPDF implements Runnable {
public static String SAVE = "C:/Users/Public/Downloads";// 下载保存路径
private String url = "";// 要抓取的网页地址

public CrawlerPDF(String url) {
this.url = url;
}

public CrawlerPDF() {
}

/**
*
* @param url
* 要抓取的网页的地址
* @return 这个对应的内容
* @throws ClientProtocolException
* @throws IOException
*/
private String crawl(String url) throws ClientProtocolException,
IOException {

最低0.47元/天解锁文章