A script for download all papers on ICML conference

ICML2018论文下载脚本

最新推荐文章于 2025-04-12 01:10:06 发布

原创最新推荐文章于 2025-04-12 01:10:06 发布 · 296 阅读

0 ·

CC 4.0 BY-SA版权

本文介绍了一个使用Python编写的脚本，该脚本利用多进程池从指定URL批量下载ICML2018会议的论文PDF文件，并保存到本地目录。脚本涉及的技术包括BeautifulSoup解析网页，requests进行网络请求，正则表达式处理文件名，以及os模块管理文件系统。

A script

# coding=utf-8
from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
import traceback
import re
import os
import pdb

prefix = 'http://proceedings.mlr.press/v80/'
save_dir = 'icml2018'

def get_pdf(data):
    href, title = data
    name = re.sub(r'[\\/:*?"<>|\bx0\u2019\u2014\xb0\u2013]', ' ', title) 
    if os.path.isfile(save_dir+"/icml18-%s.pdf" % name):
        print("File already exsists, skip %s" % name)
        return
    try:
        content = requests.get(href).content
        with open(save_dir+"/icml18-%s.pdf" % name, 'wb') as f:  # You may change to "path/to/your/folder"
            f.write(content)
        print("Finish downloading %s" % title)
    except:
        print('Error when downloading %s' % href)
        print(traceback.format_exc())
        
pool = Pool(100)
if not os.path.exists(save_dir):
    os.mkdir(save_dir)
html = requests.get(prefix).content
soup = BeautifulSoup(html, "lxml")
a_list = soup.findAll("p", {"class": "links"})
title_list = soup.findAll("p", {"class": "title"})
title_list = [_.text for _ in title_list]
pdf_list = []
for everya in a_list:
    if everya.contents[3].text == "Download PDF":
        href = everya.contents[3].get("href")
        pdf_list.append(href)
assert len(pdf_list) == len(title_list), "numbers of title and pdf not euqal"
print("Find %d papers" % len(pdf_list))
pool.map(get_pdf, zip(pdf_list, title_list))
print("Find %d papers" % len(pdf_list))