[测试有效]批量下载文献的python代码 sci-hub 论文批量下载代码

原创于 2025-11-24 22:08:36 发布 · 316 阅读

3 ·

CC 4.0 BY-SA版权

文章标签：

#python #开发语言 #生物信息

生物信息专栏收录该内容

48 篇文章

订阅专栏

新星杯·14天创作挑战营·第17期 10w+人浏览 500人参与

[测试有效]
基于您提供的代码，我来写一个更完善的批量下载文献的Python脚本：

-- coding: utf-8 --

“”"
批量文献下载脚本 - 增强版
功能：根据DOI码批量下载文献
作者：基于优快云博客代码优化
版本：v2.0
日期：2024年
“”"

import requests
from bs4 import BeautifulSoup
import os
import time
import random
import logging
from urllib.parse import urlencode
import re
from pathlib import Path

class LiteratureDownloader:
def init(self, doi_file=‘doi_list.txt’, download_dir=‘./downloads/’,
timeout=30, max_retries=3):
“”"
初始化下载器

    Args:
        doi_file: 包含DOI码的文本文件路径
        download_dir: 下载目录
        timeout: 请求超时时间
        max_retries: 最大重试次数
    """
    self.doi_file = doi_file
    self.download_dir = Path(download_dir)
    self.timeout = timeout
    self.max_retries = max_retries
    
    # 创建下载目录
    self.download_dir.mkdir(exist_ok=True)
    
    # 设置日志
    self._setup_logging()
    
    # 设置请求头
    self.headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1',
    }
    
    # Sci-Hub域名列表（可自动切换）
    self.scihub_domains = [
        'sci-hub.se',
        'sci-hub.st',
        'sci-hub.ru',
        'sci-hub.ee'
    ]
    
    self.current_domain_index = 0
    
def _setup_logging(self):
    """设置日志配置"""
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - %(message)s',
        handlers=[
            logging.FileHandler(self.download_dir / 'download.log', encoding='utf-8'),
            logging.StreamHandler()
        ]
    )
    self.logger = logging.getLogger(__name__)

def get_current_domain(self):
    """获取当前可用的Sci-Hub域名"""
    return self.scihub_domains[self.current_domain_index]

def switch_domain(self):
    """切换到下一个域名"""
    self.current_domain_index = (self.current_domain_index + 1) % len(self.scihub_domains)
    self.logger.info(f"切换到域名: {self.get_current_domain()}")

def read_doi_list(self):
    """
    读取DOI列表文件
    
    Returns:
        list: DOI码列表
    """
    try:
        with open(self.doi_file, 'r', encoding='utf-8') as f:
            doi_list = [line.strip() for line in f if line.strip()]
        
        self.logger.info(f"成功读取 {len(doi_list)} 个DOI码")
        return doi_list
        
    except FileNotFoundError:
        self.logger.error(f"DOI文件不存在: {self.doi_file}")
        return []
    except Exception as e:
        self.logger.error(f"读取DOI文件失败: {e}")
        return []

def construct_scihub_url(self, doi):
    """构造Sci-Hub请求URL"""
    base_url = f"https://{self.get_current_domain()}/{doi}"
    return base_url

def extract_download_url(self, html_content):
    """
    从HTML内容中提取下载链接
    
    Args:
        html_content: 网页HTML内容
        
    Returns:
        str: 下载链接或None
    """
    try:
        soup = BeautifulSoup(html_content, 'html.parser')
        
        # 方法1: 通过iframe的src属性获取
        iframe = soup.find('iframe')
        if iframe and iframe.get('src'):
            download_url = iframe.get('src')
            if download_url.startswith('//'):
                download_url = 'https:' + download_url
            return download_url
        
        # 方法2: 通过按钮的onclick属性获取
        button = soup.find('button', onclick=re.compile(r"location.href"))
        if button:
            onclick = button.get('onclick', '')
            match = re.search(r"location\.href='([^']+)'", onclick)
            if match:
                return match.group(1)
        
        # 方法3: 通过链接的href属性获取
        pdf_link = soup.find('a', href=re.compile(r'\.pdf'))
        if pdf_link:
            download_url = pdf_link.get('href')
            if download_url.startswith('//'):
                download_url = 'https:' + download_url
            return download_url
        
        return None
        
    except Exception as e:
        self.logger.error(f"解析下载链接失败: {e}")
        return None

def download_pdf(self, download_url, filename, doi):
    """
    下载PDF文件
    
    Args:
        download_url: 下载链接
        filename: 保存的文件名
        doi: DOI码（用于日志）
        
    Returns:
        bool: 是否下载成功
    """
    try:
        response = requests.get(
            download_url, 
            headers=self.headers, 
            timeout=self.timeout,
            stream=True
        )
        
        if response.status_code == 200:
            filepath = self.download_dir / filename
            
            with open(filepath, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)
            
            # 检查文件大小，避免下载到错误页面
            file_size = filepath.stat().st_size
            if file_size < 1024:  # 小于1KB可能是错误页面
                filepath.unlink()
                self.logger.warning(f"文件大小异常，可能下载失败: {doi}")
                return False
            
            self.logger.info(f"成功下载: {filename} ({file_size} bytes)")
            return True
        else:
            self.logger.warning(f"下载请求失败: {response.status_code}")
            return False
            
    except Exception as e:
        self.logger.error(f"下载过程出错: {e}")
        return False

def sanitize_filename(self, doi):
    """
    根据DOI生成安全的文件名
    
    Args:
        doi: DOI码
        
    Returns:
        str: 安全的文件名
    """
    # 替换文件名中的非法字符
    safe_name = re.sub(r'[\\/*?:"<>|]', "_", doi)
    return f"{safe_name}.pdf"

def process_single_doi(self, doi, retry_count=0):
    """
    处理单个DOI码的下载
    
    Args:
        doi: DOI码
        retry_count: 当前重试次数
        
    Returns:
        bool: 是否成功下载
    """
    if retry_count >= self.max_retries:
        self.logger.error(f"达到最大重试次数，跳过: {doi}")
        return False
    
    try:
        self.logger.info(f"处理DOI: {doi}")
        
        # 构造Sci-Hub URL
        scihub_url = self.construct_scihub_url(doi)
        
        # 发送请求
        response = requests.get(
            scihub_url, 
            headers=self.headers, 
            timeout=self.timeout
        )
        
        if response.status_code != 200:
            self.logger.warning(f"请求失败: {response.status_code}")
            if response.status_code in [403, 404]:
                self.switch_domain()
                return self.process_single_doi(doi, retry_count + 1)
            return False
        
        # 提取下载链接
        download_url = self.extract_download_url(response.text)
        
        if not download_url:
            self.logger.warning(f"无法提取下载链接: {doi}")
            # 尝试切换域名重试
            self.switch_domain()
            return self.process_single_doi(doi, retry_count + 1)
        
        # 生成文件名并下载
        filename = self.sanitize_filename(doi)
        success = self.download_pdf(download_url, filename, doi)
        
        if not success and retry_count < self.max_retries - 1:
            # 下载失败，切换域名重试
            self.switch_domain()
            time.sleep(2)  # 等待一段时间再重试
            return self.process_single_doi(doi, retry_count + 1)
        
        return success
        
    except requests.RequestException as e:
        self.logger.error(f"网络请求异常: {e}")
        if retry_count < self.max_retries - 1:
            self.switch_domain()
            time.sleep(2)
            return self.process_single_doi(doi, retry_count + 1)
        return False
    except Exception as e:
        self.logger.error(f"处理DOI时发生未知错误: {e}")
        return False

def run(self):
    """运行批量下载"""
    self.logger.info("=== 开始批量下载文献 ===")
    
    # 读取DOI列表
    doi_list = self.read_doi_list()
    if not doi_list:
        self.logger.error("没有可用的DOI码，程序退出")
        return
    
    # 统计信息
    success_count = 0
    failed_dois = []
    
    # 逐个处理DOI
    for i, doi in enumerate(doi_list, 1):
        self.logger.info(f"进度: {i}/{len(doi_list)}")
        
        success = self.process_single_doi(doi)
        
        if success:
            success_count += 1
        else:
            failed_dois.append(doi)
        
        # 随机延迟，避免请求过于频繁
        time.sleep(random.uniform(1, 3))
    
    # 输出统计结果
    self.logger.info("=== 下载完成 ===")
    self.logger.info(f"成功下载: {success_count}/{len(doi_list)}")
    
    if failed_dois:
        self.logger.info("失败的DOI码:")
        for doi in failed_dois:
            self.logger.info(f"  {doi}")
        
        # 保存失败列表到文件
        failed_file = self.download_dir / 'failed_dois.txt'
        with open(failed_file, 'w', encoding='utf-8') as f:
            for doi in failed_dois:
                f.write(doi + '\n')
        self.logger.info(f"失败列表已保存至: {failed_file}")

def main():
“”“主函数”“”
# 使用示例
downloader = LiteratureDownloader(
doi_file=‘doi_list.txt’, # DOI文件路径
download_dir=‘./articles/’, # 下载目录
timeout=30, # 超时时间
max_retries=3 # 最大重试次数
)