爬虫限速问题解决方案

限速

动态限制函数在固定时间窗口内的调用次数，并动态调整调用间隔。

from functools import wraps
import time
import threading
from collections import deque


def dynamic_rate_limit(max_calls, time_window):
    """
    装饰器：动态限制函数在固定时间窗口内的调用次数，并动态调整调用间隔。
    :param max_calls: 时间窗口内允许的最大调用次数
    :param time_window: 时间窗口长度（秒）
    """

    def decorator(func):
        call_history = deque()  # 存储调用时间戳
        lock = threading.Lock()  # 线程锁
        next_call_time = 0  # 下一次允许调用的时间

        @wraps(func)
        def wrapper(*args, **kwargs):
            nonlocal next_call_time
            current_time = time.time()

            with lock:
                # 等待到下一个允许调用的时间
                if current_time < next_call_time:
                    sleep_duration = next_call_time - current_time
                    time.sleep(sleep_duration)
                    current_time = time.time()  # 更新当前时间

                # 清理过期的时间戳（早于当前时间窗口）
                while call_history and call_history[0] <= current_time - time_window:
                    call_history.popleft()

                # 若当前窗口内调用已达上限，等待窗口滑动
                if len(call_history) >= max_calls:
                    oldest_call = call_history[0]
                    window_end = oldest_call + time_window
                    sleep_duration = max(0, window_end - current_time)
                    time.sleep(sleep_duration)
                    current_time = time.time()  # 更新当前时间
                    # 再次清理可能过期的记录
                    while call_history and call_history[0] <= current_time - time_window:
                        call_history.popleft()

                # 记录当前调用时间
                call_history.append(current_time)

                # 计算剩余时间和剩余允许调用次数
                remaining_calls = max_calls - len(call_history)
                if call_history:
                    window_start = call_history[0]
                    remaining_time = (window_start + time_window) - current_time
                else:
                    remaining_time = time_window  # 窗口内无调用，剩余时间为整个窗口

                # 动态调整下一次调用时间（若剩余时间充足）
                if remaining_calls > 0 and remaining_time > (time_window / 2):
                    average_interval = remaining_time / (remaining_calls + 1)
                    next_call_time = current_time + average_interval
                else:
                    next_call_time = current_time  # 允许立即调用

            return func(*args, **kwargs)

        return wrapper

    return decorator


# 使用示例
@dynamic_rate_limit(max_calls=3, time_window=10)
def test_function():
    print(f"执行于: {time.strftime('%Y-%m-%d %H:%M:%S')}")


# 测试调用
for _ in range(20):
    test_function()
    # time.sleep(10)  # 模拟多次调用

# result
"""
执行于: 2025-02-27 11:32:57
执行于: 2025-02-27 11:33:00
执行于: 2025-02-27 11:33:03
执行于: 2025-02-27 11:33:07
执行于: 2025-02-27 11:33:10
执行于: 2025-02-27 11:33:13
执行于: 2025-02-27 11:33:17
执行于: 2025-02-27 11:33:20
执行于: 2025-02-27 11:33:23
执行于: 2025-02-27 11:33:27
执行于: 2025-02-27 11:33:30
执行于: 2025-02-27 11:33:33
执行于: 2025-02-27 11:33:37
执行于: 2025-02-27 11:33:40
执行于: 2025-02-27 11:33:43
执行于: 2025-02-27 11:33:47
执行于: 2025-02-27 11:33:50
执行于: 2025-02-27 11:33:53
执行于: 2025-02-27 11:33:57
执行于: 2025-02-27 11:34:00

"""