从零开始写一个IP代理池

原创

已于 2023-03-14 15:56:02 修改 · 272 阅读

2 ·

CC 4.0 BY-SA版权

文章标签：

#flask #python #后端

于 2023-03-14 15:53:34 首次发布

该代码实现了一个基于Python的HTTP代理池系统，包括代理的获取、存储、过期检查和分配策略。使用PriorityQueue管理代理优先级，支持随机、轮询和LRU三种方式获取多个代理。同时，有两个线程分别负责生产新代理和监控过期代理，Flask应用提供了API接口供外部请求。


# -*- encoding: utf-8 -*-
import os
import json
from queue import PriorityQueue
import time
import random
import logging
import threading
import requests
from flask import Flask, jsonify, request

UN_USED_EXPIRED_TIME=5
PER_SIZE_PRODUCER = 10
url = "http://192.168.1.1:50000/V0/get-ip/"

class Proxy:
    def __init__(self, ip, expired_seconds, expired_time):
        self.priority = int(time.time()) + expired_seconds - UN_USED_EXPIRED_TIME
        self.ip = ip
        self.expired_time = expired_time

    def __lt__(self, other):
        return self.priority <= other.priority

    def __gt__(self, other):
        return self.priority > other.priority

    def __str__(self):
        return "Proxy(priority[%s], ip[%s], endtime[%s])" % (self.priority, self.ip, self.expired_time)

class HttpProxyPool:
    def __init__(self):
        self._pressure_ip_proxy_queue = PriorityQueue()
        self._ip_proxies = []
        self._lru_proxies = []
        self._round_robin_index = 0
        self._lock = threading.Lock()

    def put_proxies(self):
        ips = json.loads(requests.get(url % PER_SIZE_PRODUCER).text)["data"]
        for ip in ips:
            host = ip["ip"]
            port = ip["port"]
            expred_seconds = int(ip["timeout"][:ip["timeout"].find("(")])
            expired_time = ip["end_time"]
            proxy = Proxy("%s:%s"

最低0.47元/天解锁文章