2024-简单点-MediaCrawler解构

最新推荐文章于 2025-10-01 08:15:27 发布

原创

最新推荐文章于 2025-10-01 08:15:27 发布 · 980 阅读

7 ·

CC 4.0 BY-SA版权

文章标签：

#python

MediaCrawler

var.py
recv_sms.py
async_db.py
db.py
main.py
base/base_crawler.py
config/baseconfig.py
config/db_config.py
有待更新

在这里插入图片描述

var.py

from asyncio.tasks import Task
from contextvars import ContextVar
from typing import List

import aiomysql

from async_db import AsyncMysqlDB

request_keyword_var: ContextVar[str] = ContextVar("request_keyword", default="")
crawler_type_var: ContextVar[str] = ContextVar("crawler_type", default="")
comment_tasks_var: ContextVar[List[Task]] = ContextVar("comment_tasks", default=[])
media_crawler_db_var: ContextVar[AsyncMysqlDB] = ContextVar("media_crawler_db_var")
db_conn_pool_var: ContextVar[aiomysql.Pool] = ContextVar("db_conn_pool_var")

在这里插入图片描述

recv_sms.py

import re
from typing import List

import redis
import uvicorn
from fastapi import FastAPI, HTTPException, status
from pydantic import BaseModel

import config
from tools import utils

app = FastAPI()

redis_client = redis.Redis(host=config.REDIS_DB_HOST, password=config.REDIS_DB_PWD)


class SmsNotification(BaseModel):
    platform: str
    current_number: str
    from_number: str
    sms_content: str
    timestamp: str


def extract_verification_code(message: str) -> str:
    """
    Extract verification code of 6 digits from the SMS.
    """
    pattern = re.compile(r'\b[0-9]{6}\b')
    codes: List[str] = pattern.findall(message)
    return codes[0] if codes else ""


@app.post("/")
def receive_sms_notification(sms: SmsNotification):
    """
    Receive SMS notification and send it to Redis.
    Args:
        sms:
            {
                "platform": "xhs",
                "from_number": "1069421xxx134",
                "sms_content": "【小红书】您的验证码是: 171959， 3分钟内有效。请勿向他人泄漏。如非本人操作，可忽略本消息。",
                "timestamp": "1686720601614",
                "current_number": "13152442222"
            }

    Returns:

    """
    utils.logger.info(f"Received SMS notification: {
     
     sms.platform}, {
     
     sms.current_number}")
    sms_code = extract_verification_code(sms.sms_content)
    if sms_code:
        # Save the verification code in Redis and set the expiration time to 3 minutes.
        key = f"{
     
     sms.platform}_{
     
     sms.current_number}"
        redis_client.set(key, sms_code, ex=60 * 3)

    return {
   
   "status": "ok"}


@app.get("/", status_code=status.HTTP_404_NOT_FOUND)
async def not_found():
    raise HTTPException(status_code=404, detail="Not Found")


if __name__ == '__main__':
    uvicorn.run(app, port=8000, host='0.0.0.0')

在这里插入图片描述

async_db.py

from typing import Any, Dict, List, Union

import aiomysql


class AsyncMysqlDB:
    def __init__(self, pool: aiomysql.Pool) -> None:
        self.__pool = pool

    async def query(self, sql: str, *args: Union[str, int]) -> List[Dict[str, Any]]: