Phi-3-mini Fastapi部署

一、环境

1. autodl 租用3090 

镜像选择PyTorch 最新版本

2. 安装依赖：

pip install transformers==4.46.3

pip install fastapi==0.104.1

pip install uvicorn==0.24.0.post1

pip install accelerate==0.26.1

3、下载phi模型

from modelscope import snapshot_download
model_dir = snapshot_download('LLM-Research/Phi-3-mini-4k-instruct', cache_dir='/root/autodl-tmp/phi3', revision='master')

 下载完成后，模型存储在:

/root/autodl-tmp/phi3/LLM-Research/Phi-3-mini-4k-instruct/

4、请求fastapi启动模型：

from fastapi import FastAPI, Request
from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
import uvicorn
import json
import datetime
import torch
from threading import Thread
import queue

# 设置设备参数
DEVICE = "cuda"  # 使用CUDA
DEVICE_ID = "0"  # CUDA设备ID，如果未设置则为空
CUDA_DEVICE = f"{DEVICE}:{DEVICE_ID}" if DEVICE_ID else DEVICE 
result_queue = queue.Queue()

# 清理GPU内存函数
def torch_gc():
    if torch.cuda.is_available(): 
        with torch.cuda.device(CUDA_DEVICE):
            torch.cuda.empty_cache() 
            torch.cuda.ipc_collect()

def generate_text(model, generation_kwargs):
    with torch.no_grad():
        result = model.generate(**generation_kwargs)
        result_queue.put(result)  # 将结果放入队列

# 创建FastAPI应用
app = FastAPI()

# 处理POST请求的端点
@app.post("/")
async def inference(request: Request):
    global model, tokenizer  # 声明全局变量以便在函数内部使用模型和分词器
    json_post_raw = await request.json()  # POST请求的数据
    json_post = json.dumps(json_post_raw)  # 将JSON数据转换为字符串
    json_post_list = json.loads(json_post)  # 将字符串转换为Python对象
    
    streamer = TextIteratorStreamer(tokenizer)

    prompt = json_post_list.get('prompt')  # 获取请求中的提示

    print(prompt)

    #streamer = TextIteratorStreamer(tokenizer)
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

    generation_kwargs = {
        **inputs,
        'max_new_tokens': 2048,
        'do_sample': True,
        'top_p': 0.9,
        'temperature': 0.8,
        'pad_token_id': tokenizer.eos_token_id,
        'max_time': 150
    }
    
    generated_text_ids = model.generate(**generation_kwargs)
    print(type(generated_text_ids))  # 应该是 <class 'torch.Tensor'>
    print(generated_text_ids.shape)  # 应该是一维的，例如 (N,)
    
    
    # 如果需要，可以将生成的 token IDs 转换为文本
    generated_text = tokenizer.decode(generated_text_ids[0].tolist(), skip_special_tokens=True)
    
    print(generated_text)

    
    now = datetime.datetime.now()  # 获取当前时间
    time = now.strftime("%Y-%m-%d %H:%M:%S")  # 格式化时间为字符串

    answer = {
        "response": generated_text,
        "status": 200,
        "time": time
    }
    # 构建日志信息
    log = "[" + time + "] " + '", prompt:"' + prompt + '", response:"' + repr(generated_text) + '"'
    print(log)  # 打印日志
    torch_gc()  # 执行GPU内存清理
    return answer  # 返回响应

if __name__ == '__main__':
    model_name_or_path = '/root/autodl-tmp/phi3/LLM-Research/Phi-3-mini-4k-instruct'
    tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
    model = AutoModelForCausalLM.from_pretrained(model_name_or_path, 
        device_map="auto",
        ).eval()
    # model = torch.compile(model)
    uvicorn.run(app, host='0.0.0.0', port=9002, workers=1)

5、测试接口：

curl -X POST "http://127.0.0.1:9002" -H 'Content-Type: application/json' -d '{"prompt": "你好，wirite a c++ function"}'

大模型能返回一段c++代码