使用Transformer实现Qwen模型私有化部署

Python+Flask

非流式输出：

from PIL import Image
from flask import Flask, request, jsonify
from flask_cors import CORS
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor

model = Qwen2VLForConditionalGeneration.from_pretrained(
    "../Model/Qwen2-VL-2B-Instruct", torch_dtype="auto", device_map="auto"
)
processor = AutoProcessor.from_pretrained("../Model/Qwen2-VL-2B-Instruct")
#这里的两个目录都是你模型根目录

app = Flask(__name__)
CORS(app)

@app.route('/', methods=['POST'])
def process_message():
    try:
        message = request.form.get('question')
        image_file = request.files.get('picture')

        if not message or not image_file:
            return jsonify({'success': False, 'message': 'No question or image received!'})


        image = Image.open(image_file.stream)
        conversation = [
            {
                "role": "user",
                "content": [
                    {"type": "image"},
                    {"type": "text", "text": message},  
                ],
            }
        ]


        text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)


        inputs = processor(
            text=[text_prompt], images=[image], padding=True, return_tensors="pt"
        )
        inputs = inputs.to("cuda") 

        output_ids = model.generate(**inputs, max_new_tokens=128)

        generated_ids = [
            output_ids[len(input_ids):]
            for input_ids, output_ids in zip(inputs.input_ids, output_ids)
        ]

        output_text = processor.batch_decode(
            generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True
        )

        print(output_text[0])
        return jsonify({'success': True, 'message': output_text[0]})

    except Exception as e:
        print(f"Error: {e}")
        return jsonify({'success': False, 'message': 'An error occurred while processing the request.'})

if __name__ == '__main__':
    app.run(debug=True)

流式输出：

from PIL import Image
from flask import Flask, request, jsonify, Response
from flask_cors import CORS
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
import time
import torch

model_string = '../Model/Qwen2-VL-2B-Instruct'

app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": "*"}})

model = Qwen2VLForConditionalGeneration.from_pretrained(
    model_string,
    torch_dtype=torch.float16,
    device_map="cuda"
)

processor = AutoProcessor.from_pretrained(model_string)

@app.route('/', methods=['POST'])
def process_message():
    try:
        message = request.form.get('question')
        image_file = request.files.get('picture')

        if not message or not image_file:
            return jsonify({'success': False, 'message': '缺少问题或图片！'})

        image = Image.open(image_file.stream).convert("RGB")
        image = image.resize((512, 512), resample=Image.Resampling.BILINEAR) 

        conversation = [
            {
                "role": "user",
                "content": [
                    {"type": "image"},
                    {"type": "text", "text": message},
                ],
            }
        ]

        text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)

        inputs = processor(
            text=[text_prompt], images=[image], padding=True, return_tensors="pt"
        ).to("cuda")

        output_ids = model.generate(
            **inputs,
            max_new_tokens=400, 
            do_sample=False,     
            temperature=0.7,
            top_p=0.9,
            return_dict_in_generate=True,
            output_scores=True
        )

        def stream_output():
            text = ''
            OK = False
            for token_id in output_ids.sequences[0]:
                token = processor.decode([token_id], skip_special_tokens=True)
                if OK:
                    yield token
                    time.sleep(0.02)
                else:
                    text += token
                    if text.count('assistant') == 2:
                        OK = True

        return Response(stream_output(), content_type='text/plain;charset=utf-8')

    except Exception as e:
        print(f"错误: {e}")
        return jsonify({'success': False, 'message': '处理请求时发生错误。'})
    finally:
        torch.cuda.empty_cache()

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)