unsloth微调gemma3图文代码简析

代码使用了unsloth gemma3-4B的微调示例。
加载本地已经下载好的模型，使用了bnb 4bit量化，加载方便。

# 用户部分代码
    model, processor = FastVisionModel.from_pretrained(
        model_name = "/data/……/……/unsloth/gemma-3-4b-it-bnb-4bit",
        load_in_4bit = True,  # 4 bit quantization to reduce memory
    )

unsloth 加载模型FastVisionModel.from_pretrained函数逻辑：

        # unsloth FastVisionModel.from_pretrained函数检查模型是否包含vision模块
        model_config = AutoConfig.from_pretrained(
                model_name,
                token = token,
                trust_remote_code = trust_remote_code,
        )
        ……
       # Check if VLM
        is_vlm = any(x.endswith("ForConditionalGeneration") for x in model_config.architectures)
        is_vlm = is_vlm or hasattr(model_config, "vision_config")
        if auto_model is None:
            auto_model = AutoModelForVision2Seq if is_vlm else AutoModelForCausalLM
        model, tokenizer = FastBaseModel.from_pretrained(
            model_name        = model_name,
            max_seq_length    = max_seq_length,
            dtype             = _get_dtype(dtype),
            load_in_4bit      = load_in_4bit,
            load_in_8bit      = load_in_8bit,
            full_finetuning   = full_finetuning,
            token             = token,
            device_map        = device_map,
            trust_remote_code = trust_remote_code,
            revision          = revision if not is_peft else None,
            model_types       = model_types,
            tokenizer_name    = tokenizer_name,
            auto_model        = auto_model,
            use_gradient_checkpointing = use_gradient_checkpointing,
            supports_sdpa     = supports_sdpa,
            whisper_language  = whisper_language,
            whisper_task      = whisper_task,
            *args, **kwargs,
        )

在内层FastBaseModel.from_pretrained判断和加载模型，识别出是否需要适配视觉模型处理器

        # unsloth FastBaseModel.from_pretrained
        # 这行判断是否为为vlm模型，使用对应的处理器加载函数
        is_vlm = (auto_model is AutoModelForVision2Seq)
        is_whisper = (whisper_language is not None and whisper_task is not None)
        auto_processor = AutoProcessor if (is_vlm or is_whisper) else AutoTokenizer
        if (whisper_language and whisper_task) or auto_model.__name__.endswith("ForConditionalGeneration"):
           tokenizer = auto_processor.from_pretrained(
                tokenizer_name,
                padding_side = "right",
                token        = token,
                language     = whisper_language,
                task         = whisper_task,
            )
        else:
            tokenizer = auto_processor.from_pretrained(
                tokenizer_name,
                padding_side = "right",
                token        = token,
            )

用户配置模型lora参数

# 用户代码部分
model = FastVisionModel.get_peft_model(
        model,
        finetune_vision_layers     = True, # Turn off for just text!
        finetune_language_layers   = True,  # Should leave on!
        finetune_attention_modules = True,  # Attention good for GRPO
        finetune_mlp_modules       = True,  # SHould leave on always!

        r = 16,