lora训练超参

原创已于 2025-04-11 16:15:41 修改 · 134 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#lora训练

于 2025-04-10 17:31:28 首次发布

lora训练超参

### model
model_name_or_path: pt_Qwen-Qwen2.5-7B-Instruct

### method
stage: sft
do_train: true
finetuning_type: lora
lora_target: all
lora_rank: 16
lora_alpha: 32
lora_dropout: 0.1
# additional_target: embed_tokens, lm_head
create_new_adapter: true

### dataset
dataset: v7.0_train_data
dataset_dir: data
template: qwen
cutoff_len: 4096
max_samples: 99999999
overwrite_cache: true
preprocessing_num_workers: 16

### output
output_dir: ./saves/Qwen2.5-7B-Instruct/lora/sft_2
logging_steps: 10
save_strategy: steps
save_steps: 500
plot_loss: true
overwrite_output_dir: true

### train
per_device_train_batch_size: 8
gradient_accumulation_steps: 1
learning_rate: 1.0e-4
num_train_epochs: 10
lr_scheduler_type: cosine
warmup_ratio: 0.1
#fp16: true
bf16: true
ddp_timeout: 180000000
seed: 42
optim: adamw_torch
gradient_checkpointing: true
# use_unsloth: true

### eval
do_eval: true
val_size: 0.02
per_device_eval_batch_size: 1
eval_strategy: steps
eval_steps: 500