import gradio as gr
import pandas as pd
import os
import shutil
import json
from transformers import (
AutoTokenizer,
AutoModelForCausalLM,
TrainingArguments,
Trainer,
DataCollatorForLanguageModeling
)
from datasets import Dataset, load_dataset
import torch
import evaluate
import docx
from docx import Document
from io import BytesIO
import re
# 常量定义
MODEL_OPTIONS = {
"DeepSeek-7B": "deepseek-ai/deepseek-llm-7b-base",
"Qwen-7B": "Qwen/Qwen-7B",
"Llama3-8B": "meta-llama/Meta-Llama-3-8B"
}
TEMP_DIR = "temp_data"
SAVE_DIR = "saved_models"
DOCX_DIR = "uploaded_docx"
# 确保目录存在
os.makedirs(TEMP_DIR, exist_ok=True)
os.makedirs(SAVE_DIR, exist_ok=True)
os.makedirs(DOCX_DIR, exist_ok=True)
def convert_docx_to_jsonl(docx_file, output_file):
"""将DOCX文件转换为JSONL格式"""
try:
# 读取DOCX文件内容
doc = Document(BytesIO(docx_file))
# 提取所有段落
paragraphs = [p.text.strip() for p in doc.paragraphs if p.text.strip()]
# 解析问题-答案对
qa_pairs = []
current_question = None
for text in paragraphs:
# 检查是否为问题行
if re.match(r'^\d+[\.\)]?\s*问题[:：]', text) or re.match(r'^问题[:：]', text, re.IGNORECASE):
if current_question:
# 保存上一个问题（如果没有答案）
qa_pairs.append({"question": current_question, "answer": ""})
current_question = re.sub(r'^\d+[\.\)]?\s*问题[:：]\s*', '', text, flags=re.IGNORECASE).strip()
# 检查是否为答案行
elif re.match(r'^\d+[\.\)]?\s*答案[:：]', text) or re.match(r'^答案[:：]', text, re.IGNORECASE):
if current_question:
answer = re.sub(r'^\d+[\.\)]?\s*答案[:：]\s*', '', text, flags=re.IGNORECASE).strip()
qa_pairs.append({"question": current_question, "answer": answer})
current_question = None
else:
# 没有对应问题的答案
pass
# 如果是问题或答案的延续
elif current_question:
if not qa_pairs or qa_pairs[-1]["question"] != current_question:
# 当前问题还没有添加到列表
qa_pairs.append({"question": current_question, "answer": text})
else:
# 添加到最后一个答案
qa_pairs[-1]["answer"] += "\n" + text
# 保存最后一个问题（如果有）
if current_question:
qa_pairs.append({"question": current_question, "answer": ""})
# 写入JSONL文件
with open(output_file, 'w', encoding='utf-8') as f:
for pair in qa_pairs:
json_line = json.dumps(pair, ensure_ascii=False)
f.write(json_line + '\n')
return f"成功转换 {len(qa_pairs)} 个问题-答案对"
except Exception as e:
return f"转换失败: {str(e)}"
def fine_tune_model(model_name, batch_size, learning_rate, num_epochs, progress=gr.Progress()):
"""微调大语言模型"""
progress(0.1, desc="加载数据集")
# 检查数据集是否存在
required_files = ["train.jsonl", "valid.jsonl", "test.jsonl"]
missing_files = [f for f in required_files if not os.path.exists(os.path.join(TEMP_DIR, f))]
if missing_files:
return f"数据集不完整，缺少文件: {', '.join(missing_files)}"
# 使用正确的load_dataset方法加载数据
progress(0.2, desc="加载数据集")
train_dataset = load_dataset('json', data_files=os.path.join(TEMP_DIR, "train.jsonl"))['train']
valid_dataset = load_dataset('json', data_files=os.path.join(TEMP_DIR, "valid.jsonl"))['train']
# 预处理数据
progress(0.3, desc="预处理数据")
model_checkpoint = MODEL_OPTIONS[model_name]
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
tokenizer.pad_token = tokenizer.eos_token
# 格式化指令
def format_instruction(example):
text = f"### 问题:\n{example['question']}\n\n### 回答:\n{example['answer']}"
return {"text": text}
# 应用格式化
train_dataset = train_dataset.map(format_instruction)
valid_dataset = valid_dataset.map(format_instruction)
# 分词函数
def tokenize_function(examples):
return tokenizer(
examples["text"],
truncation=True,
max_length=512,
padding="max_length"
)
# 应用分词
tokenized_train = train_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
tokenized_valid = valid_dataset.map(tokenize_function, batched=True, remove_columns=["text"])
# 加载模型
progress(0.4, desc="加载模型")
model = AutoModelForCausalLM.from_pretrained(
model_checkpoint,
torch_dtype=torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float32
)
# 训练参数
training_args = TrainingArguments(
output_dir=SAVE_DIR,
evaluation_strategy="epoch",
learning_rate=learning_rate,
per_device_train_batch_size=batch_size,
per_device_eval_batch_size=batch_size,
num_train_epochs=num_epochs,
weight_decay=0.01,
save_strategy="epoch",
logging_steps=10,
report_to="none",
fp16=torch.cuda.is_available(),
gradient_accumulation_steps=2,
warmup_ratio=0.1,
save_total_limit=2
)
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=False
)
# 训练器
trainer = Trainer(
model=model,
args=training_args,
train_dataset=tokenized_train,
eval_dataset=tokenized_valid,
data_collator=data_collator,
)
# 开始训练
progress(0.5, desc="开始训练")
trainer.train()
# 保存模型
progress(0.9, desc="保存模型")
model_save_path = os.path.join(SAVE_DIR, f"final_model_{model_name}")
model.save_pretrained(model_save_path)
tokenizer.save_pretrained(model_save_path)
return f"模型训练完成！模型已保存到 {model_save_path}"
def evaluate_model():
"""评估模型性能"""
# 查找最新模型
model_dirs = [d for d in os.listdir(SAVE_DIR) if d.startswith("final_model_")]
if not model_dirs:
return "请先训练模型"
# 使用最后训练的模型
model_dir = sorted(model_dirs)[-1]
model_path = os.path.join(SAVE_DIR, model_dir)
# 加载测试数据
test_path = os.path.join(TEMP_DIR, "test.jsonl")
if not os.path.exists(test_path):
return "测试集不存在"
test_data = pd.read_json(test_path, lines=True)
# 加载模型
model = AutoModelForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)
# 评估指标
bleu = evaluate.load("bleu")
rouge = evaluate.load("rouge")
bertscore = evaluate.load("bertscore")
predictions = []
references = []
for _, row in test_data[:50].iterrows(): # 抽样评估
input_text = f"### 问题:\n{row['question']}\n\n### 回答:\n"
inputs = tokenizer(input_text, return_tensors="pt", max_length=512, truncation=True)
# 生成回答
outputs = model.generate(
inputs.input_ids,
max_new_tokens=200,
pad_token_id=tokenizer.eos_token_id,
temperature=0.7,
top_p=0.9,
do_sample=True
)
prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)
actual = row['answer']
# 提取生成的答案部分
try:
prediction = prediction.split("### 回答:")[1].strip()
except:
prediction = prediction
predictions.append(prediction)
references.append(actual)
# 计算指标
bleu_score = bleu.compute(predictions=predictions, references=references)['bleu']
rouge_score = rouge.compute(predictions=predictions, references=references)
bert_score = bertscore.compute(predictions=predictions, references=references, lang="zh")
# 计算BERTScore平均值
bert_precision = sum(bert_score['precision']) / len(bert_score['precision'])
bert_recall = sum(bert_score['recall']) / len(bert_score['recall'])
bert_f1 = sum(bert_score['f1']) / len(bert_score['f1'])
report = f"""
### 模型评估报告 (50个样本)
- **BLEU 分数**: {bleu_score:.4f}
- **ROUGE 分数**:
- ROUGE-1: {rouge_score['rouge1']:.4f}
- ROUGE-2: {rouge_score['rouge2']:.4f}
- ROUGE-L: {rouge_score['rougeL']:.4f}
- **BERTScore**:
- Precision: {bert_precision:.4f}
- Recall: {bert_recall:.4f}
- F1: {bert_f1:.4f}
"""
return report
def ask_question(question):
"""使用微调后的模型回答问题"""
# 查找最新模型
model_dirs = [d for d in os.listdir(SAVE_DIR) if d.startswith("final_model_")]
if not model_dirs:
return "请先训练模型"
# 使用最后训练的模型
model_dir = sorted(model_dirs)[-1]
model_path = os.path.join(SAVE_DIR, model_dir)
model = AutoModelForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)
input_text = f"### 问题:\n{question}\n\n### 回答:\n"
inputs = tokenizer(input_text, return_tensors="pt", max_length=512, truncation=True)
# 生成回答
outputs = model.generate(
inputs.input_ids,
max_new_tokens=300,
temperature=0.7,
top_p=0.9,
do_sample=True,
pad_token_id=tokenizer.eos_token_id
)
response = tokenizer.decode(outputs[0], skip_special_tokens=True)
# 提取回答部分
try:
response = response.split("### 回答:")[1].strip()
except:
response = response
return response
# 创建Gradio界面
with gr.Blocks(title="专业智能问答助手", theme=gr.themes.Soft()) as demo:
gr.Markdown("# 🧠 专业智能问答助手")
gr.Markdown("上传您的专业问答数据集，微调大语言模型，创建专属问答助手")
with gr.Tab("数据集上传"):
gr.Markdown("### 上传问答数据集")
with gr.Row():
train_file = gr.File(label="训练集 (支持.jsonl或.docx)", file_types=[".jsonl", ".docx"])
valid_file = gr.File(label="验证集 (支持.jsonl或.docx)", file_types=[".jsonl", ".docx"])
test_file = gr.File(label="测试集 (支持.jsonl或.docx)", file_types=[".jsonl", ".docx"])
upload_info = gr.Textbox(label="上传状态", interactive=False)
def save_files(train, valid, test):
if not (train and valid and test):
return "请上传所有三个文件"
try:
# 处理训练集
train_path = train.name
if train_path.endswith('.docx'):
output_path = os.path.join(TEMP_DIR, "train.jsonl")
result = convert_docx_to_jsonl(open(train_path, 'rb').read(), output_path)
shutil.copy(train_path, os.path.join(DOCX_DIR, "train.docx"))
train_msg = f"训练集: {result}"
else:
shutil.copy(train_path, os.path.join(TEMP_DIR, "train.jsonl"))
train_msg = "训练集: JSONL格式已保存"
# 处理验证集
valid_path = valid.name
if valid_path.endswith('.docx'):
output_path = os.path.join(TEMP_DIR, "valid.jsonl")
result = convert_docx_to_jsonl(open(valid_path, 'rb').read(), output_path)
shutil.copy(valid_path, os.path.join(DOCX_DIR, "valid.docx"))
valid_msg = f"验证集: {result}"
else:
shutil.copy(valid_path, os.path.join(TEMP_DIR, "valid.jsonl"))
valid_msg = "验证集: JSONL格式已保存"
# 处理测试集
test_path = test.name
if test_path.endswith('.docx'):
output_path = os.path.join(TEMP_DIR, "test.jsonl")
result = convert_docx_to_jsonl(open(test_path, 'rb').read(), output_path)
shutil.copy(test_path, os.path.join(DOCX_DIR, "test.docx"))
test_msg = f"测试集: {result}"
else:
shutil.copy(test_path, os.path.join(TEMP_DIR, "test.jsonl"))
test_msg = "测试集: JSONL格式已保存"
return f"文件上传成功！\n{train_msg}\n{valid_msg}\n{test_msg}"
except Exception as e:
return f"上传失败: {str(e)}"
upload_btn = gr.Button("上传并保存数据集")
upload_btn.click(save_files, [train_file, valid_file, test_file], upload_info)
with gr.Tab("模型训练"):
gr.Markdown("### 微调大语言模型")
with gr.Row():
model_choice = gr.Dropdown(
choices=list(MODEL_OPTIONS.keys()),
value="DeepSeek-7B",
label="选择基座模型"
)
batch_size = gr.Slider(1, 16, value=4, step=1, label="批处理大小")
with gr.Row():
learning_rate = gr.Number(value=2e-5, label="学习率")
num_epochs = gr.Slider(1, 10, value=3, step=1, label="训练轮数")
train_btn = gr.Button("开始训练", variant="primary")
train_output = gr.Textbox(label="训练日志", lines=8)
train_btn.click(
fine_tune_model,
[model_choice, batch_size, learning_rate, num_epochs],
train_output
)
with gr.Tab("模型评估"):
gr.Markdown("### 评估模型性能")
eval_btn = gr.Button("运行评估", variant="primary")
eval_report = gr.Markdown("评估结果将显示在这里...")
eval_btn.click(evaluate_model, [], eval_report)
with gr.Tab("问答测试"):
gr.Markdown("### 与您专属的智能助手对话")
question_input = gr.Textbox(
label="输入您的问题",
placeholder="例如: 风振控制中如何计算传导桁架屋盖的风振频率?",
lines=3
)
ask_btn = gr.Button("提问", variant="primary")
answer_output = gr.Textbox(label="模型回答", lines=8, interactive=False)
ask_btn.click(ask_question, question_input, answer_output)
with gr.Tab("DOCX转换工具"):
gr.Markdown("### DOCX格式转换工具")
gr.Markdown("上传单个DOCX文件转换为JSONL格式")
docx_file = gr.File(label="上传DOCX文件", file_types=[".docx"])
convert_btn = gr.Button("转换为JSONL", variant="primary")
convert_info = gr.Textbox(label="转换结果", interactive=False)
download_jsonl = gr.File(label="下载JSONL文件", interactive=False)
def convert_single_file(file):
if not file:
return "请上传文件", None
try:
# 创建临时目录
os.makedirs("temp_conversions", exist_ok=True)
output_path = os.path.join("temp_conversions", "converted.jsonl")
# 转换文件
result = convert_docx_to_jsonl(open(file.name, 'rb').read(), output_path)
return result, output_path
except Exception as e:
return f"转换失败: {str(e)}", None
convert_btn.click(
convert_single_file,
docx_file,
[convert_info, download_jsonl]
)
gr.Markdown("---")
gr.Markdown("### 使用说明\n1. 在'数据集上传'页面上传三个JSONL或DOCX格式的数据集文件\n2. DOCX文件会自动转换为JSONL格式\n3. 在'模型训练'页面选择参数并开始训练\n4. 训练完成后在'问答测试'页面提问")
# 启动应用
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7860)
运行出现问题：
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 349 examples [00:00, 8773.17 examples/s]
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 85 examples [00:00, 4559.32 examples/s]
Traceback (most recent call last):
File "D:\Lib\site-packages\urllib3\connection.py", line 198, in _new_conn
sock = connection.create_connection(
(self._dns_host, self.port),
...<2 lines>...
socket_options=self.socket_options,
)
File "D:\Lib\site-packages\urllib3\util\connection.py", line 85, in create_connection
raise err
File "D:\Lib\site-packages\urllib3\util\connection.py", line 73, in create_connection
sock.connect(sa)
~~~~~~~~~~~~^^^^
TimeoutError: timed out
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 787, in urlopen
response = self._make_request(
conn,
...<10 lines>...
**response_kw,
)
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 488, in _make_request
raise new_e
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 464, in _make_request
self._validate_conn(conn)
~~~~~~~~~~~~~~~~~~~^^^^^^
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 1093, in _validate_conn
conn.connect()
~~~~~~~~~~~~^^
File "D:\Lib\site-packages\urllib3\connection.py", line 704, in connect
self.sock = sock = self._new_conn()
~~~~~~~~~~~~~~^^
File "D:\Lib\site-packages\urllib3\connection.py", line 207, in _new_conn
raise ConnectTimeoutError(
...<2 lines>...
) from e
urllib3.exceptions.ConnectTimeoutError: (<urllib3.connection.HTTPSConnection object at 0x000001A453580550>, 'Connection to huggingface.co timed out. (connect timeout=10)')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Lib\site-packages\requests\adapters.py", line 667, in send
resp = conn.urlopen(
method=request.method,
...<9 lines>...
chunked=chunked,
)
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 841, in urlopen
retries = retries.increment(
method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
)
File "D:\Lib\site-packages\urllib3\util\retry.py", line 519, in increment
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /deepseek-ai/deepseek-llm-7b-base/resolve/main/config.json (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001A453580550>, 'Connection to huggingface.co timed out. (connect timeout=10)'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1533, in _get_metadata_or_catch_error
metadata = get_hf_file_metadata(
url=url, proxies=proxies, timeout=etag_timeout, headers=headers, token=token
)
File "D:\Lib\site-packages\huggingface_hub\utils\_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1450, in get_hf_file_metadata
r = _request_wrapper(
method="HEAD",
...<5 lines>...
timeout=timeout,
)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 286, in _request_wrapper
response = _request_wrapper(
method=method,
...<2 lines>...
**params,
)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 309, in _request_wrapper
response = http_backoff(method=method, url=url, **params, retry_on_exceptions=(), retry_on_status_codes=(429,))
File "D:\Lib\site-packages\huggingface_hub\utils\_http.py", line 310, in http_backoff
response = session.request(method=method, url=url, **kwargs)
File "D:\Lib\site-packages\requests\sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "D:\Lib\site-packages\requests\sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "D:\Lib\site-packages\huggingface_hub\utils\_http.py", line 96, in send
return super().send(request, *args, **kwargs)
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Lib\site-packages\requests\adapters.py", line 688, in send
raise ConnectTimeout(e, request=request)
requests.exceptions.ConnectTimeout: (MaxRetryError("HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /deepseek-ai/deepseek-llm-7b-base/resolve/main/config.json (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001A453580550>, 'Connection to huggingface.co timed out. (connect timeout=10)'))"), '(Request ID: 43ece770-ec9c-415a-b046-2c922abd8388)')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Lib\site-packages\transformers\utils\hub.py", line 470, in cached_files
hf_hub_download(
~~~~~~~~~~~~~~~^
path_or_repo_id,
^^^^^^^^^^^^^^^^
...<10 lines>...
local_files_only=local_files_only,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "D:\Lib\site-packages\huggingface_hub\utils\_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1008, in hf_hub_download
return _hf_hub_download_to_cache_dir(
# Destination
...<14 lines>...
force_download=force_download,
)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1115, in _hf_hub_download_to_cache_dir
_raise_on_head_call_error(head_call_error, force_download, local_files_only)
~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1648, in _raise_on_head_call_error
raise LocalEntryNotFoundError(
...<3 lines>...
) from head_call_error
huggingface_hub.errors.LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Lib\site-packages\gradio\queueing.py", line 625, in process_events
response = await route_utils.call_process_api(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...<5 lines>...
)
^
File "D:\Lib\site-packages\gradio\route_utils.py", line 322, in call_process_api
output = await app.get_blocks().process_api(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...<11 lines>...
)
^
File "D:\Lib\site-packages\gradio\blocks.py", line 2193, in process_api
result = await self.call_function(
^^^^^^^^^^^^^^^^^^^^^^^^^
...<8 lines>...
)
^
File "D:\Lib\site-packages\gradio\blocks.py", line 1704, in call_function
prediction = await anyio.to_thread.run_sync( # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
fn, *processed_input, limiter=self.limiter
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "D:\Lib\site-packages\anyio\to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
func, args, abandon_on_cancel=abandon_on_cancel, limiter=limiter
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "D:\Lib\site-packages\anyio\_backends\_asyncio.py", line 2470, in run_sync_in_worker_thread
return await future
^^^^^^^^^^^^
File "D:\Lib\site-packages\anyio\_backends\_asyncio.py", line 967, in run
result = context.run(func, *args)
File "D:\Lib\site-packages\gradio\utils.py", line 894, in wrapper
response = f(*args, **kwargs)
File "C:\Users\Y\Desktop\python\101.py", line 109, in fine_tune_model
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
File "D:\Lib\site-packages\transformers\models\auto\tokenization_auto.py", line 970, in from_pretrained
config = AutoConfig.from_pretrained(
pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
)
File "D:\Lib\site-packages\transformers\models\auto\configuration_auto.py", line 1153, in from_pretrained
config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Lib\site-packages\transformers\configuration_utils.py", line 595, in get_config_dict
config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Lib\site-packages\transformers\configuration_utils.py", line 654, in _get_config_dict
resolved_config_file = cached_file(
pretrained_model_name_or_path,
...<10 lines>...
_commit_hash=commit_hash,
)
File "D:\Lib\site-packages\transformers\utils\hub.py", line 312, in cached_file
file = cached_files(path_or_repo_id=path_or_repo_id, filenames=[filename], **kwargs)
File "D:\Lib\site-packages\transformers\utils\hub.py", line 543, in cached_files
raise OSError(
...<3 lines>...
) from e
OSError: We couldn't connect to 'https://huggingface.co' to load the files, and couldn't find them in the cached files.
Checkout your internet connection or see how to run the library in offline mode at 'https://huggingface.co/docs/transformers/installation#offline-mode'.
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 349 examples [00:00, 24640.81 examples/s]
Generating train split: 0 examples [00:00, ? examples/s]
Generating train split: 85 examples [00:00, 5696.14 examples/s]
Traceback (most recent call last):
File "D:\Lib\site-packages\urllib3\connection.py", line 198, in _new_conn
sock = connection.create_connection(
(self._dns_host, self.port),
...<2 lines>...
socket_options=self.socket_options,
)
File "D:\Lib\site-packages\urllib3\util\connection.py", line 85, in create_connection
raise err
File "D:\Lib\site-packages\urllib3\util\connection.py", line 73, in create_connection
sock.connect(sa)
~~~~~~~~~~~~^^^^
TimeoutError: timed out
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 787, in urlopen
response = self._make_request(
conn,
...<10 lines>...
**response_kw,
)
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 488, in _make_request
raise new_e
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 464, in _make_request
self._validate_conn(conn)
~~~~~~~~~~~~~~~~~~~^^^^^^
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 1093, in _validate_conn
conn.connect()
~~~~~~~~~~~~^^
File "D:\Lib\site-packages\urllib3\connection.py", line 704, in connect
self.sock = sock = self._new_conn()
~~~~~~~~~~~~~~^^
File "D:\Lib\site-packages\urllib3\connection.py", line 207, in _new_conn
raise ConnectTimeoutError(
...<2 lines>...
) from e
urllib3.exceptions.ConnectTimeoutError: (<urllib3.connection.HTTPSConnection object at 0x000001A453580690>, 'Connection to huggingface.co timed out. (connect timeout=10)')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Lib\site-packages\requests\adapters.py", line 667, in send
resp = conn.urlopen(
method=request.method,
...<9 lines>...
chunked=chunked,
)
File "D:\Lib\site-packages\urllib3\connectionpool.py", line 841, in urlopen
retries = retries.increment(
method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
)
File "D:\Lib\site-packages\urllib3\util\retry.py", line 519, in increment
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /deepseek-ai/deepseek-llm-7b-base/resolve/main/config.json (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001A453580690>, 'Connection to huggingface.co timed out. (connect timeout=10)'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1533, in _get_metadata_or_catch_error
metadata = get_hf_file_metadata(
url=url, proxies=proxies, timeout=etag_timeout, headers=headers, token=token
)
File "D:\Lib\site-packages\huggingface_hub\utils\_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1450, in get_hf_file_metadata
r = _request_wrapper(
method="HEAD",
...<5 lines>...
timeout=timeout,
)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 286, in _request_wrapper
response = _request_wrapper(
method=method,
...<2 lines>...
**params,
)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 309, in _request_wrapper
response = http_backoff(method=method, url=url, **params, retry_on_exceptions=(), retry_on_status_codes=(429,))
File "D:\Lib\site-packages\huggingface_hub\utils\_http.py", line 310, in http_backoff
response = session.request(method=method, url=url, **kwargs)
File "D:\Lib\site-packages\requests\sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "D:\Lib\site-packages\requests\sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "D:\Lib\site-packages\huggingface_hub\utils\_http.py", line 96, in send
return super().send(request, *args, **kwargs)
~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Lib\site-packages\requests\adapters.py", line 688, in send
raise ConnectTimeout(e, request=request)
requests.exceptions.ConnectTimeout: (MaxRetryError("HTTPSConnectionPool(host='huggingface.co', port=443): Max retries exceeded with url: /deepseek-ai/deepseek-llm-7b-base/resolve/main/config.json (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001A453580690>, 'Connection to huggingface.co timed out. (connect timeout=10)'))"), '(Request ID: 2559bc67-bf46-437c-8680-4a379783d3f0)')
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Lib\site-packages\transformers\utils\hub.py", line 470, in cached_files
hf_hub_download(
~~~~~~~~~~~~~~~^
path_or_repo_id,
^^^^^^^^^^^^^^^^
...<10 lines>...
local_files_only=local_files_only,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "D:\Lib\site-packages\huggingface_hub\utils\_validators.py", line 114, in _inner_fn
return fn(*args, **kwargs)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1008, in hf_hub_download
return _hf_hub_download_to_cache_dir(
# Destination
...<14 lines>...
force_download=force_download,
)
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1115, in _hf_hub_download_to_cache_dir
_raise_on_head_call_error(head_call_error, force_download, local_files_only)
~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Lib\site-packages\huggingface_hub\file_download.py", line 1648, in _raise_on_head_call_error
raise LocalEntryNotFoundError(
...<3 lines>...
) from head_call_error
huggingface_hub.errors.LocalEntryNotFoundError: An error happened while trying to locate the file on the Hub and we cannot find the requested files in the local cache. Please check your connection and try again or make sure your Internet connection is on.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "D:\Lib\site-packages\gradio\queueing.py", line 625, in process_events
response = await route_utils.call_process_api(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...<5 lines>...
)
^
File "D:\Lib\site-packages\gradio\route_utils.py", line 322, in call_process_api
output = await app.get_blocks().process_api(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
...<11 lines>...
)
^
File "D:\Lib\site-packages\gradio\blocks.py", line 2193, in process_api
result = await self.call_function(
^^^^^^^^^^^^^^^^^^^^^^^^^
...<8 lines>...
)
^
File "D:\Lib\site-packages\gradio\blocks.py", line 1704, in call_function
prediction = await anyio.to_thread.run_sync( # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
fn, *processed_input, limiter=self.limiter
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "D:\Lib\site-packages\anyio\to_thread.py", line 56, in run_sync
return await get_async_backend().run_sync_in_worker_thread(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
func, args, abandon_on_cancel=abandon_on_cancel, limiter=limiter
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
)
^
File "D:\Lib\site-packages\anyio\_backends\_asyncio.py", line 2470, in run_sync_in_worker_thread
return await future
^^^^^^^^^^^^
File "D:\Lib\site-packages\anyio\_backends\_asyncio.py", line 967, in run
result = context.run(func, *args)
File "D:\Lib\site-packages\gradio\utils.py", line 894, in wrapper
response = f(*args, **kwargs)
File "C:\Users\Y\Desktop\python\101.py", line 109, in fine_tune_model
tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
File "D:\Lib\site-packages\transformers\models\auto\tokenization_auto.py", line 970, in from_pretrained
config = AutoConfig.from_pretrained(
pretrained_model_name_or_path, trust_remote_code=trust_remote_code, **kwargs
)
File "D:\Lib\site-packages\transformers\models\auto\configuration_auto.py", line 1153, in from_pretrained
config_dict, unused_kwargs = PretrainedConfig.get_config_dict(pretrained_model_name_or_path, **kwargs)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Lib\site-packages\transformers\configuration_utils.py", line 595, in get_config_dict
config_dict, kwargs = cls._get_config_dict(pretrained_model_name_or_path, **kwargs)
~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "D:\Lib\site-packages\transformers\configuration_utils.py", line 654, in _get_config_dict
resolved_config_file = cached_file(
pretrained_model_name_or_path,
...<10 lines>...
_commit_hash=commit_hash,
)
File "D:\Lib\site-packages\transformers\utils\hub.py", line 312, in cached_file
file = cached_files(path_or_repo_id=path_or_repo_id, filenames=[filename], **kwargs)
File "D:\Lib\site-packages\transformers\utils\hub.py", line 543, in cached_files
raise OSError(
...<3 lines>...
) from e
OSError: We couldn't connect to 'https://huggingface.co' to load the files, and couldn't find them in the cached files.
Checkout your internet connection or see how to run the library in offline mode at 'https://huggingface.co/docs/transformers/installation#offline-mode'.
修改并给出完整代码
博客主要介绍了函数模块的异常处理机制。系统默认有捕捉异常机制，后台会自行处理。使用EXCEPTIONS可将返回值分配给非基于类的异常，不同消息类型后台处理不同。还提及了不同版本下函数模块对基于类异常的处理，以及消息处理的相关注意事项。
扫一扫
1060
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
