BIlstm训练中文

最新推荐文章于 2025-05-14 09:19:56 发布

原创

最新推荐文章于 2025-05-14 09:19:56 发布 · 491 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#pytorch

本文档详细介绍了如何运用PyTorch框架，构建双向LSTM（Bi-LSTM）模型来处理和理解中文文本数据。通过实例演示了数据预处理、模型构建、训练过程以及评估指标，帮助读者掌握Bi-LSTM在中文自然语言处理中的应用。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

from torch.utils.data import DataLoader
import os
from torch.utils import data
import pandas as pd
import torch
import numpy as np
import re
from tqdm import tqdm
import pickle
from torch import nn
import torch.nn.functional as F
from torch import optim

max_len = 250
train_batch_size = 64
test_batch_size = 500
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class ESISADataset(data.Dataset):
    def __init__(self, csv_file):
        self.data = pd.read_csv(csv_file)

    def __len__(self):
        return self.data.shape[0]

    def __getitem__(self, idx):
        item = self.data.iloc[idx]
        text = item["text"]
        label = item["label"]

        return text, label

# 读取数据
train_dataset = ESISADataset("data/SMP2019_ECISA/train.csv")
test_dataset = ESISADataset("data/SMP2019_ECISA/test.csv")
dev_dataset = ESISADataset("data/SMP2019_ECISA/dev.csv")

def tokenize(text):

    fileters = ['!', '"', '#', '$', '%', '&', '\(', '\)', '\*', '\+', ',', '-', '\.', '/', ':', ';', '<', '=', '>',
                '\?', '@', '\[', '\\', '\]', '^', '_', '`', '\{', '\|', '\}', '~', '\t', '\n', '\x97', '\x96', '”',
                '“', ]
    text = re.sub("<.*?>", " ", text, flags=re.S)
    text = re.sub("|".join(fileters), " ", text, flags=re.S)
    return [i.strip() for i in text.split()]

"""文本序列化"""
# Word2Sequence
class Word2Sequence:
    # 未出现过的词
    UNK_TAG = "UNK"
    PAD_TAG = "PAD"
    # 填充的词
    UNK = 0
    PAD = 1

    def __init__(self):
        self.dict = {
            self.UNK_TAG: self.UNK,
            self.PAD_TAG: self.PAD
        }
        self.fited