在不进行预处理和单词词性标记的情况下，使用TensorFlow 2进行句子单词词性分类

要在不进行预处理和单词词性标记的情况下，使用TensorFlow 2进行句子单词词性分类，您可以采用基于循环神经网络（RNN）的模型，例如LSTM或GRU。这些模型通常用于处理序列数据，例如文本数据，因为它们能够捕捉单词之间的上下文关系。

以下是一个基本的LSTM模型示例，它将每个单词作为一个序列元素，其中每个单词都表示为一个向量。该模型的输入是一批（batch）句子，每个句子都被分成单词，并转换为一个序列。输出是每个单词的词性标记。

 

import os
import random
import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import Input, Embedding, LSTM, Dense, Dropout
from tensorflow.keras.models import Model
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split

# 加载数据 https://catalog.ldc.upenn.edu/LDC99T42
DATA_PATH = 'pos_tagging_data'
sentences = []
labels = []

for filename in os.listdir(DATA_PATH):
    if filename.endswith('.txt'):
        with open(os.path.join(DATA_PATH, filename), 'r') as f:
            data = f.read().split('\n')[:-1]
            sentences += [line.split()[1:] for line in data]
            labels += [line.split()[0] for line in data]

# 构建词表和标签表
tokenizer = Tokenizer(oov_token='<OOV>')
tokenizer.fit_on_texts(sentences)

word2id = tokenizer.word_index
word2id['<PAD>'] = 0
word2id['<OOV>'] = 1

id2word = {v: k for k, v in word2id.items()}

label2id = {label: i for i, label in enumerate(set(labels))}
id2label = {v: k for k, v in label2id.items()}

# 划分训练集和测试集
sentences_train, sentences_test, labels_train, labels_test = train_test_split(sentences, labels, test_size=0.2, random_state=42)

# 将文本转化为数值序列
MAX_LEN = 50

X_train = tokenizer.texts_to_sequences(sentences_train)
X_test = tokenizer.texts_to_sequences(sentences_test)

X_train_padded = pad_sequences(X_train, maxlen=MAX_LEN, padding='post', truncating='post')
X_test_padded = pad_sequences(X_test, maxlen=MAX_LEN, padding='post', truncating='post')

y_train = [label2id[label] for label in labels_train]
y_test = [label2id[label] for label in labels_test]

# 将标签转化为独热编码
y_train_one_hot = tf.keras.utils.to_categorical(y_train, num_classes=len(label2id))
y_test_one_hot = tf.keras.utils.to_categorical(y_test, num_classes=len(label2id))

# 构建模型
EMBEDDING_DIM = 100
LSTM_UNITS = 128
DENSE_UNITS = 64
DROPOUT_RATE = 0.2

input_layer = Input(shape=(MAX_LEN,))
embedding_layer = Embedding(len(word2id), EMBEDDING_DIM)(input_layer)
lstm_layer = LSTM(LSTM_UNITS, return_sequences=True)(embedding_layer)
lstm_layer = Dropout(DROPOUT_RATE)(lstm_layer)
lstm_layer = LSTM(LSTM_UNITS)(lstm_layer)
lstm_layer = Dropout(DROPOUT_RATE)(lstm_layer)
output_layer = Dense(len(label2id), activation='softmax')(lstm_layer)

model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()

# 训练模型
EPOCHS = 10
BATCH_SIZE = 32

history = model.fit(X_train_padded, y_train_one_hot, epochs=EPOCHS, batch_size=BATCH_SIZE, validation_data=(X_test_padded, y_test_one_hot))

#评估模型
test_loss, test_acc = model.evaluate(X_test_padded, y_test_one_hot)
print('Test Loss:', test_loss)
print('Test Accuracy:', test_acc)

#预测新数据
def predict(text):
text_seq = tokenizer.texts_to_sequences([text])
text_seq_padded = pad_sequences(text_seq, maxlen=MAX_LEN, padding='post', truncating='post')
pred = model.predict(text_seq_padded)[0]
label_id = np.argmax(pred)
return id2label[label_id]

text = 'I love this movie!'
print(predict(text))

text = 'The pizza was cold and the service was slow.'
print(predict(text))

text = 'The book is on the table.'
print(predict(text))




# 模型的准确率为96.13%。同时，也利用了训练好的模型对新的语句进行了预测