一.基于Logistic回归的二分类任务
数据来自带噪音的两个弯月形状函数，每个弯月对一个类别。采集1000条样本，每个样本包含2个特征。随机采集1000个样本，并进行可视化。将1000条样本数据拆分成训练集、验证集和测试集，其中训练集640条、验证集160条、测试集200条。
（依据《神经网络与深度学习：案例与实践》这本书来写，大部分代码源自于书上）
import math
import copy
import torch
import matplotlib.pyplot as plt
from abc import abstractmethod
# 固定随机种子，保持每次运行结果一致
torch.seed()
# 生成数据集
def make_moons(n_samples=1000, shuffle=True, noise=None):
"""生成带噪音的弯月形状数据
输入：n_samples：数据量大小，数据类型为int shuffle：是否打乱数据，数据类型为bool noise：以多大的程度增加噪声，数据类型为None或float，noise为None时表示不增加噪声
输出：X：特征数据，shape=[n_samples,2] y：标签数据, shape=[n_samples]
"""
n_samples_out = n_samples // 2
n_samples_in = n_samples - n_samples_out
# 采集第1类数据，特征为(x,y)
# 使用'torch.linspace'在0到pi上均匀取n_samples_out个值
# 使用'torch.cos'计算上述取值的余弦值作为特征1，使用'torch.sin'计算上述取值的正弦值作为特征2
outer_circ_x = torch.cos(torch.linspace(0, math.pi, n_samples_out))
outer_circ_y = torch.sin(torch.linspace(0, math.pi, n_samples_out))
inner_circ_x = 1 - torch.cos(torch.linspace(0, math.pi, n_samples_in))
inner_circ_y = 0.5 - torch.sin(torch.linspace(0, math.pi, n_samples_in))
print('outer_circ_x.shape:', outer_circ_x.shape, 'outer_circ_y.shape:', outer_circ_y.shape)
print('inner_circ_x.shape:', inner_circ_x.shape, 'inner_circ_y.shape:', inner_circ_y.shape)
# 使用'torch.concat'将两类数据的特征1和特征2分别延维度0拼接在一起，得到全部特征1和特征2
# 使用'torch.stack'将两类特征延维度1堆叠在一起
X = torch.stack(
[torch.concat([outer_circ_x, inner_circ_x]),
torch.concat([outer_circ_y, inner_circ_y])],
dim=1 # 确定在哪个维度拼接
)
print('after concat shape:', torch.concat([outer_circ_x, inner_circ_x]).shape)
print('X shape:', X.shape)
# 使用'paddle. zeros'将第一类数据的标签全部设置为0
# 使用'paddle. ones'将第一类数据的标签全部设置为1
y = torch.concat(
[torch.zeros([n_samples_out]), torch.ones([n_samples_in])]
)
print('y shape:', y.shape)
# 如果shuffle为True，将所有数据打乱
if shuffle:
# 使用'torch.randperm'生成一个数值在0到X.shape[0]，随机排列的一维Tensor做索引值，用于打乱数据
idx = torch.randperm(X.shape[0])
X = X[idx]
y = y[idx]
# 如果noise不为None，则给特征值加入噪声
if noise is not None:
# 使用'torch.normal'生成符合正态分布的随机Tensor作为噪声，并加到原始特征上
X += torch.normal(mean=torch.zeros_like(X), std=noise * torch.ones_like(X))
return X, y
# 激活函数
def logistic(x):
return 1 / (1 + torch.exp(-x))
def decision_boundary(w, b, x1):
w1, w2 = w
x2 = (- w1 * x1 - b) / w2
return x2
class Op(object):
def __init__(self):
pass
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
raise NotImplementedError
def backward(self, inputs):
raise NotImplementedError
class model_LR(Op):
def __init__(self, input_dim):
super(model_LR, self).__init__()
# 存放线性层参数
self.params = {}
# 将线性层的权重参数全部初始化为0
self.params['w'] = torch.zeros(size=[input_dim, 1])
self.params['w'] = torch.normal(mean=0, std=0.01, size=[input_dim, 1])# 初始化权重的值服从均值为 0，标准差为 0.01 的正态分布
# 将线性层的偏置参数初始化为0
self.params['b'] = torch.zeros(size=[1])
# 存放参数的梯度
self.grads = {}
self.X = None
self.outputs = None
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
self.X = inputs
# 线性计算
score = torch.matmul(inputs, self.params['w']) + self.params['b']
# Logistic 函数
self.outputs = logistic(score)
return self.outputs
def backward(self, labels):
"""
输入：labels：真实标签，shape=[N, 1]
"""
N = labels.shape[0]
# 计算偏导数
self.grads['w'] = -1 / N * torch.matmul(self.X.t(), (labels - self.outputs))
self.grads['b'] = -1 / N * torch.sum(labels - self.outputs)
class BinaryCrossEntropyLoss(torch.nn.Module):
def __init__(self):
self.predicts = None
self.labels = None
self.num = None
def __call__(self, predicts, labels):
return self.forward(predicts, labels)
def forward(self, predicts, labels):
"""
输入：predicts：预测值，shape=[N, 1]，N为样本数量
labels：真实标签，shape=[N, 1]
输出：损失值：shape=[1]
"""
self.predicts = predicts
self.labels = labels
self.num = self.predicts.shape[0]
loss = -1. / self.num * ( torch.matmul(self.labels.t(), torch.log(self.predicts)) + torch.matmul((1 - self.labels.t()),torch.log(1 - self.predicts)))
loss = torch.squeeze(loss, 1)
return loss
class model_LR(Op):
def __init__(self, input_dim):
super(model_LR, self).__init__()
# 存放线性层参数
self.params = {}
# 将线性层的权重参数全部初始化为0
self.params['w'] = torch.zeros(size=[input_dim, 1])
# self.params['w'] = torch.normal(mean=0, std=0.01, shape=[input_dim, 1])
# 将线性层的偏置参数初始化为0
self.params['b'] = torch.zeros(size=[1])
# 存放参数的梯度
self.grads = {}
self.X = None
self.outputs = None
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
self.X = inputs
# 线性计算
score = torch.matmul(inputs, self.params['w']) + self.params['b']
# Logistic 函数
self.outputs = logistic(score)
return self.outputs
def backward(self, labels):
"""
输入： labels：真实标签，shape=[N, 1]
"""
N = labels.shape[0]
# 计算偏导数
self.grads['w'] = -1 / N * torch.matmul(self.X.t(), (labels - self.outputs))
self.grads['b'] = -1 / N * torch.sum(labels - self.outputs)
# 优化器基类
class Optimizer(object):
def __init__(self, init_lr, model):
"""优化器类初始化"""
# 初始化学习率，用于参数更新的计算
self.init_lr = init_lr
# 指定优化器需要优化的模型
self.model = model
@abstractmethod
def step(self):
"""定义每次迭代如何更新参数"""
pass
class SimpleBatchGD(Optimizer):
def __init__(self, init_lr, model):
super(SimpleBatchGD, self).__init__(init_lr=init_lr, model=model)
def step(self):
# 参数更新
# 遍历所有参数
if isinstance(self.model.params, dict):
for key in self.model.params.keys():
self.model.params[key] = self.model.params[key] - self.init_lr * self.model.grads[key]
def accuracy(preds, labels):
"""
输入： preds：预测值，二分类时，shape=[N, 1]，N为样本数量，多分类时，shape=[N, C]，C为类别数量
labels：真实标签，shape=[N, 1]
输出：准确率：shape=[1]
"""
# 判断是二分类任务还是多分类任务，preds.shape[1]=1时为二分类任务，preds.shape[1]>1时为多分类任务
if preds.shape[1] == 1:
# 二分类时，判断每个概率值是否大于0.5，当大于0.5时，类别为1，否则类别为0
# 使用'torch.cast'将preds的数据类型转换为float32类型
preds = (preds >= 0.5).float()
else:
# 多分类时，使用'torch.argmax'计算最大元素索引作为类别
preds = torch.argmax(preds, 1)
accuracy = torch.mean((preds == labels).float())
return accuracy
# 用RunnerV2类封装整个训练过程，最速下降法
class RunnerV2(object):
def __init__(self, model, optimizer, metric, loss_fn):
self.model = model
self.optimizer = optimizer
self.loss_fn = loss_fn
self.metric = metric
# 记录训练过程中的评价指标变化情况
self.train_scores = []
self.dev_scores = []
# 记录训练过程中的损失函数变化情况
self.train_loss = []
self.dev_loss = []
def train(self, train_set, dev_set, **kwargs):
# 传入训练轮数，如果没有传入值则默认为0
num_epochs = kwargs.get("num_epochs", 0)
# 传入log打印频率，如果没有传入值则默认为100
log_epochs = kwargs.get("log_epochs", 100)
# 传入模型保存路径，如果没有传入值则默认为"best_model.pdparams"
save_path = kwargs.get("save_path", "best_model.pdparams")
# 梯度打印函数，如果没有传入则默认为"None"
print_grads = kwargs.get("print_grads", None)
# 记录全局最优指标
best_score = 0
# 进行num_epochs轮训练
for epoch in range(num_epochs):
X, y = train_set
# 获取模型预测
logits = self.model(X)
# 计算交叉熵损失
trn_loss = self.loss_fn(logits, y).item()
self.train_loss.append(trn_loss)
# 计算评价指标
trn_score = self.metric(logits, y).item()
self.train_scores.append(trn_score)
# 计算参数梯度
self.model.backward(y)
if print_grads is not None:
# 打印每一层的梯度
print_grads(self.model)
# 更新模型参数
self.optimizer.step()
dev_score, dev_loss = self.evaluate(dev_set)
# 如果当前指标为最优指标，保存该模型
if dev_score > best_score:
self.save_model(save_path)
print(f"best accuracy performence has been updated: {best_score:.5f} --> {dev_score:.5f}")
best_score = dev_score
if epoch % log_epochs == 0:
print(f"[Train] epoch: {epoch}, loss: {trn_loss}, score: {trn_score}")
print(f"[Dev] epoch: {epoch}, loss: {dev_loss}, score: {dev_score}")
def evaluate(self, data_set):
X, y = data_set
# 计算模型输出
logits = self.model(X)
# 计算损失函数
loss = self.loss_fn(logits, y).item()
self.dev_loss.append(loss)
# 计算评价指标
score = self.metric(logits, y).item()
self.dev_scores.append(score)
return score, loss
def predict(self, X):
return self.model(X)
def save_model(self, save_path):
torch.save(self.model.params, save_path)
def load_model(self, model_path):
self.model.params = torch.load(model_path)
# 可视化观察训练集与验证集的指标变化情况
def plot(runner, fig_name):
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
epochs = [i for i in range(len(runner.train_scores))]
# 绘制训练损失变化曲线
plt.plot(epochs, runner.train_loss, color='#e4007f', label="Train loss")
# 绘制评价损失变化曲线
plt.plot(epochs, runner.dev_loss, color='#f19ec2', linestyle='--', label="Dev loss")
# 绘制坐标轴和图例
plt.ylabel("loss", fontsize='large')
plt.xlabel("epoch", fontsize='large')
plt.legend(loc='upper right', fontsize='x-large')
plt.subplot(1, 2, 2)
# 绘制训练准确率变化曲线
plt.plot(epochs, runner.train_scores, color='#e4007f', label="Train accuracy")
# 绘制评价准确率变化曲线
plt.plot(epochs, runner.dev_scores, color='#f19ec2', linestyle='--', label="Dev accuracy")
# 绘制坐标轴和图例
plt.ylabel("score", fontsize='large')
plt.xlabel("epoch", fontsize='large')
plt.legend(loc='lower right', fontsize='x-large')
plt.tight_layout()
plt.savefig(fig_name)
plt.show()
# 在[-10,10]的范围内生成一系列的输入值，用于绘制函数曲线
x = torch.linspace(-10, 10, 10000)
plt.figure()
plt.plot(x.tolist(), logistic(x).tolist(), color="#E20079", label="Logistic Function")
# 设置坐标轴
ax = plt.gca()
# 取消右侧和上侧坐标轴
ax.spines['top'].set_color('none')
ax.spines['right'].set_color('none')
# 设置默认的x轴和y轴方向
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')
# 设置坐标原点为(0,0)
ax.spines['left'].set_position(('data', 0))
ax.spines['bottom'].set_position(('data', 0))
# 添加图例
plt.legend()
plt.savefig('linear-logistic.pdf')
plt.show()
n_samples = 1000
X, y = make_moons(n_samples=n_samples, shuffle=True, noise=0.18)
# 可视化生产的数据集，不同颜色代表不同类别
plt.figure(figsize=(5, 5))
plt.scatter(x=X[:, 0].tolist(), y=X[:, 1].tolist(), marker='*', c=y.tolist())
plt.xlim(-3, 4)
plt.ylim(-3, 4)
plt.savefig('linear-dataset-vis.pdf')
plt.show()
# 划分训练集测试集验证集
num_train = 640
num_dev = 160
num_test = 200
X_train, y_train = X[:num_train], y[:num_train]
X_dev, y_dev = X[num_train:num_train + num_dev], y[num_train:num_train + num_dev]
X_test, y_test = X[num_train + num_dev:], y[num_train + num_dev:]
y_train = y_train.reshape([-1, 1])
y_dev = y_dev.reshape([-1, 1])
y_test = y_test.reshape([-1, 1])
# 随机生成3条长度为4的数据
inputs = torch.randn([3, 4])
print('Input is:', inputs)
# 实例化模型
model = model_LR(4)
outputs = model(inputs)
print('Output is:', outputs)
labels = torch.ones(3, 1)
# 计算风险函数
bce_loss = BinaryCrossEntropyLoss()
print(bce_loss(outputs, labels))
# 假设模型的预测值为[[0.],[1.],[1.],[0.]]，真实类别为[[1.],[1.],[0.],[0.]]，计算准确率
preds = torch.tensor([[0.], [1.], [1.], [0.]])
labels = torch.tensor([[1.], [1.], [0.], [0.]])
print("accuracy is:", accuracy(preds, labels))
# 特征维度
input_dim = 2
# 学习率
lr = 0.01
# 实例化模型
model = model_LR(input_dim=input_dim)
# 指定优化器
optimizer = SimpleBatchGD(init_lr=lr, model=model)
# 指定损失函数
loss_fn = BinaryCrossEntropyLoss()
# 指定评价方式
metric = accuracy
# 实例化RunnerV2类，并传入训练配置
runner = RunnerV2(model, optimizer, metric, loss_fn)
# 训练并绘图
runner.train([X_train, y_train], [X_dev, y_dev], num_epochs=500, log_epochs=50, save_path="best_model.pdparams")
plot(runner, fig_name='linear-acc.pdf')
score, loss = runner.evaluate([X_test, y_test])
print("[Test] score/loss: {:.4f}/{:.4f}".format(score, loss))
plt.figure(figsize=(5,5))
# 绘制原始数据
plt.scatter(X[:, 0].tolist(), X[:, 1].tolist(), marker='*', c=y.tolist())
w = model.params['w']
b = model.params['b']
x1 = torch.linspace(-2, 3, 1000)
x2 = decision_boundary(w, b, x1)
# 绘制决策边界
plt.plot(x1.tolist(), x2.tolist(), color="red")
plt.show()1.查看生成的数据集，并进行可视化
2.损失函数图像，与线性回归不同的是，Logistic回归引入了非线性函数，可以解决连续的线性函数不适合进行分类的问题。
3.Logistic回归模型其实就是线性层与Logistic函数的组合，通常会将 Logistic回归模型中的权重和偏置初始化为0，构建Logistic回归算子并进行简单测试。
4.损失函数，用损失函数来量化预测值和真实值之间的差异，交叉熵损失函数。 在给定y的情况下，如果预测的概率分布y^与标签真实的分布yy越接近，则交叉熵越小；如果p(x)和y越远，交叉熵就越大。
5.用梯度下降法进行模型优化，并进行训练（此学习率设置为0.01）
将学习率改为0.001和0.1，再次观察结果
0.001
0.1
可以观察得出学习率为0.1较为合适。
6.模型评价，观察准确率即正确预测的数量与总的预测数量的比值
7.使用测试集对训练完成后的最终模型进行测试
二.基于Softmax回归的多分类任务，数据来自3个不同的簇，每个簇对一个类别。采集1000条样本，每个样本包含2个特征。（与二分类有许多类似之处）
from abc import abstractmethod
import torch
import matplotlib.pyplot as plt
import numpy as np
torch.seed()
def make_multiclass_classification(n_samples=100, n_features=2, n_classes=3, shuffle=True, noise=0.1):
"""
生成带噪音的多类别数据
输入：n_samples：数据量大小，数据类型为int
n_features：特征数量，数据类型为int
shuffle：是否打乱数据，数据类型为bool
noise：以多大的程度增加噪声，数据类型为None或float，noise为None时表示不增加噪声
输出：X：特征数据，shape=[n_samples,2]
y：标签数据, shape=[n_samples,1]
"""
# 计算每个类别的样本数量
n_samples_per_class = [int(n_samples / n_classes) for k in range(n_classes)]
for i in range(n_samples - sum(n_samples_per_class)):
n_samples_per_class[i % n_classes] += 1
# 将特征和标签初始化为0
X = torch.zeros([n_samples, n_features])
y = torch.zeros([n_samples], dtype=torch.int32)
# 随机生成3个簇中心作为类别中心
centroids = torch.randperm(2 ** n_features)[:n_classes]
centroids_bin = np.unpackbits(centroids.numpy().astype('uint8')).reshape((-1, 8))[:, -n_features:]
centroids = torch.tensor(centroids_bin, dtype=torch.float32)
# 控制簇中心的分离程度
centroids = 1.5 * centroids - 1
# 随机生成特征值
X[:, :n_features] = torch.randn([n_samples, n_features])
stop = 0
# 将每个类的特征值控制在簇中心附近
for k, centroid in enumerate(centroids):
start, stop = stop, stop + n_samples_per_class[k]
# 指定标签值
y[start:stop] = k % n_classes
X_k = X[start:stop, :n_features]
# 控制每个类别特征值的分散程度
A = 2 * torch.rand(size=[n_features, n_features]) - 1
X_k[...] = torch.matmul(X_k, A)
X_k += centroid
X[start:stop, :n_features] = X_k
# 如果noise不为None，则给特征加入噪声
if noise > 0.0:
# 生成noise掩膜，用来指定给那些样本加入噪声
noise_mask = torch.rand([n_samples]) < noise
for i in range(len(noise_mask)):
if noise_mask[i]:
# 给加噪声的样本随机赋标签值
y[i] = torch.randint(n_classes, (1,)).item() # 如果shuffle为True，将所有数据打乱
if shuffle:
idx = torch.randperm(X.shape[0])
X = X[idx]
y = y[idx]
return X, y
def softmax(X):
"""
输入：X：shape=[N, C]，N为向量数量，C为向量维度
"""
x_max = torch.max(X, dim=1, keepdim=True).values # N, 1
x_exp = torch.exp(X - x_max)
partition = torch.sum(x_exp, dim=1, keepdim=True) # N, 1
return x_exp / partition
class Op(object):
def __init__(self):
pass
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
raise NotImplementedError
def backward(self, inputs):
raise NotImplementedError
class model_SR(Op):
def __init__(self, input_dim, output_dim):
super(model_SR, self).__init__()
self.params = {}
# 将线性层的权重参数全部初始化为0
self.params['W'] = torch.zeros([input_dim, output_dim])
self.params['W'] = torch.normal(mean=0, std=0.01, size=[input_dim, output_dim])
# 将线性层的偏置参数初始化为0
self.params['b'] = torch.zeros([output_dim])
self.outputs = None
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
"""
输入：inputs: shape=[N,D], N是样本数量，D是特征维度
输出：outputs：预测值，shape=[N,C]，C是类别数
"""
# 线性计算
score = torch.matmul(inputs, self.params['W']) + self.params['b']
# Softmax 函数
self.outputs = softmax(score)
return self.outputs
class MultiCrossEntropyLoss(Op):
def __init__(self):
self.predicts = None
self.labels = None
self.num = None
def __call__(self, predicts, labels):
return self.forward(predicts, labels)
def forward(self, predicts, labels):
"""
输入：predicts：预测值，shape=[N, 1]，N为样本数量
labels：真实标签，shape=[N, 1]
输出：损失值：shape=[1]
"""
self.predicts = predicts
self.labels = labels
self.num = self.predicts.shape[0]
loss = 0
for i in range(0, self.num):
index = self.labels[i]
loss -= torch.log(self.predicts[i][index])
return loss / self.num
class model_SR(Op):
def __init__(self, input_dim, output_dim):
super(model_SR, self).__init__()
self.params = {}
# 将线性层的权重参数全部初始化为0
self.params['W'] = torch.zeros(input_dim, output_dim)
# self.params['W'] = paddle.normal(mean=0, std=0.01, shape=[input_dim, output_dim])
# 将线性层的偏置参数初始化为0
self.params['b'] = torch.zeros(output_dim)
# 存放参数的梯度
self.grads = {}
self.X = None
self.outputs = None
self.output_dim = output_dim
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
self.X = inputs
# 线性计算
score = torch.matmul(self.X, self.params['W']) + self.params['b']
# Softmax 函数
self.outputs = softmax(score)
return self.outputs
def backward(self, labels):
"""
输入：labels：真实标签，shape=[N, 1]，其中N为样本数量
"""
# 计算偏导数
N =labels.shape[0]
labels = torch.nn.functional.one_hot(labels.to(torch.int64), self.output_dim)
self.grads['W'] = -1 / N * torch.matmul(self.X.t(), (labels-self.outputs))
self.grads['b'] = -1 / N * torch.matmul(torch.ones(N), (labels-self.outputs))
class Optimizer(object):
def __init__(self, init_lr, model):
"""优化器类初始化"""
# 初始化学习率，用于参数更新的计算
self.init_lr = init_lr
# 指定优化器需要优化的模型
self.model = model
@abstractmethod
def step(self):
"""定义每次迭代如何更新参数"""
pass
class SimpleBatchGD(Optimizer):
def __init__(self, init_lr, model):
super(SimpleBatchGD, self).__init__(init_lr=init_lr, model=model)
def step(self):
# 参数更新
# 遍历所有参数
if isinstance(self.model.params, dict):
for key in self.model.params.keys():
self.model.params[key] = self.model.params[key] - self.init_lr * self.model.grads[key]
def accuracy(preds, labels):
"""
输入： preds：预测值，二分类时，shape=[N, 1]，N为样本数量，多分类时，shape=[N, C]，C为类别数量
labels：真实标签，shape=[N, 1]
输出：准确率：shape=[1]
"""
# 判断是二分类任务还是多分类任务，preds.shape[1]=1时为二分类任务，preds.shape[1]>1时为多分类任务
if preds.shape[1] == 1:
# 二分类时，判断每个概率值是否大于0.5，当大于0.5时，类别为1，否则类别为0
# 使用'torch.cast'将preds的数据类型转换为float32类型
preds = (preds >= 0.5).float()
else:
# 多分类时，使用'torch.argmax'计算最大元素索引作为类别
preds = torch.argmax(preds, 1)
accuracy = torch.mean((preds == labels).float())
return accuracy
class RunnerV2(object):
def __init__(self, model, optimizer, metric, loss_fn):
self.model = model
self.optimizer = optimizer
self.loss_fn = loss_fn
self.metric = metric
# 记录训练过程中的评价指标变化情况
self.train_scores = []
self.dev_scores = []
# 记录训练过程中的损失函数变化情况
self.train_loss = []
self.dev_loss = []
def train(self, train_set, dev_set, **kwargs):
# 传入训练轮数，如果没有传入值则默认为0
num_epochs = kwargs.get("num_epochs", 0)
# 传入log打印频率，如果没有传入值则默认为100
log_epochs = kwargs.get("log_epochs", 100)
# 传入模型保存路径，如果没有传入值则默认为"best_model.pdparams"
save_path = kwargs.get("save_path", "best_model.pdparams")
# 梯度打印函数，如果没有传入则默认为"None"
print_grads = kwargs.get("print_grads", None)
# 记录全局最优指标
best_score = 0
# 进行num_epochs轮训练
for epoch in range(num_epochs):
X, y = train_set
# 获取模型预测
logits = self.model(X)
# 计算交叉熵损失
trn_loss = self.loss_fn(logits, y).item()
self.train_loss.append(trn_loss)
# 计算评价指标
trn_score = self.metric(logits, y).item()
self.train_scores.append(trn_score)
# 计算参数梯度
self.model.backward(y)
if print_grads is not None:
# 打印每一层的梯度
print_grads(self.model)
# 更新模型参数
self.optimizer.step()
dev_score, dev_loss = self.evaluate(dev_set)
# 如果当前指标为最优指标，保存该模型
if dev_score > best_score:
self.save_model(save_path)
print(f"best accuracy performence has been updated: {best_score:.5f} --> {dev_score:.5f}")
best_score = dev_score
if epoch % log_epochs == 0:
print(f"[Train] epoch: {epoch}, loss: {trn_loss}, score: {trn_score}")
print(f"[Dev] epoch: {epoch}, loss: {dev_loss}, score: {dev_score}")
def evaluate(self, data_set):
X, y = data_set
# 计算模型输出
logits = self.model(X)
# 计算损失函数
loss = self.loss_fn(logits, y).item()
self.dev_loss.append(loss)
# 计算评价指标
score = self.metric(logits, y).item()
self.dev_scores.append(score)
return score, loss
def predict(self, X):
return self.model(X)
def save_model(self, save_path):
torch.save(self.model.params, save_path)
def load_model(self, model_path):
self.model.params = torch.load(model_path)
# 可视化观察训练集与验证集的指标变化情况
def plot(runner, fig_name):
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
epochs = [i for i in range(len(runner.train_scores))]
# 绘制训练损失变化曲线
plt.plot(epochs, runner.train_loss, color='#e4007f', label="Train loss")
# 绘制评价损失变化曲线
plt.plot(epochs, runner.dev_loss, color='#f19ec2', linestyle='--', label="Dev loss")
# 绘制坐标轴和图例
plt.ylabel("loss", fontsize='large')
plt.xlabel("epoch", fontsize='large')
plt.legend(loc='upper right', fontsize='x-large')
plt.subplot(1, 2, 2)
# 绘制训练准确率变化曲线
plt.plot(epochs, runner.train_scores, color='#e4007f', label="Train accuracy")
# 绘制评价准确率变化曲线
plt.plot(epochs, runner.dev_scores, color='#f19ec2', linestyle='--', label="Dev accuracy")
# 绘制坐标轴和图例
plt.ylabel("score", fontsize='large')
plt.xlabel("epoch", fontsize='large')
plt.legend(loc='lower right', fontsize='x-large')
plt.tight_layout()
plt.savefig(fig_name)
plt.show()
# 采样1000个样本
n_samples = 1000
X, y = make_multiclass_classification(n_samples=n_samples, n_features=2, n_classes=3, noise=0.2)
# 可视化生产的数据集，不同颜色代表不同类别
plt.figure(figsize=(5,5))
plt.scatter(x=X[:, 0].tolist(), y=X[:, 1].tolist(), marker='*', c=y.tolist())
plt.savefig('linear-dataset-vis2.pdf')
plt.show()
# 划分训练集测试集验证集
num_train = 640
num_dev = 160
num_test = 200
X_train, y_train = X[:num_train], y[:num_train]
X_dev, y_dev = X[num_train:num_train + num_dev], y[num_train:num_train + num_dev]
X_test, y_test = X[num_train + num_dev:], y[num_train + num_dev:]
# 打印X_train和y_train的维度
print("X_train shape: ", X_train.shape, "y_train shape: ", y_train.shape)
# 打印前5个数据的标签
print(y_train[:5])
# 观察softmax的计算方式
X = torch.tensor([[0.1, 0.2, 0.3, 0.4],[1,2,3,4]])
predict = softmax(X)
print(predict)
# 随机生成1条长度为4的数据
inputs = torch.randn(size=[1,4])
print('Input is:', inputs)
# 实例化模型，这里令输入长度为4，输出类别数为3
model = model_SR(input_dim=4, output_dim=3)
outputs = model(inputs)
print('Output is:', outputs)
# 测试
labels = torch.tensor([0])
# 计算风险函数
mce_loss = MultiCrossEntropyLoss()
print(mce_loss(outputs, labels))
# 特征维度
input_dim = 2
# 类别数
output_dim = 3
# 学习率
lr = 0.1
# 实例化模型
model = model_SR(input_dim=input_dim, output_dim=output_dim)
# 指定优化器
optimizer = SimpleBatchGD(init_lr=lr, model=model)
# 指定损失函数
loss_fn = MultiCrossEntropyLoss()
# 指定评价方式
metric = accuracy
# 实例化RunnerV2类
runner = RunnerV2(model, optimizer, metric, loss_fn)
# 模型训练
runner.train([X_train, y_train], [X_dev, y_dev], num_epochs=500, log_eopchs=50, eval_epochs=1, save_path="best_model.pdparams")
# 可视化观察训练集与验证集的准确率变化情况
plot(runner,fig_name='linear-acc2.pdf')
score, loss = runner.evaluate([X_test, y_test])
print("[Test] score/loss: {:.4f}/{:.4f}".format(score, loss))
# 均匀生成40000个数据点
x1, x2 = torch.meshgrid(torch.linspace(-3.5, 2, 200), torch.linspace(-4.5, 3.5, 200),indexing = 'ij')
x = torch.stack([torch.flatten(x1), torch.flatten(x2)], 1)
# 预测对应类别
y = runner.predict(x)
y = torch.argmax(y, 1)
# 绘制类别区域
plt.ylabel('x2')
plt.xlabel('x1')
plt.scatter(x[:,0].tolist(), x[:,1].tolist(), c=y.tolist(), cmap=plt.cm.Spectral)
n_samples = 1000
X, y = make_multiclass_classification(n_samples=n_samples, n_features=2, n_classes=3, noise=0.2)
plt.scatter(X[:, 0].tolist(), X[:, 1].tolist(), marker='*', c=y.tolist())
plt.show()1.查看构建的数据集
2.划分训练集测试集并显示数据集维度
3.观察Softmax函数计算情况，假设值为[[0.1, 0.2, 0.3, 0.4],[1,2,3,4]]
4.构建类似于二分类的算子，并进行简单测试
5.对构建好的模型进行训练（学习率0.1）
更改学习率为0.01再次观察
6.对最终模型进行评估和测试
学习率改为0.01后
三.基于Softmax回归完成鸢尾花分类任务
数据：Iris数据集
Iris鸢尾花数据集是一个经典数据集，在机器学习领域都经常被用作示例。
数据集内包含3类共150条记录，每类各50个数据。
每条记录有4项特征：花萼长度、花萼宽度、花瓣长度、花瓣宽度。
可以通过4个特征预测鸢尾花卉属于（setosa 山鸢尾, versicolour 变色鸢尾, virginica 弗吉尼亚鸢尾）中的哪一品种。
注：此分类为多分类任务，softmax，Op，model_SR，MultiCrossEntropyLoss，Optimizer，RunnerV2，plot等类和函数
from abc import abstractmethod
import torch
from sklearn.datasets import load_iris
import pandas
import numpy as np
import matplotlib.pyplot as plt #可视化工具
# 固定随机种子
torch.seed()
# 加载数据集
def load_data(shuffle=True):
"""
加载鸢尾花数据
输入：shuffle：是否打乱数据，数据类型为bool
输出：X：特征数据，shape=[150,4]
y：标签数据, shape=[150]
"""
# 加载原始数据
X = np.array(load_iris().data, dtype=np.float32)
y = np.array(load_iris().target, dtype=np.int32)
X = torch.tensor(X)
y = torch.tensor(y)
# 数据归一化
X_min = torch.min(X, dim=0).values # 获取最小值，dim=0 表示按列
X_max = torch.max(X, dim=0).values # 获取最大值，dim=0 表示按列
X = (X-X_min) / (X_max-X_min)
# 如果shuffle为True，随机打乱数据
if shuffle:
idx = torch.randperm(X.shape[0])
X = X[idx]
y = y[idx]
return X, y
num_train = 120
num_dev = 15
num_test = 15
X, y = load_data(shuffle=True)
X_train, y_train = X[:num_train], y[:num_train]
X_dev, y_dev = X[num_train:num_train + num_dev], y[num_train:num_train + num_dev]
X_test, y_test = X[num_train + num_dev:], y[num_train + num_dev:]
def softmax(X):
"""
输入：X：shape=[N, C]，N为向量数量，C为向量维度
"""
x_max = torch.max(X, dim=1, keepdim=True).values # N, 1
x_exp = torch.exp(X - x_max)
partition = torch.sum(x_exp, dim=1, keepdim=True) # N, 1
return x_exp / partition
class Op(object):
def __init__(self):
pass
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
raise NotImplementedError
def backward(self, inputs):
raise NotImplementedError
class model_SR(Op):
def __init__(self, input_dim, output_dim):
super(model_SR, self).__init__()
self.params = {}
# 将线性层的权重参数全部初始化为0
self.params['W'] = torch.zeros([input_dim, output_dim])
self.params['W'] = torch.normal(mean=0, std=0.01, size=[input_dim, output_dim])
# 将线性层的偏置参数初始化为0
self.params['b'] = torch.zeros([output_dim])
self.outputs = None
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
"""
输入：inputs: shape=[N,D], N是样本数量，D是特征维度
输出：outputs：预测值，shape=[N,C]，C是类别数
"""
# 线性计算
score = torch.matmul(inputs, self.params['W']) + self.params['b']
# Softmax 函数
self.outputs = softmax(score)
return self.outputs
class MultiCrossEntropyLoss(Op):
def __init__(self):
self.predicts = None
self.labels = None
self.num = None
def __call__(self, predicts, labels):
return self.forward(predicts, labels)
def forward(self, predicts, labels):
"""
输入：predicts：预测值，shape=[N, 1]，N为样本数量
labels：真实标签，shape=[N, 1]
输出：损失值：shape=[1]
"""
self.predicts = predicts
self.labels = labels
self.num = self.predicts.shape[0]
loss = 0
for i in range(0, self.num):
index = self.labels[i]
loss -= torch.log(self.predicts[i][index])
return loss / self.num
class model_SR(Op):
def __init__(self, input_dim, output_dim):
super(model_SR, self).__init__()
self.params = {}
# 将线性层的权重参数全部初始化为0
self.params['W'] = torch.zeros(input_dim, output_dim)
# self.params['W'] = paddle.normal(mean=0, std=0.01, shape=[input_dim, output_dim])
# 将线性层的偏置参数初始化为0
self.params['b'] = torch.zeros(output_dim)
# 存放参数的梯度
self.grads = {}
self.X = None
self.outputs = None
self.output_dim = output_dim
def __call__(self, inputs):
return self.forward(inputs)
def forward(self, inputs):
self.X = inputs
# 线性计算
score = torch.matmul(self.X, self.params['W']) + self.params['b']
# Softmax 函数
self.outputs = softmax(score)
return self.outputs
def backward(self, labels):
"""
输入：labels：真实标签，shape=[N, 1]，其中N为样本数量
"""
# 计算偏导数
N =labels.shape[0]
labels = torch.nn.functional.one_hot(labels.to(torch.int64), self.output_dim)
self.grads['W'] = -1 / N * torch.matmul(self.X.t(), (labels-self.outputs))
self.grads['b'] = -1 / N * torch.matmul(torch.ones(N), (labels-self.outputs))
class Optimizer(object):
def __init__(self, init_lr, model):
"""优化器类初始化"""
# 初始化学习率，用于参数更新的计算
self.init_lr = init_lr
# 指定优化器需要优化的模型
self.model = model
@abstractmethod
def step(self):
"""定义每次迭代如何更新参数"""
pass
class RunnerV2(object):
def __init__(self, model, optimizer, metric, loss_fn):
self.model = model
self.optimizer = optimizer
self.loss_fn = loss_fn
self.metric = metric
# 记录训练过程中的评价指标变化情况
self.train_scores = []
self.dev_scores = []
# 记录训练过程中的损失函数变化情况
self.train_loss = []
self.dev_loss = []
def train(self, train_set, dev_set, **kwargs):
# 传入训练轮数，如果没有传入值则默认为0
num_epochs = kwargs.get("num_epochs", 0)
# 传入log打印频率，如果没有传入值则默认为100
log_epochs = kwargs.get("log_epochs", 100)
# 传入模型保存路径，如果没有传入值则默认为"best_model.pdparams"
save_path = kwargs.get("save_path", "best_model.pdparams")
# 梯度打印函数，如果没有传入则默认为"None"
print_grads = kwargs.get("print_grads", None)
# 记录全局最优指标
best_score = 0
# 进行num_epochs轮训练
for epoch in range(num_epochs):
X, y = train_set
# 获取模型预测
logits = self.model(X)
# 计算交叉熵损失
trn_loss = self.loss_fn(logits, y).item()
self.train_loss.append(trn_loss)
# 计算评价指标
trn_score = self.metric(logits, y).item()
self.train_scores.append(trn_score)
# 计算参数梯度
self.model.backward(y)
if print_grads is not None:
# 打印每一层的梯度
print_grads(self.model)
# 更新模型参数
self.optimizer.step()
dev_score, dev_loss = self.evaluate(dev_set)
# 如果当前指标为最优指标，保存该模型
if dev_score > best_score:
self.save_model(save_path)
print(f"best accuracy performence has been updated: {best_score:.5f} --> {dev_score:.5f}")
best_score = dev_score
if epoch % log_epochs == 0:
print(f"[Train] epoch: {epoch}, loss: {trn_loss}, score: {trn_score}")
print(f"[Dev] epoch: {epoch}, loss: {dev_loss}, score: {dev_score}")
def evaluate(self, data_set):
X, y = data_set
# 计算模型输出
logits = self.model(X)
# 计算损失函数
loss = self.loss_fn(logits, y).item()
self.dev_loss.append(loss)
# 计算评价指标
score = self.metric(logits, y).item()
self.dev_scores.append(score)
return score, loss
def predict(self, X):
return self.model(X)
def save_model(self, save_path):
torch.save(self.model.params, save_path)
def load_model(self, model_path):
self.model.params = torch.load(model_path)
class SimpleBatchGD(Optimizer):
def __init__(self, init_lr, model):
super(SimpleBatchGD, self).__init__(init_lr=init_lr, model=model)
def step(self):
# 参数更新
# 遍历所有参数
if isinstance(self.model.params, dict):
for key in self.model.params.keys():
self.model.params[key] = self.model.params[key] - self.init_lr * self.model.grads[key]
def accuracy(preds, labels):
"""
输入： preds：预测值，二分类时，shape=[N, 1]，N为样本数量，多分类时，shape=[N, C]，C为类别数量
labels：真实标签，shape=[N, 1]
输出：准确率：shape=[1]
"""
# 判断是二分类任务还是多分类任务，preds.shape[1]=1时为二分类任务，preds.shape[1]>1时为多分类任务
if preds.shape[1] == 1:
# 二分类时，判断每个概率值是否大于0.5，当大于0.5时，类别为1，否则类别为0
# 使用'torch.cast'将preds的数据类型转换为float32类型
preds = (preds >= 0.5).float()
else:
# 多分类时，使用'torch.argmax'计算最大元素索引作为类别
preds = torch.argmax(preds, 1)
accuracy = torch.mean((preds == labels).float())
return accuracy
def plot(runner, fig_name):
plt.figure(figsize=(10, 5))
plt.subplot(1, 2, 1)
epochs = [i for i in range(len(runner.train_scores))]
# 绘制训练损失变化曲线
plt.plot(epochs, runner.train_loss, color='#e4007f', label="Train loss")
# 绘制评价损失变化曲线
plt.plot(epochs, runner.dev_loss, color='#f19ec2', linestyle='--', label="Dev loss")
# 绘制坐标轴和图例
plt.ylabel("loss", fontsize='large')
plt.xlabel("epoch", fontsize='large')
plt.legend(loc='upper right', fontsize='x-large')
plt.subplot(1, 2, 2)
# 绘制训练准确率变化曲线
plt.plot(epochs, runner.train_scores, color='#e4007f', label="Train accuracy")
# 绘制评价准确率变化曲线
plt.plot(epochs, runner.dev_scores, color='#f19ec2', linestyle='--', label="Dev accuracy")
# 绘制坐标轴和图例
plt.ylabel("score", fontsize='large')
plt.xlabel("epoch", fontsize='large')
plt.legend(loc='lower right', fontsize='x-large')
plt.tight_layout()
plt.savefig(fig_name)
plt.show()
# 箱线图查看异常值分布
def boxplot(features):
feature_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width']
# 连续画几个图片
plt.figure(figsize=(5, 5), dpi=200)
# 子图调整
plt.subplots_adjust(wspace=0.6)
# 每个特征画一个箱线图
for i in range(4):
plt.subplot(2, 2, i+1)
# 画箱线图
plt.boxplot(features[:, i],
showmeans=True,
whiskerprops={"color":"#E20079", "linewidth":0.4, 'linestyle':"--"},
flierprops={"markersize":0.4},
meanprops={"markersize":1})
# 图名
plt.title(feature_names[i], fontdict={"size":5}, pad=2)
# y方向刻度
plt.yticks(fontsize=4, rotation=90)
plt.tick_params(pad=0.5)
# x方向刻度
plt.xticks([])
plt.savefig('ml-vis.pdf')
plt.show()
iris_features = np.array(load_iris().data, dtype=np.float32)
iris_labels = np.array(load_iris().target, dtype=np.int32)
print(pandas.isna(iris_features).sum())
print(pandas.isna(iris_labels).sum())
boxplot(iris_features)
# 输入维度
input_dim = 4
# 类别数
output_dim = 3
# 实例化模型
model = model_SR(input_dim=input_dim, output_dim=output_dim)
# 学习率
lr = 0.2
# 梯度下降法
optimizer = SimpleBatchGD(init_lr=lr, model=model)
# 交叉熵损失
loss_fn = MultiCrossEntropyLoss()
# 准确率
metric = accuracy
# 实例化RunnerV2
runner = RunnerV2(model, optimizer, metric, loss_fn)
# 启动训练
runner.train([X_train, y_train], [X_dev, y_dev], num_epochs=200, log_epochs=10, save_path="best_model.pdparams")
plot(runner, fig_name='linear-acc2.pdf')
# 加载最优模型
runner.load_model('best_model.pdparams')
# 模型评价
score, loss = runner.evaluate([X_test, y_test])
print("[测试] 分数/损失: {:.4f}/{:.4f}".format(score, loss))
# 预测测试集数据
logits = runner.predict(X_test)
# 观察其中一条样本的预测结果
pred = torch.argmax(logits[0]).numpy()
# 获取该样本概率最大的类别
label = y_test[0].numpy()
# 输出类别
print("真实类别为 {}，预测类别为 {}".format(label.item(), pred.item()))1.查看鸢尾花中的缺失值，并划分训练集，用箱线图观察数据集
表示无缺失值
2.调用相关类和函数对模型进行训练（学习率0.2）
更改学习率为0.3
更改学习率为0.1
更改学习率为0.01
观察可知学习率0.2时最优
3.对模型进行测试
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
