pytorch: 学习笔记12, pytorch丢弃法

pytorch丢弃法dropout

网络：
输入层：28*28=784个单元 (每个样本) --> 隐藏层1: 256个单元 --> 隐藏层2: 256个单元 --> 输出层：10(分类)

代码：

import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms
import sys


class FlattenLayer(torch.nn.Module):
    def __init__(self):
        super(FlattenLayer, self).__init__()
    def forward(self, x): # x shape: (batch, *, *, ...)
        return x.view(x.shape[0], -1)


def evaluate_accuracy(data_iter, net):
    acc_sum, n = 0.0, 0
    for X, y in data_iter:
        if isinstance(net, torch.nn.Module):
            net.eval() # 评估模式, 这会关闭dropout
            acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()
            net.train() # 改回训练模式
        else: # 自定义的模型
            if('is_training' in net.__code__.co_varnames): # 如果有is_training这个参数
                # 将is_training设置成False
                acc_sum += (net(X, is_training=False).argmax(dim=1) == y).float().sum().item()
            else:
                acc_sum += (net(X).argmax(dim=1) == y).float().sum().item()
        n += y.shape[0]
    return acc_sum / n

def sgd(params, lr, batch_size):
    # 为了和原书保持一致，这里除以了batch_size，但是应该是不用除的，因为一般用PyTorch计算loss时就默认已经
    # 沿batch维求了平均了。
    for param in params:
        param.data -= lr * param.grad / batch_size # 注意这里更改param时用的param.data

def train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size,
              params=None, lr=None, optimizer=None):
    for epoch in range(num_epochs):
        train_l_sum, train_acc_sum, n = 0.0, 0.0, 0
        for X, y in train_iter:
            y_hat = net(X)
            l = loss(y_hat, y).sum()

            # 梯度清零
            if optimizer is not None:
                optimizer.zero_grad()
            elif params is not None and params[0].grad is not None:
                for param in params:
                    param.grad.data.zero_()

            l.backward()
            if optimizer is None:
                sgd(params, lr, batch_size)
            else:
                optimizer.step()  # “softmax回归的简洁实现”一节将用到

            train_l_sum += l.item()
            train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()
            n += y.shape[0]
        test_acc = evaluate_accuracy(test_iter, net)
        print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f'
              % (epoch + 1, train_l_sum / n, train_acc_sum / n, test_acc))

def load_data_fashion_mnist(batch_size, root='Datasets/FashionMNIST'):
    mnist_train = torchvision.datasets.FashionMNIST(root=root, train=True, download=False, transform=transforms.ToTensor())
    mnist_test = torchvision.datasets.FashionMNIST(root=root, train=False, download=False, transform=transforms.ToTensor())
    if sys.platform.startswith('win'):
        num_workers = 0  # 0表示不用额外的进程来加速读取数据
    else:
        num_workers = 4
    train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=num_workers)
    test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False, num_workers=num_workers)

    return train_iter, test_iter


if __name__ == '__main__':
    # 超参数
    num_inputs, num_outputs, num_hiddens1, num_hiddens2 = 784, 10, 256, 256
    drop_prob1, drop_prob2 = 0.2, 0.5
    num_epochs, lr, batch_size = 5, 100.0, 256

    # 定义网络
    net = nn.Sequential(
            FlattenLayer(),
            nn.Linear(num_inputs, num_hiddens1),
            nn.ReLU(),
            nn.Dropout(drop_prob1),
            nn.Linear(num_hiddens1, num_hiddens2),
            nn.ReLU(),
            nn.Dropout(drop_prob2),
            nn.Linear(num_hiddens2, 10)
            )

    # 网络参数初始化
    for param in net.parameters():
        nn.init.normal_(param, mean=0, std=0.01)

    # 定义损失函数、优化器
    loss = torch.nn.CrossEntropyLoss()
    optimizer = torch.optim.SGD(net.parameters(), lr=0.5)

    # 加载数据集并训练
    train_iter, test_iter = load_data_fashion_mnist(batch_size)
    train_ch3(net, train_iter, test_iter, loss, num_epochs, batch_size, None, None, optimizer)

结果：
epoch 1, loss 0.0046, train acc 0.541, test acc 0.746
epoch 2, loss 0.0023, train acc 0.788, test acc 0.770
epoch 3, loss 0.0019, train acc 0.822, test acc 0.772
epoch 4, loss 0.0017, train acc 0.837, test acc 0.803
epoch 5, loss 0.0016, train acc 0.849, test acc 0.819

参考摘录：
https://tangshusen.me/Dive-into-DL-PyTorch/#/chapter03_DL-basics/3.13_dropout?id=_3131-%e6%96%b9%e6%b3%95