ResNet34 on CIFAR-10 基准

时间20210502

作者：知道许多的橘子

实现：ResNet34对CIFAR-10数据集的分类

测试集准确度：95.89%

实现框架pytorch

数据增强方法：Normalize+Fix等

训练次数：200

阶段学习率[0-200]：smooth_step(10,40,100,150,epoch_s)

优化器optimizer = torch.optim.SGD(model.parameters(),lr=smooth_step(10,40,100,150,epoch_s), momentum=0.9,weight_decay=1e-5)

# 数据见Fcat_20210419
# In[1] 导入所需工具包
import os
import torch
import torch.nn as nn
import torchvision
from torchvision import datasets,transforms
import time
from torch.nn import functional as F
from math import floor, ceil
import math
import numpy as np
import sys
sys.path.append(r'/home/megstudio/workspace/')
from FMIX.fmix import sample_and_apply, sample_mask
#import torchvision.transforms as transforms
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(device)
import random
# In[1] 设置超参数
num_epochs = 200
batch_size = 100
tbatch_size = 5000
#learning_rate = 0.1
test_name = '/_R34_SJGB3_BLIN_S_C10'

# In[1] 加载数据
# In[2]#图像预处理变换的定义
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4), #在一个随机的位置进行裁剪，32正方形裁剪，每个边框上填充4
transforms.RandomHorizontalFlip(), #以给定的概率随机水平翻转给定的PIL图像，默认值为0.5
transforms.ToTensor(), #将PIL Image或者 ndarray 转换为tensor，并且归一化至[0-1]
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),#用平均值和标准偏差归一化张量图像，
#(M1,…,Mn)和(S1,…,Sn)将标准化输入的每个通道
])
transform_test = transforms.Compose([ #测试集同样进行图像预处理
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
cifar10_train = torchvision.datasets.CIFAR10(root='/home/megstudio/dataset', train=True, download=False, transform=transform_train)
cifar10_test = torchvision.datasets.CIFAR10(root='/home/megstudio/dataset', train=False, download=False, transform=transform_test)

train_loader = torch.utils.data.DataLoader(cifar10_train, batch_size=batch_size, shuffle=True, num_workers=2)
test_loader = torch.utils.data.DataLoader(cifar10_test, batch_size=tbatch_size, shuffle=False, num_workers=2)

# In[1] 加载模型
# 用于ResNet18和34的残差块，用的是2个3x3的卷积
class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.shortcut = nn.Sequential()
        # 经过处理后的x要与x的维度相同(尺寸和深度)
        # 如果不相同，需要添加卷积+BN来变换为同一维度
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out


# 用于ResNet50,101和152的残差块，用的是1x1+3x3+1x1的卷积
class Bottleneck(nn.Module):
    # 前面1x1和3x3卷积的filter个数相等，最后1x1卷积是其expansion倍
    expansion = 4

    def __init__(self, in_planes, planes, stride=1):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, self.expansion*planes,
                               kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(self.expansion*planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = F.relu(self.bn2(self.conv2(out)))
        out = self.bn3(self.conv3(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out


class ResNet(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out


def ResNet18():
    return ResNet(BasicBlock, [2,2,2,2])

def ResNet34():
    return ResNet(BasicBlock, [3,4,6,3])

def ResNet50():
    return ResNet(Bottleneck, [3,4,6,3])

def ResNet101():
    return ResNet(Bottleneck, [3,4,23,3])

def ResNet152():
    return ResNet(Bottleneck, [3,8,36,3])

def smooth_step(a,b,c,d,x):
    level_s=0.01
    level_m=0.1
    level_n=0.01
    level_r=0.005
    if x<=a:
        return level_s
    if a<x<=b:
        return (((x-a)/(b-a))*(level_m-level_s)+level_s)
    if b<x<=c:
        return level_m
    if c<x<=d:
        return level_n
    if d<x:
        return level_r
# In[1] 设置一个通过优化器更新学习率的函数
def update_lr(optimizer, lr):
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr
# In[1] 定义测试函数
def test(model,test_loader):
    
    model.eval()
    with torch.no_grad():
        correct = 0
        total = 0
        for images, labels in test_loader:
            images = images.to(device)
            labels = labels.to(device)
            outputs= model(images)

            _, predicted = torch.max(outputs.data, 1)

            total += labels.size(0)
            correct += (predicted == labels).sum().item()
        acc=100 * correct / total
    
        print('Accuracy of the model on the test images: {} %'.format(acc))
    return acc

# In[1] 定义模型和损失函数
# =============================================================================
def mkdir(path):
    folder = os.path.exists(path)
    if not folder:                   #判断是否存在文件夹如果不存在则创建为文件夹
        os.makedirs(path)            #makedirs 创建文件时如果路径不存在会创建这个路径
        print("---  new folder...  ---")
        print("---  OK  ---")
    else:
        print("---  There is this folder!  ---")
path = os.getcwd()
path = path+test_name
print(path)
mkdir(path)             #调用函数
# In[1] 定义模型和损失函数
#$$
# =============================================================================
try:
    model = torch.load(path+'/model.pkl').to(device)
    epoch_s = np.load(path+'/learning_rate.npy')
    #learning_rate *= 3
    print(epoch_s)
    train_loss = np.load(path+'/test_acc.npy').tolist()
    test_acc = np.load(path+'/test_acc.npy').tolist()
    print("---  There is a model in the folder...  ---")
except:
    print("---  Create a new model...  ---")
    epoch_s = 0
    model = ResNet34().to(device)
    train_loss=[]#准备放误差     
    test_acc=[]#准备放测试准确率
# =============================================================================
def saveModel(model,epoch,test_acc,train_loss):
    torch.save(model, path+'/model.pkl')
    # torch.save(model.state_dict(), 'resnet.ckpt')
    epoch_save=np.array(epoch)
    np.save(path+'/learning_rate.npy',epoch_save)
    test_acc=np.array(test_acc)
    np.save(path+'/test_acc.npy',test_acc)
    train_loss=np.array(train_loss)
    np.save(path+'/train_loss.npy',train_loss)
    
criterion = nn.CrossEntropyLoss()
#optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay=1e-5)
optimizer = torch.optim.SGD(model.parameters(),lr=smooth_step(10,40,100,150,epoch_s), momentum=0.9,weight_decay=1e-5) 
#scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,  milestones = [100, 150], gamma = 0.1, last_epoch=-1)
# In[1] 训练模型更新学习率
total_step = len(train_loader)
#curr_lr = learning_rate
for epoch in range(epoch_s, num_epochs):
    #scheduler.step()
    in_epoch = time.time()
    for i, (images, labels) in enumerate(train_loader):
        
        # =============================================================================
        # FMiX
        # print(type(images))
        images, index, lam = sample_and_apply(images, alpha=1, decay_power=3, shape=(32,32))
        images = images.type(torch.FloatTensor) 
        shuffled_label = labels[index].to(device)
        
        images = images.to(device)
        labels = labels.to(device)
        # Forward pass
        outputs = model(images)
        loss = lam*criterion(outputs, labels) + (1-lam)*criterion(outputs, shuffled_label)
        # loss = criterion(outputs, labels)
        # =============================================================================

        # Backward and optimize
        optimizer.zero_grad()
        loss.backward(retain_graph=False)
        optimizer.step()
        
        if (i + 1) % 100 == 0:
            print("Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}"
                  .format(epoch + 1, num_epochs, i + 1, total_step, loss.item()))
    # 记录误差和精度
    train_loss.append(loss.item())
    acctemp = test(model, test_loader)
    test_acc.append(acctemp)
    # 更新学习率
    curr_lr = smooth_step(10, 40, 100, 150, epoch)
    update_lr(optimizer, curr_lr)
    # 保存模型和一些参数、指标
    saveModel(model, epoch, test_acc, train_loss)
    # 记录时间
    out_epoch = time.time()
    print(f"use {(out_epoch-in_epoch)//60}min{(out_epoch-in_epoch)%60}s")
#$$
# In[1] 测试训练集
test(model, train_loader)