深度学习——模型训练流程（四）-优快云博客

本文链接：https://blog.youkuaiyun.com/qq_52952281/article/details/144485642

实战Kaggle⽐赛：预测房价

、下载和缓存数据集方法

⾸先，我们建⽴字典DATA_HUB，它可以将数据集名称的字符串映射到数据集相关的⼆元组上，这个⼆元组包含数据集的url和验证⽂件完整性的sha-1密钥。所有类似的数据集都托管在地址为DATA_URL的站点上。

import hashlib
import os
import tarfile
import zipfile
import requests

Data_Hub = dict()
Data_Url = 'http://d2l-data.s3-accelerate.amazonaws.com/'

下⾯的download函数⽤来下载数据集，将数据集缓存在本地⽬录（默认情况下为…/data）中，并返回下载⽂件的名称。

"""下载⼀个DATA_HUB中的⽂件，返回本地⽂件名"""

def download(name, cache_dir = os.path.join('..','data')):
    assert name in Data_Hub, f"{name}不存在于{Data_Hub}"
    url, sha1_hash = Data_Hub[name]
    os.makedirs(cache_dir, exist_ok=True)
    fname = os.path.join(cache_dir, url.split('/')[-1])
    if os.path.exists(fname):
        sha1 = hashlib.sha1()
        with open(fname, 'rb') as f:
            while True:
                data = f.read(1048576)
                if not data:
                    break
            sha1.update(data)
        if sha1.hexdigest() == sha1_hash:
            return fname # 命中缓存
    print(f'正在从{url}下载{fname}...')
    r = requests.get(url, stream=True, verify= True)
    with open(fname, 'wb') as f:
        f.write(r.content)
    print(f'下载{fname}完毕...')
    return fname

我们还需实现两个实⽤函数：⼀个将下载并解压缩⼀个zip或tar⽂件，另⼀个是将本书中使⽤的所有数据集从DATA_HUB下载到缓存⽬录中。

"""下载并解压zip/tar⽂件"""
def download_extract(name, folder = None):
    fname = download(name)
    base_dir = os.path.dirname(fname)
    data_dir, ext = os.path.splitext(fname)

    if ext == '.zip':
        fp = zipfile.ZipFile(fname, 'r')
    elif ext in ('.tar', '.gz'):
        fp = tarfile.open(fname, 'r')
    else:
        AssertionError(f"不支持的压缩文件格式: {ext}") 
    fp.extract(base_dir)
    return os.path.join(base_dir, folder) if folder else data_dir

def download_all(): #@save
    """下载DATA_HUB中的所有⽂件"""
    for name in Data_Hub:
        download(name)

、获取和读取数据集

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
from torch import nn
from d2l import torch as d2l
Data_Hub['kaggle_house_train'] = (Data_Url + 'kaggle_house_pred_train.csv',
                                  '585e9cc93e70b39160e7921475f9bcd7d31219ce')
Data_Hub['kaggle_house_test'] = (Data_Url + 'kaggle_house_pred_test.csv',
                                 'fa19780a7b011d9b009e8bff8e99922a8ee2eb90')

for key, value in Data_Hub.items():
    print(key, value)
download('kaggle_house_train')
download('kaggle_house_test')

、数据预处理——重点

all_features[numeric_features]：选择了 all_features 中的数值特征列，这些列由变量 numeric_features 所指定。
.apply(lambda x: (x - x.mean()) / (x.std()))：对选定的数值特征列进行了标准化处理。使用了 apply 函数，它可以接受一个函数作为参数，并将该函数应用到数据的每一列（或每一行）上。在这里，传入的函数是一个 lambda 函数，它对每一列进行操作。
x.mean() 计算了每一列的均值。
x.std() 计算了每一列的标准差。

(x - x.mean()) / x.std() 对每一列进行了标准化处理，即将每个元素减去该列的均值，然后除以该列的标准差。
标准化处理的目的是将数据按比例缩放，使之落入一个标准的范围，有利于提高模型的训练效果和收敛速度。

# 从文件中获取数据
train_data = pd.read_csv('..\\data\\kaggle_house_pred_train.csv')
test_data = pd.read_csv('..\\data\\kaggle_house_pred_test.csv')
print(train_data.shape, test_data.shape)
# print(train_data.iloc[0:4, [0, 1, 2, 3, -3, -2, -1]])


all_features = pd.concat((train_data.iloc[:, 1:-1], test_data.iloc[:, 1:]))

print(all_features.shape)

# 标准化数据
numeric_features = all_features.dtypes[all_features.dtypes != 'object'].index
all_features[numeric_features] = all_features[numeric_features].apply(
                        lambda x: (x - x.mean()) / (x.std()))

# 在标准化数据之后，所有均值消失，因此我们可以将缺失值设置为0
all_features[numeric_features] = all_features[numeric_features].fillna(0)
# print(all_features.iloc[0:4, [0, 1, 2, 3, -3, -2, -1]])
# print(all_features.shape)
# 处理离散值:独热编码
# “Dummy_na=True”将“na”（缺失值）视为有效的特征值，并为其创建指⽰符特征
all_features = pd.get_dummies(all_features, dummy_na=True)
# print(all_features.iloc[0:4, [0, 1, 2, 3, -3, -2, -1]])



n_train = train_data.shape[0]
train_features = torch.tensor(all_features[:n_train].values , 
                              dtype = torch.float32, device='cuda:0')
test_features = torch.tensor(all_features[n_train:].values , 
                              dtype = torch.float32, device='cuda:0')

train_labels = torch.tensor(train_data.SalePrice.values.reshape(-1, 1),
                             dtype=torch.float32, device= 'cuda:0')

train_features.shape
print(train_features.shape, train_labels.shape)

、模型建立

in_features = train_features.shape[1]
net = nn.Sequential(nn.Linear(in_features, 1, device='cuda:0'))

# 损失函数
loss = torch.nn.MSELoss()

def log_rmse(net, features, labels):
# 为了在取对数时进⼀步稳定该值，将⼩于1的值设置为1
    clipped_preds = torch.clamp(net(features), 1, float('inf'))
    rmse = torch.sqrt(loss(torch.log(clipped_preds), torch.log(labels)))
    return rmse.item()

def train(net, train_features, train_labels, test_features, test_labels, num_epochs, lr, wd, batch_size, loss):
    train_ls, test_ls = [], []
    train_iter = d2l.load_array((train_features,train_labels), batch_size=batch_size)
    optimer = torch.optim.Adam(net.parameters(), lr = lr, weight_decay = wd)
    for epoch in range(num_epochs):
        for x, y in train_iter:
            optimer.zero_grad()
            l = loss(net(x), y)
            l.backward()
            optimer.step()
        train_ls.append(log_rmse(net, train_features, train_labels))
        if test_labels is not None:
            test_ls.append(log_rmse(net, test_features, test_labels))
    return train_ls, test_ls


def get_k_fold_data(k, i, x, y):
    assert k > 1
    fold_size = x.shape[0] // k
    x_train ,y_train = None, None
    for j in range(k):
        idx = slice(j * fold_size, (j + 1) * fold_size)
        x_part , y_part = x[idx, :], y[idx]
        if j == i:
            x_valid, y_valid = x_part, y_part           # 遍历找验证集
        elif x_train is None:
            x_train, y_train = x_part, y_part
        else:
            x_train = torch.cat([x_train, x_part], 0)
            y_train = torch.cat([y_train, y_part], 0)
    return x_train, y_train, x_valid, y_valid

def k_fold(k, x_train, y_train, num_epochs, lr, wd, batch_size):
    train_l_sum, valid_l_sum = 0, 0
    for i in range(k):
        data = get_k_fold_data(k, i, x_train, y_train)
        train_ls, valid_ls = train(net, *data, num_epochs, lr, wd, batch_size, loss)
        train_l_sum += train_ls[-1]
        valid_l_sum += valid_ls[-1]
        if i == 0:
            plt.plot(list(range(1, num_epochs + 1)), train_ls, label='train')
            plt.plot(list(range(1, num_epochs + 1)), valid_ls, label='valid')
            plt.xlabel('epoch')
            plt.ylabel('rmse')
            plt.legend()
            plt.yscale('log')
            plt.show()

        print(f'折{i + 1}，训练log rmse{float(train_ls[-1]):f}, 'f'验证log rmse{float(valid_ls[-1]):f}')
    return train_l_sum / k, valid_l_sum / k   

k, num_epochs, lr, wd, batch_size = 5, 100, 5, 0, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr, wd, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, 'f'平均验证log rmse: {float(valid_l):f}')

、训练并预测

# 输入训练集和测试集，以及模型参数，输出测试集预测值

def train_and_pred(net, train_data, train_label, test_data, test_label, num_epochs, lr, wd, batch_size):
    train_ls , _ = train(net, train_data, train_label, None, None, num_epochs, lr, wd, batch_size, loss)
    plt.plot(list(range(1, num_epochs + 1)), train_ls, label='train')
    plt.xlabel('epoch')
    plt.ylabel('rmse')
    # plt.legend()
    # plt.yscale('log')
    plt.show()
    print(f'训练log rmse： {float(train_ls[-1]):f}')
    preds = net(test_features).cpu().detach().numpy()
    return preds.shape


train_and_pred(net, train_features, train_labels, test_features, test_features,num_epochs, lr, wd, batch_size)