【机器学习 - 7】：梯度下降法（第二篇）

在线性回归中使用梯度下降法

---

1. 准备数据，其中np.random.normal(size=100) 得到100个标准正态分布的点，主要是为了制造噪音

import numpy as np
import matplotlib.pyplot as plt
x = np.random.random(size=100)
# np.random.normal(size=100) 得到100个标准正太分布的点，制造噪音
y = x*3 + 4 + np.random.normal(size=100)
plt.scatter(x, y)
plt.show()

2. 使用梯度下降法求解theta，其中的 X_b.dot(theta)为y的预测值，即y`

def j(theta, X_b, y):
    return np.sum((y-X_b.dot(theta))**2)/len(X_b)

def dj(theta, X_b, y): # 求偏导数  
    res = np.empty(len(theta))
    res[0] = np.sum((X_b.dot(theta)-y))
    for i in range(1, len(theta)):
        res[i] = (X_b.dot(theta)-y).dot(X_b[:,i])
    return res*2/len(X_b)

def gradient_descent(X_b,y,eta,initial_theta,n_iters=1e3,epsilon=1e-8):
    theta = initial_theta
    i_iter = 1
    while i_iter<n_iters:
        last_theta = theta
        theta = theta - eta*dj(theta, X_b, y)
        if abs(j(theta,X_b,y) - j(last_theta,X_b,y))<epsilon:
            break
        i_iter += 1
    return theta

3. 构造X_b，使用np.hstack进行两个向量的拼接

X_b = np.hstack([np.ones(len(x)).reshape(-1,1), x.reshape(-1,1)])

eta = 0.01
initial_theta = np.zeros(X_b.shape[1])
gradient_descent(X_b, y, eta, initial_theta)

梯度下降代码的封装

---

封装的代码

import numpy as np


def r2_score(y_true, y_predict):
    return 1 - ((np.sum((y_true - y_predict) ** 2) / len(y_true)) / np.var(y_true))


class MyLinearGression:
    def __init__(self):
        self._theta = None  # theta参数
        self.coef_ = None   # 系数
        self.interception_ = None   # 截距

    def fit_gd(self, X_train, y, eta=0.01, n_iters=1e3, epsilon=1e-8):
        def j(theta, X_b, y):
            try:
                return np.sum((y - X_b.dot(theta)) ** 2) / len(X_b)
            except:
                return float('inf')

        def dj(theta, X_b, y):
            return X_b.T.dot(X_b.dot(theta)-y)*2./len(X_b)

        def gradient_descent(X_b, y, eta, initial_theta, n_iters=1e3, epsilon=1e-8):
            theta = initial_theta
            i_iter = 1
            while i_iter<n_iters:
                last_theta = theta
                theta = theta - eta * dj(theta, X_b, y)
                if abs(j(theta,X_b,y)-j(last_theta,X_b,y)) < epsilon:
                    break
                i_iter += 1
            return theta
        X_b = np.hstack([np.ones(len(X_train)).reshape(-1,1), X_train])
        initial_theta = np.zeros(X_b.shape[1])
        self._theta = gradient_descent(X_b,y,eta,initial_theta)
        self.interception_ = self._theta[0]     # 截距
        self.coef_ = self._theta[1:]    # 系数
        return self

    def score(self, X_predict, y_test):
        y_predict = self.predict(X_predict)
        return r2_score(y_test, y_predict)

    def predict(self, X_predict):
        X_b = np.hstack([np.ones(len(X_predict)).reshape(-1,1), X_predict])
        return X_b.dot(self._theta)

    def __repr__(self):
        return "MyLinearGression()"

封装代码的使用

import numpy as np

x = np.random.random(size=100)
y = x*3 + 4 + np.random.normal(size=100)

%run LinearRegression.py
lin_reg = MyLinearGression()
lin_reg.fit_gd(x.reshape(-1,1), y)

使用真实数据来进行梯度下降的过程

from sklearn.datasets import load_boston
import numpy as np

boston = load_boston()
x = boston.data
y = boston.target

X = x[y<50]
Y = y[y<50]

%run LinearRegression1.py
lin_reg = MyLinearGression()

观察如上结果，出现nan的原因：在真实数据中我们的跨度太大了，使我们的步长很大，最终使得梯度下降法的过程是不收敛的。

解决办法：数据归一化

from sklearn.preprocessing import StandardScaler
std_scaler = StandardScaler()
std_scaler.fit(X_train)
X_train_std = std_scaler.transform(X_train)
X_test_std = std_scaler.transform(X_test)

from sklearn.linear_model import SGDRegressor # 使用sklearn中的梯度下降法
sgd_reg = SGDRegressor()
sgd_reg.fit(X_train_std, Y_train)
sgd_reg.score(X_test_std, Y_test)