【机器学习入门】6、梯度下降法

梯度下降法（Gradient Descent）

不是机器学习方法，是一种基于搜索的最优化方法，最小化损失函数。
过程类似球从碗口滚落到碗底。

并不是所有函数都有唯一极值点

局部最优解
极小值点

全局最优解
最小值点

在线性回归中使用梯度下降法

模拟梯度下降法

import numpy as np
import matplotlib.pyplot as plt

plot_x = np.linspace(-1, 6, 141) # 模拟一组数据

def dJ(theta):
	"""损失函数的导函数 """
    return 2*(theta-2.5)

def J(theta):
	""" 损失函数"""
    return (theta-2.5)**2-1

def gradient_descent(initial_theta, eta, n_iters = 1e4, epsilon=1e-8):
	"""梯度下降 """
    # n_iters 最大循环次数
    theta = initial_theta
    theta_history.append(theta)
    i_iter = 0
    while i_iter < n_iters:
        gradient = dJ(theta) # 梯度
        last_theta = theta
        theta = theta - eta * gradient # 梯度下降
        theta_history.append(theta)
        if(abs(J(theta) - J(last_theta)) < epsilon):
        	# 如果下降到最低，就退出，epsilon是最低值，计算机的浮点型精度不准
            break
        i_iter += 1

def plot_theta_history():
	"""画图 """
    plt.plot(plot_x,J(plot_x))
    plt.plot(np.array(theta_history),J(np.array(theta_history)),color="r",marker=".")
    plt.show()

测试

eta =0.01
theta_history = []  # 自变量列表
gradient_descent(0.,eta)
plot_theta_history()

多元线性回归中使用梯度下降法：
2个参数的情况

MSE中使用梯度下降

代码实现：

# 模拟数据
np.random.seed(666)
x = 2 * np.random.random(size=100) # 一维向量
y = x * 3. + 4. + np.random.normal(size=100)

X=x.reshape(-1, 1)

class LinearRegression:
    def __init__(self):
        self.coef_ = None # 系数列表
        self.interception_ = None  # 截距
        self._theta = None 
        
    def fit(self,X_train,y_train, eta=0.01, n_iters=1e4):
        """根据训练数据集，训练模型"""
        def J(theta, X_b, y):
            try:
                return np.sum((y - X_b.dot(theta))**2) / len(X_b)
            except:
                return float('inf')
    
        def dJ(theta, X_b, y):
            res = np.empty(len(theta))
            res[0] = np.sum(X_b.dot(theta) - y)
            for i in range(1,len(theta)):
                res[i] = (X_b.dot(theta) - y).dot(X_b[:,i])
            return res * 2 / len(X_b)

        def gradient_descent(X_b, y, initial_theta, eta, n_iters = 1e4, epsilon=1e-8):
            """梯度下降 """
            # n_iters 最大循环次数
            theta = initial_theta

            i_iter = 0
            while i_iter < n_iters:
                gradient = dJ(theta, X_b, y)
                last_theta = theta
                theta = theta - eta * gradient

                if(abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon):
                    break
                i_iter += 1
            return theta
        
        X_b = np.hstack([np.ones((len(X_train),1)), X_train])
        initial_theta = np.zeros(X_b.shape[1])
        self._theta = gradient_descent(X_b, y_train, initial_theta, eta)
        self.coef_ = self._theta[1:]
        return self
    
    def predict(self,X_predict):
        """给定待遇测数据集，返回相应的结果向量"""
        X_b = np.hstack([np.ones((len(X_predict),1)),X_predict])
        return X_b.dot(self._theta)
    
    def score(self, X_test, y_test):
        y_predict = self.predict(X_test)        
        return r2_score(y_test,y_predict)

使用

lrg = LinearRegression()
lrg.fit(X, y)

向量化和标准化

即最终函数为：

代码实现：
将上面的代码中dJ函数修改为

def dJ(theta, X_b, y):
#             res = np.empty(len(theta))
#             res[0] = np.sum(X_b.dot(theta) - y)
#             for i in range(1,len(theta)):
#                 res[i] = (X_b.dot(theta) - y).dot(X_b[:,i])
            return X_b.T.dot(X_b.dot(theta) - y) * 2. / len(y)

在上面代码的基础上需要的额外的包

from sklearn.metrics import r2_score
from sklearn import datasets
from sklearn.model_selection import train_test_split as tts
from sklearn.preprocessing import StandardScaler

调用

boston=datasets.load_boston()
x=boston.data
y=boston.target
x=x[y<50.0]#去除不利
y=y[y<50.0]
X_train,X_test,y_train,y_test =tts(x,y,random_state=666)

standardScaler = StandardScaler()
standardScaler.fit(X_train)
X_train_standard = standardScaler.transform(X_train)

lrg=LinearRegression()
lrg.fit(X_train_standard, y_train)

X_test_standard = standardScaler.transform(X_test)
lrg.score(X_test_standard, y_test)

梯度下降算法的优势
速度比正规方程快

随机梯度下降法

当样本量非常大时，可用随机梯度下降法提升效率


设计学习率函数

这种思想叫模拟退火思想

代码实现
梯度函数

def dJ_sgd(theta, X_b_i, y_i):
	return X_b_i.T.dot(X_b_i.dot(theta) - y_i) * 2.

随机梯度下降

def sgd(X_b, y, initial_theta, n_iters):
	t0 = 5
	t1 = 50
	def learning_rate(t):
		return t0 / (t + t1)
	theta = initial_theta
	for cur_iter in range(n_iters):
		rand_i = np.random.randint(len(X_b))
		gradient = dJ_sgd(theta, X_b[rand_i], y[rand_i])
		theta = theta - learning_rate(cur_iter) * gradient
	return theta

最终代码：

class LinearRegression:
    def __init__(self):
        self.coef_ = None # 系数列表
        self.interception_ = None  # 截距
        self._theta = None 
        
    def fit(self,X_train,y_train, n_iters=100000):
        """根据训练数据集，训练模型"""
        def J(theta, X_b, y):
            try:
                return np.sum((y - X_b.dot(theta))**2) / len(X_b)
            except:
                return float('inf')
    
        def dJ_sgd(theta, X_b_i, y_i):
            return X_b_i.T.dot(X_b_i.dot(theta) - y_i) * 2.

        def sgd(X_b, y, initial_theta, n_iters):
            t0 = 5
            t1 = 50
            def learning_rate(t):
                return t0 / (t + t1)
            theta = initial_theta
            for cur_iter in range(n_iters):
                rand_i = np.random.randint(len(X_b))
                gradient = dJ_sgd(theta, X_b[rand_i], y[rand_i])
                theta = theta - learning_rate(cur_iter) * gradient
            return theta
        
        
        X_b = np.hstack([np.ones((len(X_train),1)), X_train])
        initial_theta = np.zeros(X_b.shape[1])
        self._theta = sgd(X_b, y_train, initial_theta, n_iters)
        self.coef_ = self._theta[1:]
        return self
    
    def predict(self,X_predict):
        """给定待遇测数据集，返回相应的结果向量"""
        X_b = np.hstack([np.ones((len(X_predict),1)),X_predict])
        return X_b.dot(self._theta)
    
    def score(self, X_test, y_test):
        y_predict = self.predict(X_test)        
        return r2_score(y_test,y_predict)

测试

boston=datasets.load_boston()
x=boston.data
y=boston.target
x=x[y<50.0]#去除不利
y=y[y<50.0]
X_train,X_test,y_train,y_test =tts(x,y,random_state=666)

standardScaler = StandardScaler()
standardScaler.fit(X_train)
X_train_standard = standardScaler.transform(X_train)

lrg=LinearRegression()
lrg.fit(X_train_standard, y_train)

X_test_standard = standardScaler.transform(X_test)
lrg.score(X_test_standard, y_test)

sklearn中使用随机梯度下降法

form sklearn.linear_model import SGDRegressor
sgd_reg = SGDRegressor(n_iters=100)
sgd_reg.fit(X_train_standard, y_train)
sgd_reg.score(X_test_standard, y_test)

总结