3.线性模型——逻辑回归

数据集: T={(x⃗1,y1),(x⃗2,y2),⋯(x⃗N,yN)}T = \{(\vec{x}_1,y_1),(\vec{x}_2,y_2),\cdots(\vec{x}_{N},y_N)\}T={(x1​,y1​),(x2​,y2​),⋯(xN​,yN​)}
xi⃗∈X⊆ Rn,yi∈Y={0,1}\qquad\quad\vec{x_i} \in \mathcal{X} \subseteq\ \mathbb{R^n},y_i \in \mathcal{Y}=\{0,1\}xi​​∈X⊆ Rn,yi​∈Y={0,1}
求解已知 $\vec{x}$ 对应的 $y$，即：$P(y\mid \vec{x})$ ，也即：$P(y=1\mid \vec{x})$

对数概率函数：
$$P(y=1\mid \vec{x})=\frac{1}{1+e^{-({\vec{w}}^T\cdot \vec{x}+b)}}$$记：$\overrightarrow{\stackrel{~}{w}}=(\vec{w},b{)}^T$
$\overrightarrow{\stackrel{~}{x}}=(\vec{x},1{)}^T$
$$D(\overrightarrow{\stackrel{~}{x}})=P(y=1\mid \overrightarrow{\stackrel{~}{x}})=\frac{1}{1+e^{-({\overrightarrow{\stackrel{~}{w}}}^T\cdot \overrightarrow{\stackrel{~}{x}})}}=\frac{e^{{\overrightarrow{\stackrel{~}{w}}}^T\cdot \overrightarrow{\stackrel{~}{x}}}}{1+e^{{\overrightarrow{\stackrel{~}{w}}}^T\cdot \overrightarrow{\stackrel{~}{x}}}}$$同理:$$1-D(\overrightarrow{\stackrel{~}{x}})=P(y=0\mid \overrightarrow{\stackrel{~}{x}})=\frac{1}{1+e^{{\overrightarrow{\stackrel{~}{w}}}^T\cdot \overrightarrow{\stackrel{~}{x}}}}$$似然函数：
S(w~⃗∣X)=∏i=1N[D(x~⃗i)]yi[1−D(x~⃗i)]1−yi,yi∈{0,1}S(\vec{\tilde{w}}\mid X) = \prod_{i=1}^{N} [D(\vec{\tilde{x}}_i)]^{y_i}[1-D(\vec{\tilde{x}}_i)]^{_1- y_i}\hspace{0.1cm}_,\hspace{0.1cm}y_i\in\{0,1\}S(w~∣X)=i=1∏N​[D(x~i​)]yi​[1−D(x~i​)]1​−yi​,​yi​∈{0,1}对数似然函数：
$$L(\overrightarrow{\stackrel{~}{w}})=\log S(\overrightarrow{\stackrel{~}{w}}\mid X)$$$$\Rightarrow L(\overrightarrow{\stackrel{~}{w}})=\sum _i^N\left[y_i\log D({\overrightarrow{\stackrel{~}{x}}}_i)+(1-y_i)\log (1-D({\overrightarrow{\stackrel{~}{x}}}_i))\right]$$$$\Rightarrow L(\overrightarrow{\stackrel{~}{w}})=\sum _i^N\left[y_i\log \frac{D({\overrightarrow{\stackrel{~}{x}}}_i)}{1-D({\overrightarrow{\stackrel{~}{x}}}_i)}+\log (1-D({\overrightarrow{\stackrel{~}{x}}}_i))\right]$$$$\Rightarrow L(\overrightarrow{\stackrel{~}{w}})=\sum _i^N\left[y_i({\overrightarrow{\stackrel{~}{w}}}^T\cdot {\overrightarrow{\stackrel{~}{x}}}_i)-\log (1+e^{{\overrightarrow{\stackrel{~}{w}}}^T\cdot {\overrightarrow{\stackrel{~}{x}}}_i})\right]$$

极大似然估计法：
$${\overrightarrow{\stackrel{~}{w}}}^{\ast }=\arg \underset{{\overrightarrow{\stackrel{~}{w}}}^T}{\max }\sum _i^N\left[y_i({\overrightarrow{\stackrel{~}{w}}}^T\cdot {\overrightarrow{\stackrel{~}{x}}}_i)-\log (1+e^{{\overrightarrow{\stackrel{~}{w}}}^T\cdot {\overrightarrow{\stackrel{~}{x}}}_i})\right]$$

梯度下降法：

$$最大化：L(\overrightarrow{\stackrel{~}{w}})=\sum _i^N\left[y_i\log D({\overrightarrow{\stackrel{~}{x}}}_i)+(1-y_i)\log (1-D({\overrightarrow{\stackrel{~}{x}}}_i))\right]$$$$令：J(\overrightarrow{\stackrel{~}{w}})=-\frac{1}{N}L(\overrightarrow{\stackrel{~}{w}})$$$$最小化：J(\overrightarrow{\stackrel{~}{w}})=-\frac{1}{N}\sum _i^N\left[y_i\log D({\overrightarrow{\stackrel{~}{x}}}_i)+(1-y_i)\log (1-D({\overrightarrow{\stackrel{~}{x}}}_i))\right]$$$$\frac{\partial J(\overrightarrow{\stackrel{~}{w}})}{\partial {\overrightarrow{\stackrel{~}{w}}}_j}=-\frac{1}{N}\sum _i^N\left[y_i\frac{1}{D({\overrightarrow{\stackrel{~}{x}}}_i)}\frac{\partial D({\overrightarrow{\stackrel{~}{x}}}_i)}{\partial {\overrightarrow{\stackrel{~}{w}}}_j}-(1-y_i)\frac{1}{1-D({\overrightarrow{\stackrel{~}{x}}}_i)}\frac{\partial D({\overrightarrow{\stackrel{~}{x}}}_i)}{\partial {\overrightarrow{\stackrel{~}{w}}}_j}\right]$$$$=-\frac{1}{N}\sum _i^N\left(y_i\frac{1}{D({\overrightarrow{\stackrel{~}{x}}}_i)}-(1-y_i)\frac{1}{1-D({\overrightarrow{\stackrel{~}{x}}}_i)}\right)\frac{\partial D({\overrightarrow{\stackrel{~}{x}}}_i)}{\partial {\overrightarrow{\stackrel{~}{w}}}_j}$$$$=-\frac{1}{N}\sum _i^N\left(y_i\frac{1}{D({\overrightarrow{\stackrel{~}{x}}}_i)}-(1-y_i)\frac{1}{1-D({\overrightarrow{\stackrel{~}{x}}}_i)}\right)D({\overrightarrow{\stackrel{~}{x}}}_i)(1-D({\overrightarrow{\stackrel{~}{x}}}_i))\frac{\partial ({\overrightarrow{\stackrel{~}{w}}}^T\cdot \overrightarrow{\stackrel{~}{x}})}{\partial {\overrightarrow{\stackrel{~}{w}}}_j}$$$$=-\frac{1}{N}\sum _i^N\left(y_i(1-D({\overrightarrow{\stackrel{~}{x}}}_i))-(1-y_i)D({\overrightarrow{\stackrel{~}{x}}}_i)\right)\frac{\partial ({\overrightarrow{\stackrel{~}{w}}}^T\cdot \overrightarrow{\stackrel{~}{x}})}{\partial {\overrightarrow{\stackrel{~}{w}}}_j}$$$$=-\frac{1}{N}\sum _i^N\left(y_i-D({\overrightarrow{\stackrel{~}{x}}}_i)\right){\overrightarrow{\stackrel{~}{x}}}_i^j$$$$=\frac{1}{N}\sum _i^N\left(D({\overrightarrow{\stackrel{~}{x}}}_i)-y_i\right){\overrightarrow{\stackrel{~}{x}}}_i^j$$ $$\Rightarrow \frac{1}{N}(D(\tilde{X})-\vec{y}){\tilde{X}}_{\cdot j}$$最后：$${\overrightarrow{\stackrel{~}{w}}}_j={\overrightarrow{\stackrel{~}{w}}}_j-\frac{1}{N}\eta (D(\tilde{X})-\vec{y}{)}^T{\tilde{X}}_{\cdot j}$$$$其中，D(\tilde{X})=\frac{e^{\tilde{X}\cdot \overrightarrow{\stackrel{~}{w}}}}{1+e^{\tilde{X}\cdot \overrightarrow{\stackrel{~}{w}}}}$$

python实例-1

import numpy as np
import matplotlib.pyplot as plt

def computeCost(X, y, w_e):
    return 1.0 / (2 * len(y)) * np.dot((np.dot(X, w_e) - y).T, (np.dot(X, w_e) - y))

def D(X,w_e):
    return np.exp(np.dot(X, w_e)) / (1 + np.exp(-np.dot(X, w_e)))


def gradientDescent(X, y, w_e, alpha):
    for j in range(len(w_e)):
        w_e[j] = w_e[j] - alpha * 1.0 / len(y) * np.dot((D(X,w_e) - y).T, X[:, j])
    return w_e

X = np.random.rand(10, 5)
m = np.ones((10, 1))
X = np.concatenate((X, m), axis=1)
w = np.array([[1.0], [2.0], [3.0], [4.0], [5.0], [10.0]])
y = np.dot(X, w)
a = np.mean(y, axis=0)
for i in range(10):
    if y[i][0] > a:
        y[i][0] = 1
    else: y[i][0] = 0

w_e = np.zeros_like(w)
cost = []
for i in range(10000):
    w_e = gradientDescent(X, y, w_e, 0.001)
    cost.append(computeCost(X, y, w_e)[0, 0])

print(y)
print(D(X,w_e))


fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax1.plot(range(10000), cost,label='lost')
ax1.set_yscale('log')
ax1.set_xlabel("Iteration")
ax1.set_ylabel("Loss")
ax1.legend(loc='best')
plt.show()

[[1.]
 [0.]
 [1.]
 [0.]
 [0.]
 [1.]
 [0.]
 [1.]
 [1.]
 [1.]]
[[0.63103198]
 [0.54273564]
 [0.72331236]
 [0.55213718]
 [0.41564281]
 [0.78018675]
 [0.49371131]
 [0.59834743]
 [0.67121118]
 [0.86004243]]

损失函数：

增加至迭代30000轮数

import numpy as np
import matplotlib.pyplot as plt

def computeCost(X, y, w_e):
    return 1.0 / (2 * len(y)) * np.dot((np.dot(X, w_e) - y).T, (np.dot(X, w_e) - y))

def D(X,w_e):
    return np.exp(np.dot(X, w_e)) / (1 + np.exp(-np.dot(X, w_e)))


def gradientDescent(X, y, w_e, alpha):
    for j in range(len(w_e)):
        w_e[j] = w_e[j] - alpha * 1.0 / len(y) * np.dot((D(X,w_e) - y).T, X[:, j])
    return w_e

X = np.random.rand(10, 5)
m = np.ones((10, 1))
X = np.concatenate((X, m), axis=1)
w = np.array([[1.0], [2.0], [3.0], [4.0], [5.0], [10.0]])
y = np.dot(X, w)
a = np.mean(y, axis=0)
for i in range(10):
    if y[i][0] > a:
        y[i][0] = 1
    else: y[i][0] = 0

w_e = np.zeros_like(w)
cost = []
for i in range(30000):
    w_e = gradientDescent(X, y, w_e, 0.001)
    cost.append(computeCost(X, y, w_e)[0, 0])

print(y)
print(D(X,w_e))


fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax1.plot(range(30000), cost,label='lost')
ax1.set_yscale('log')
ax1.set_xlabel("Iteration")
ax1.set_ylabel("Loss")
ax1.legend(loc='best')
plt.show()

损失有点上升！

可以接着减小学习率

import numpy as np
import matplotlib.pyplot as plt

def computeCost(X, y, w_e):
    return 1.0 / (2 * len(y)) * np.dot((np.dot(X, w_e) - y).T, (np.dot(X, w_e) - y))

def D(X,w_e):
    return np.exp(np.dot(X, w_e)) / (1 + np.exp(-np.dot(X, w_e)))


def gradientDescent(X, y, w_e, alpha):
    for j in range(len(w_e)):
        w_e[j] = w_e[j] - alpha * 1.0 / len(y) * np.dot((D(X,w_e) - y).T, X[:, j])
    return w_e

X = np.random.rand(10, 5)
m = np.ones((10, 1))
X = np.concatenate((X, m), axis=1)
w = np.array([[1.0], [2.0], [3.0], [4.0], [5.0], [10.0]])
y = np.dot(X, w)
a = np.mean(y, axis=0)
for i in range(10):
    if y[i][0] > a:
        y[i][0] = 1
    else: y[i][0] = 0

w_e = np.zeros_like(w)
cost = []
for i in range(30000):
    w_e = gradientDescent(X, y, w_e, 0.0001)
    cost.append(computeCost(X, y, w_e)[0, 0])

print(y)
print(D(X,w_e))

fig = plt.figure()
ax1 = fig.add_subplot(2, 1, 1)
ax1.plot(range(30000), cost,label='lost')
ax1.set_yscale('log')
ax1.set_xlabel("Iteration")
ax1.set_ylabel("Loss")
ax1.legend(loc='best')
plt.show()