线性判别分析(Fisher)

线性判别分析(Fisher)

思想：类内小，类间大
$$\begin{gathered} 一个数据的投影：Z_i=w^T{x}_i \\ 投影的均值：\bar{Z}=\frac{1}{N}\sum _{i=1}^N{Z}_i=\frac{1}{N}\sum _{i=1}^N{w}^T{x}_i \\ {S}_z=\frac{1}{N}\sum _{i=1}^N(Z_i-\bar{Z})(Z_i-\bar{Z}{)}^T \\ =\frac{1}{N}\sum _{i=1}^N(w^T{x}_i-\bar{Z})(w^T{x}_i-\bar{Z}{)}^T对于两个类有： \\ {C}_1:\overline{Z_1}=\frac{1}{N_1}\sum _{i=1}^{N_1}{w}^T{x}_i{S}_1=\frac{1}{N_1}\sum _{i=1}^{N_1}(w^T{x}_i-{\bar{Z}}_1)(w^T{x}_i-{\bar{Z}}_1{)}^T \\ {C}_2:\overline{Z_2}=\frac{1}{N_2}\sum _{i=1}^{N_2}{w}^T{x}_i{S}_2=\frac{1}{N_2}\sum _{i=1}^{N_2}(w^T{x}_i-{\bar{Z}}_2)(w^T{x}_i-{\bar{Z}}_2{)}^T \end{gathered}$$
类间：$(\overline{Z_1}-\overline{Z_2}{)}^2$

类内：$S_1+S_2$

目标函数：$\frac{(\overline{Z_1}-\overline{Z_2}{)}^2}{S_1+S_2}$
$$\begin{gathered} (\overline{Z_1}-\overline{Z_2}{)}^2=(\frac{1}{N_1}\sum _{i=1}^N{w}^T{x}_i-\frac{1}{N_2}\sum _{i=1}^N{w}^T{x}_i{)}^2 \\ =[w^T(\frac{1}{N_1}\sum _{i=1}^N{x}_i-\frac{1}{N_2}\sum _{i=1}^N{x}_i){]}^2 \\ =[w^T({\bar{x}}_{c1}-{\bar{x}}_{c2}){]}^2 \\ =w^T({\bar{x}}_{c1}-{\bar{x}}_{c2})({\bar{x}}_{c1}-{\bar{x}}_{c2}{)}^{T}w \end{gathered}$$

$$\begin{gathered} S_1=\frac{1}{N_1}\sum _{i=1}^{N_1}(w^T{x}_i-{\bar{Z}}_1)(w^T{x}_i-{\bar{Z}}_1{)}^T \\ =\frac{1}{N_1}\sum _{i=1}^{N_1}[w^T(x_i-\frac{1}{N_1}\sum _{i=1}^{N_1}{x}_i)(x_i-\frac{1}{N_1}\sum _{i=1}^{N_1}{x}_i{)}^{T}w] \\ =w^T[\frac{1}{N_1}\sum _{i=1}^{N_1}(x_i-{\bar{x}}_{c1})(x_i-{\bar{x}}_{c1}{)}^T]w \\ =w^T{S}_{c1}w \\ 因此有：S_1+S_2=\frac{1}{N_1}\sum _{i=1}^N(w^T{x}_i-\bar{Z})(w^T{x}_i-\bar{Z}{)}^T+\frac{1}{N_2}\sum _{i=1}^N(w^T{x}_i-\bar{Z})(w^T{x}_i-\bar{Z}{)}^T \\ =w^T{S}_{c1}w+w^T{S}_{c2}w \\ =w^T(S_{c1}+S_{c2})w \end{gathered}$$

$$目标函数转换为：J(w)=\frac{w^T({\bar{x}}_{c1}-{\bar{x}}_{c2})({\bar{x}}_{c1}-{\bar{x}}_{c2}{)}^{T}w}{w^T(S_{c1}+S_{c2})w}$$

$$\begin{gathered} 我们用S_b表示为between-class:类间方差 \\ {S}_w表示为with-class:类内方差 \\ J(w)=\frac{w^T{S}_{b}w}{w^T{S}_{w}w} \\ \frac{\partial J(w)}{\partial w}=2S_{b}w(w^T{S}_{w}w{)}^{-1}+w^T{S}_{b}w(-1)(w^T{S}_{w}w{)}^{-2}\ast 2S_{w}w=0 \\ 于是我们得到：S_{b}w(w^T{S}_{w}w)-w^T{S}_{b}w{S}_{w}w=0(其中w^T{S}_{w}w、w^T{S}_{b}w都是1\ast 1维的常数) \\ {S}_{w}w=\frac{w^T{S}_{w}w}{w^T{S}_{b}w}{S}_{b}w \\ w=\frac{w^T{S}_{w}w}{w^T{S}_{b}w}{S}_w^{-1}{S}_{b}w \\ \propto S_w^{-1}{S}_{b}w=S_w^{-1}({\bar{x}}_{c1}-{\bar{x}}_{c2})({\bar{x}}_{c1}-{\bar{x}}_{c2}{)}^{T}w(其中({\bar{x}}_{c1}-{\bar{x}}_{c2}{)}^{T}w是一个1\ast 1的常数) \\ \propto S_w^{-1}({\bar{x}}_{c1}-{\bar{x}}_{c2}) \end{gathered}$$

因此$w$的方向就是由$S_w^{-1}({\bar{x}}_{c1}-{\bar{x}}_{c2})$确定