机器学习5 高斯判别法

混合高斯模型参数推导

最新推荐文章于 2022-09-01 19:42:59 发布

原创最新推荐文章于 2022-09-01 19:42:59 发布 · 1.0k 阅读

0 ·

CC 4.0 BY-SA版权

machine_learning 专栏收录该内容

9 篇文章

订阅专栏

如果
$y$ ~ $Bernoulli(\phi )$
$x|y=1$ ~ $N({\mu _0},\Sigma )$
$x|y=0$ ~ $N({\mu _1},\Sigma )$
则
$p(y) = {\phi ^y}{(1 - \phi )^{1 - y}}$
$p(x|y = 1) = {1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}\exp \left( { - {1 \over 2}{{\left( {x - {\mu _0}} \right)}^T}{\Sigma ^{ - 1}}\left( {x - {\mu _0}} \right)} \right) = {\left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}\exp \left( { - {1 \over 2}{{\left( {x - {\mu _0}} \right)}^T}{\Sigma ^{ - 1}}\left( {x - {\mu _0}} \right)} \right)} \right)^y}$
$p(x|y = 0) = {1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}\exp \left( { - {1 \over 2}{{\left( {x - {\mu _1}} \right)}^T}{\Sigma ^{ - 1}}\left( {x - {\mu _1}} \right)} \right) = {\left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}\exp \left( { - {1 \over 2}{{\left( {x - {\mu _1}} \right)}^T}{\Sigma ^{ - 1}}\left( {x - {\mu _1}} \right)} \right)} \right)^{1 - y}}$
$l\left( {\phi ,{\mu _0},{\mu _1},\Sigma } \right) = \log \prod\limits_{i = 1}^m {p\left( {{x^{(i)}},{y^{\left( i \right)}};\phi ,{\mu _0},{\mu _1},\Sigma } \right)}$
$= \log \prod\limits_{i = 1}^m {p\left( {{x^{(i)}}|{y^{\left( i \right)}};{\mu _0},{\mu _1},\Sigma } \right)} p\left( {{y^{\left( i \right)}};\phi } \right)$
$= \log \prod\limits_{i = 1}^m {{{\left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}\exp \left( { - {1 \over 2}{{\left( {{x^{(i)}} - {\mu _0}} \right)}^T}{\Sigma ^{ - 1}}\left( {{x^{(i)}} - {\mu _0}} \right)} \right)\phi } \right)}^{{y^{(i)}}}}{{\left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}\exp \left( { - {1 \over 2}{{\left( {{x^{(i)}} - {\mu _1}} \right)}^T}{\Sigma ^{ - 1}}\left( {{x^{(i)}} - {\mu _1}} \right)} \right)(1 - \phi )} \right)}^{1 - {y^{(i)}}}}}$
$= \sum\limits_{i = 1}^m {{y^{(i)}}\log \left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}\exp \left( { - {1 \over 2}{{\left( {{x^{(i)}} - {\mu _0}} \right)}^T}{\Sigma ^{ - 1}}\left( {{x^{(i)}} - {\mu _0}} \right)} \right)\phi } \right) + } \left( {1 - {y^{(i)}}} \right)\log \left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}\exp \left( { - {1 \over 2}{{\left( {{x^{(i)}} - {\mu _1}} \right)}^T}{\Sigma ^{ - 1}}\left( {{x^{(i)}} - {\mu _1}} \right)} \right)(1 - \phi )} \right)$
$= \sum\limits_{i = 1}^m {{y^{(i)}}\left( { - {1 \over 2}{{\left( {{x^{(i)}} - {\mu _0}} \right)}^T}{\Sigma ^{ - 1}}\left( {{x^{(i)}} - {\mu _0}} \right) + \log \left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}} \right) + \log \left( \phi \right)} \right) + } \left( {1 - {y^{(i)}}} \right)\left( { - {1 \over 2}{{\left( {{x^{(i)}} - {\mu _1}} \right)}^T}{\Sigma ^{ - 1}}\left( {{x^{(i)}} - {\mu _1}} \right) + \log \left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}} \right) + \log \left( {1 - \phi } \right)} \right)$
$= \sum\limits_{i = 1}^m {\log \left( {{1 \over {{{\left( {2\pi } \right)}^{{n \over 2}}}{{\left| \Sigma \right|}^{{1 \over 2}}}}}} \right) + \log \left( {1 - \phi } \right) + {y^{\left( i \right)}}\left( {\log \left( \phi \right) - \log \left( {1 - \phi } \right)} \right) + {y^{(i)}}\left( { - {1 \over 2}{{\left( {{x^{(i)}} - {\mu _0}} \right)}^T}{\Sigma ^{ - 1}}\left( {{x^{(i)}} - {\mu _0}} \right)} \right) + } \left( {1 - {y^{(i)}}} \right)\left( { - {1 \over 2}{{\left( {{x^{(i)}} - {\mu _1}} \right)}^T}{\Sigma ^{ - 1}}\left( {{x^{(i)}} - {\mu _1}} \right)} \right)$
$\Rightarrow$
${{\partial l} \over {\partial \phi }} = \sum\limits_{i = 1}^m {{{ - 1} \over {1 - \phi }} + } {y^{\left( i \right)}}\left( {{1 \over {\phi \left( {1 - \phi } \right)}}} \right)$
$= \sum\limits_{i = 1}^m {{1 \over {1 - \phi }}\left( { - 1 + {y^{\left( i \right)}}{1 \over \phi }} \right)}$
$= {1 \over {1 - \phi }}\left( { - m + {{\sum\limits_{i = 1}^m {\left( {{y^{\left( i \right)}}} \right)} } \over \phi }} \right) = = 0$
$\Rightarrow \phi = {{\sum\limits_{i = 1}^m {\left( {{y^{\left( i \right)}}} \right)} } \over m}$
先证
$d = {\left( {A - B} \right)^T}C\left( {A - B} \right)$
$\eqalign{ & {\nabla _B}d = {\nabla _B}tr\left( d \right) = {\nabla _B}tr\left( { - {A^T}CB - {B^T}CA + {B^T}CB} \right) \cr & = \left( { - {C^T}A - CA + CB + {C^T}B} \right) \cr}$
$\eqalign{ & f{\kern 1pt} {\kern 1pt} {\kern 1pt} C = = {C^T} \cr & {\nabla _B}d = 2CB - 2CA \cr}$

\Rightarrow \partial l \partial μ 0 = \sum i = 1 m y (i) Σ - 1 (- μ 0 + x (i)) = Σ - 1 (\sum i = 1 m y (i) x (i) - \sum i = 1 m y (i) μ 0) \Rightarrow μ 0 = \sum i = 1 m y ( i ) x ( i ) \sum i = 1 m y ( i ) \partial l \partial μ 1 = \sum i = 1 m (1 - y (i)) Σ - 1 (- μ 1 + x (i)) \Rightarrow μ 1 = \sum i = 1 m ( 1 - y ( i ) ) x ( i ) \sum i = 1 m ( 1 - y ( i ) )

$\eqalign{ & \Rightarrow \cr & {{\partial l} \over {\partial {\mu _0}}} = \sum\limits_{i = 1}^m {{y^{(i)}}{\Sigma ^{ - 1}}\left( { - {\mu _0} + {x^{\left( i \right)}}} \right)} \cr & = {\Sigma ^{ - 1}}\left( {\sum\limits_{i = 1}^m {{y^{(i)}}{x^{\left( i \right)}}} - \sum\limits_{i = 1}^m {{y^{(i)}}{\mu _0}} } \right) \cr & \Rightarrow \cr & {\mu _0} = {{\sum\limits_{i = 1}^m {{y^{(i)}}{x^{\left( i \right)}}} } \over {\sum\limits_{i = 1}^m {{y^{(i)}}} }} \cr & \cr & {{\partial l} \over {\partial {\mu _1}}} = \sum\limits_{i = 1}^m {\left( {1 - {y^{(i)}}} \right){\Sigma ^{ - 1}}\left( { - {\mu _1} + {x^{\left( i \right)}}} \right)} \cr & \Rightarrow \cr & {\mu _1} = {{\sum\limits_{i = 1}^m {\left( {1 - {y^{(i)}}} \right){x^{\left( i \right)}}} } \over {\sum\limits_{i = 1}^m {\left( {1 - {y^{(i)}}} \right)} }} \cr}$

\nabla Σ - 1 l = \sum i = 1 m 1 2 ∣ ∣ Σ - 1 ∣ ∣ \nabla Σ - 1 ∣ ∣ Σ - 1 ∣ ∣ + y (i) (- 1 2 (x (i) - μ 0) (x (i) - μ 0) T) + (1 - y (i)) (- 1 2 (x (i) - μ 1) (x (i) - μ 1) T) = m Σ 2 + \sum i = 1 m y (i) (- 1 2 (x (i) - μ 0) (x (i) - μ 0) T) + (1 - y (i)) (- 1 2 (x (i) - μ 1) (x (i) - μ 1) T) \Rightarrow Σ = \sum i = 1 m y ( i ) ( - 1 2 ( x ( i ) - μ 0 ) ( x ( i ) - μ 0 ) T ) + ( 1 - y ( i ) ) ( - 1 2 ( x ( i ) - μ 1 ) ( x ( i ) - μ 1 ) T ) m

$\eqalign{ & {\nabla _{{\Sigma ^{ - 1}}}}l = \sum\limits_{i = 1}^m {{1 \over {2\left| {{\Sigma ^{ - 1}}} \right|}}{\nabla _{{\Sigma ^{ - 1}}}}\left| {{\Sigma ^{ - 1}}} \right| + {y^{(i)}}\left( { - {1 \over 2}\left( {{x^{(i)}} - {\mu _0}} \right){{\left( {{x^{(i)}} - {\mu _0}} \right)}^T}} \right) + \left( {1 - {y^{(i)}}} \right)\left( { - {1 \over 2}\left( {{x^{(i)}} - {\mu _1}} \right){{\left( {{x^{(i)}} - {\mu _1}} \right)}^T}} \right)} \cr & = {{m\Sigma } \over 2} + \sum\limits_{i = 1}^m {{y^{(i)}}\left( { - {1 \over 2}\left( {{x^{(i)}} - {\mu _0}} \right){{\left( {{x^{(i)}} - {\mu _0}} \right)}^T}} \right) + \left( {1 - {y^{(i)}}} \right)\left( { - {1 \over 2}\left( {{x^{(i)}} - {\mu _1}} \right){{\left( {{x^{(i)}} - {\mu _1}} \right)}^T}} \right)} \Rightarrow \Sigma = {{\sum\limits_{i = 1}^m {{y^{(i)}}\left( { - {1 \over 2}\left( {{x^{(i)}} - {\mu _0}} \right){{\left( {{x^{(i)}} - {\mu _0}} \right)}^T}} \right) + \left( {1 - {y^{(i)}}} \right)\left( { - {1 \over 2}\left( {{x^{(i)}} - {\mu _1}} \right){{\left( {{x^{(i)}} - {\mu _1}} \right)}^T}} \right)} } \over m} \cr}$