ProbabilityModeling.tex

\section*{Probability modeling}
Find $h:X\rightarrow Y$ that min. pred. error: 
$R(h) = \int P(x,y)l(y;h(x)) \partial yx \partial y = \mathbb{E}_{x,y}[l(y;h(x))]$

\subsection*{For least squares regression}
Best $h$: $h^*(x) = \mathbb{E}[Y|X=x]$ \\
Pred.: $\hat{y} = \hat{\mathbb{E}}[Y|X=\hat{x}] = \int \hat{P}(y|X=\hat{x}) y \partial y$

\subsection*{Maximum Likelihood Estimation (MLE)}
$\theta^* = \underset{\theta}{\operatorname{argmax}} ~ \hat{P}(y_1,...,y_n|x_1,...,x_n,\theta)$\\
E.g. lin. + Gauss: $y_i = w^T x_i + \varepsilon_i, \varepsilon_i \sim \mathcal{N}(0, \sigma^2)$\\
i.e. $y_i \sim \mathcal{N}(w^T x_i, \sigma^2)$, With MLE (use\\ $\operatorname{argmin} ~ - \operatorname{log}$): $w^* = \underset{w}{\operatorname{argmin}} \sum (y_i-w^Tx_i)^2$

\subsection*{Bias/Variance/Noise}
Prediction error = $Bias^2 + Variance + Noise$

\subsection*{Maximum a posteriori estimate (MAP)}
Assume bias on parameters, e.g. $w_i \in \mathcal{N}(0, \beta^2)$\\
Bay.: $P(w|x,y) = \frac{P(w|x) P(y|x,w)}{P(y|x)} = \frac{P(w) P(y|x,w)}{P(y|x)}$

\subsection*{Logistic regression}
Link func.: $\sigma(w^Tx) = \frac{1}{1+exp(-w^Tx)}$ (Sigmoid)\\
$P(y|x,w) = Ber(y; \sigma(w^Tx)) = \frac{1}{1+exp(-y w^T x)}$
Classification: Use $P(y|x,w)$, predict most likely class label.\\
MLE: $\underset{w}{\operatorname{argmax}} ~ P(y_{1:n}|w,x_{1:n})\\
\Rightarrow w^* = \underset{w}{\operatorname{argmin}} \sum_{i=1}^n log(1+exp(-y_i w^T x_i))$\\
SGD update: $w = w + \eta_t y x \hat{P}(Y = -y|w,x)$\\
$\hat{P}(Y = -y|w,x) = \frac{1}{1+exp(yw^Tx)}$\\
MAP: Gauss. prior $\Rightarrow ||w||_2^2$, Lap. p. $\Rightarrow||w||_1$\\
SGD: $w = w (1-2\lambda \eta_t) + \eta_t y x \hat{P}(Y = -y|w,x)$