diff --git a/.gitattributes b/.gitattributes index defcde4..b4a264a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,7 @@ main.pdf filter=lfs diff=lfs merge=lfs -text +figures/schemes/regression_plan_3D.pdf filter=lfs diff=lfs merge=lfs -text +figures/schemes/vector_orthogonality.pdf filter=lfs diff=lfs merge=lfs -text +figures/schemes/base_plan.pdf filter=lfs diff=lfs merge=lfs -text +figures/schemes/coordinates_systems.pdf filter=lfs diff=lfs merge=lfs -text +figures/schemes/ordinary_least_squares.pdf filter=lfs diff=lfs merge=lfs -text +figures/schemes/orthogonal_projection.pdf filter=lfs diff=lfs merge=lfs -text diff --git a/content/chapters/part1/1.tex b/content/chapters/part1/1.tex index e31f254..8219935 100644 --- a/content/chapters/part1/1.tex +++ b/content/chapters/part1/1.tex @@ -129,6 +129,13 @@ $\X \hat{\beta}$ is the closest point to $\Y$ in the subspace generated by $\X$. If $H$ is the projection matrix of the subspace generated by $\X$, $X\Y$ is the projection on $\Y$ on this subspace, that corresponds to $\X\hat{\beta}$. +\section{Sum of squares} + +$\Y - \X \hat{\beta} \perp \X \hat{\beta} - \Y \One$ if $\One \in V$, so +\[ + \underbrace{\norm{\Y - \bar{\Y}\One}}_{\text{Total SS}} = \underbrace{\norm{\Y - \X \hat{\beta}}^2}_{\text{Residual SS}} + \underbrace{\norm{\X \hat{\beta} - \bar{\Y} \One}^2}_{\text{Explicated SS}} +\] + \section{Coefficient of Determination: \texorpdfstring{$R^2$}{R\textsuperscript{2}}} \begin{definition}[$R^2$] \[ @@ -139,7 +146,7 @@ If $H$ is the projection matrix of the subspace generated by $\X$, $X\Y$ is the \begin{figure} \centering \includestandalone{figures/schemes/orthogonal_projection} - \caption{Orthogonal projection of $\Y$ on plan generated by the base described by $\X$. $\color{blue}a$ corresponds to $\norm{\X\hat{\beta} - \bar{\Y}}^2$ and $\color{blue}b$ corresponds to $\norm{\Y - \hat{\beta}\X}^2$} + \caption{Orthogonal projection of $\Y$ on plan generated by the base described by $\X$. $\color{blue}a$ corresponds to $\norm{\X\hat{\beta} - \bar{\Y}}^2$ and $\color{blue}b$ corresponds to $\hat{\varepsilon} = \norm{\Y - \hat{\beta}\X}^2$} and $\color{blue}c$ corresponds to $\norm{Y - \bar{Y}}^2$. \label{fig:scheme-orthogonal-projection} \end{figure} @@ -149,3 +156,165 @@ If $H$ is the projection matrix of the subspace generated by $\X$, $X\Y$ is the \caption{Ordinary least squares and regression line with simulated data.} \label{fig:ordinary-least-squares} \end{figure} + + + + +\begin{definition}[Model dimension] + Let $\M$ be a model. + The dimension of $\M$ is the dimension of the subspace generated by $\X$, that is the number of parameters in the $\beta$ vector. + + \textit{Nb.} The dimension of the model is not the number of parameter, as $\sigma^2$ is one of the model parameters. +\end{definition} + +\section{Gaussian vectors} + + +\begin{definition}[Normal distribution] + +\end{definition} + + +\begin{definition}[Gaussian vector] + A random vector $\Y \in \RR[n]$ is a gaussian vector if every linear combination of its component is ... +\end{definition} + +\begin{property} + $m = \EE(Y) = (m_1, \ldots, m_n)^T$, where $m_i = \EE(Y_i)$ + + + ... + + \[ + \Y \sim \Norm_n(m, \Sigma) + \] + where $\Sigma$ is the variance-covariance matrix! + \[ + \Sigma = \E\left[(\Y -m)(\Y - m)^T\right]. + \] + + + +\end{property} + +\begin{remark} + \[ + \Cov(Y_i, Y_i) = \Var(Y_i) + \] +\end{remark} + +\begin{definition}[Covariance] + \[ + \Cov(Y_i, Y_j) = \EE\left((Y_i-\EE(Y_j))(Y_j-\EE(Y_j))\right) + \] +\end{definition} + + +When two variable are linked, the covariance is large. + +If two variables $X, Y$ are independent, $\Cov(X, Y) = 0$. + +\begin{definition}[Correlation coefficient] + \[ + \Cor(Y_i, Y_j) = \frac{\EE\left((Y_i-\EE(Y_j))(Y_j-\EE(Y_j))\right)}{\sqrt{\EE(Y_i - \EE(Y_i)) \cdot \EE(Y_j - \EE(Y_j))}} + \] +\end{definition} + +Covariance is really sensitive to scale of variables. For instance, if we measure distance in millimeters, the covariance would be larger than in the case of a measure expressed in metters. Thus the correlation coefficient, which is a sort of normalized covariance is useful, to be able to compare the values. + +\begin{remark} + \begin{align*} + \Cov(Y_i, Y_i) &= \EE((Y_i - \EE(Y_i)) (Y_i - \EE(Y_i))) \\ + &= \EE((Y_i - \EE(Y_i))^2) \\ + &= \Var(Y_i) + \end{align*} +\end{remark} + +\begin{equation} + \Sigma = \begin{pNiceMatrix} + \VVar(Y_1) & & & &\\ + & \Ddots & & & \\ + & \Cov(Y_i, Y_j) & \VVar(Y_i) & & \\ + & & & \Ddots & \\ + & & & & \VVar(Y_n) + \end{pNiceMatrix} +\end{equation} + +\begin{definition}[Identity matrix] + \[ + \mathcal{I}_n = \begin{pNiceMatrix} + 1 & 0 & 0 \\ + 0 & \Ddots & 0\\ + 0 & 0 & 1 + \end{pNiceMatrix} + \] + +\end{definition} + + +\begin{theorem}[Cochran Theorem (Consequence)] + Let $\mathbf{Z}$ be a gaussian vector: $\mathbf{Z} \sim \Norm_n(0_n, I_n)$. + + \begin{itemize} + \item If $V_1, V_n$ are orthogonal subspaces of $\RR[n]$ with dimensions $n_1, n_2$ such that + \[ + \RR[n] = V_1 \overset{\perp}{\oplus} V_2. + \] + \item If $Z_1, Z_2$ are orthogonal of $\mathbf{Z}$ on $V_1$ and $V_2$ i.e. $Z_1 = \Pi_{V_1}(\mathbf{Z}) = \Pi_1 \Y$ and $Z_2 = \Pi_{V_2} (\mathbf{Z}) = \Pi_2 \Y$... + (\textcolor{red}{look to the slides}) + \end{itemize} +\end{theorem} + +\begin{definition}[Chi 2 distribution] + If $X_1, \ldots, X_n$ i.i.d. $\sim \Norm(0, 1)$, then;, + \[ + X_1^2 + \ldots X_n^2 \sim \chi_n^2 + \] +\end{definition} + +\subsection{Estimator's properties} + + +\[ + \Pi_V = \X(\X^T\X)^{-1} \X^T +\] + +\begin{align*} + \hat{m} &= \X \hat{\beta} = \X(\X^T\X)^{-1} \X^T \Y \\ + \text{so} \\ + &= \Pi_V \Y +\end{align*} + +According to Cochran theorem, we can deduce that the estimator of the predicted value $\hat{m}$ is independent $\hat{\sigma}^2$ + +All the sum of squares follows a $\chi^2$ distribution: +\[ + ... +\] + +\begin{property} + +\end{property} + +\subsection{Estimators consistency} + +If $q < n$, +\begin{itemize} + \item $\hat{\sigma}^2 \overunderset{\PP}{n\to\infty} \sigma^{*2}$. + \item If $(\X^T\X)^{-1}$... + \item ... +\end{itemize} + +We can derive statistical test from these properties. + + +\section{Statistical tests} + +\subsection{Student $t$-test} + + +\[ + \frac{\hat{\theta}-\theta}{\sqrt{\frac{\widehat{\VVar}(\hat{\theta})}{n}}} \underset{H_0}{\sim} t +\] + +where diff --git a/definitions.tex b/definitions.tex index e69de29..9387d0c 100644 --- a/definitions.tex +++ b/definitions.tex @@ -0,0 +1,6 @@ +\DeclareMathOperator{\VVar}{\mathbb{V}} % variance +\DeclareMathOperator{\One}{\mathbf{1}} +\DeclareMathOperator{\Cor}{\mathrm{Cor}} +\newcommand{\M}[1][]{\ensuremath{\ifstrempty{#1}{\mathcal{M}}{\mathbb{M}_{#1}}}} +\newcommand{\X}{\ensuremath{\mathbf{X}}} +\newcommand{\Y}{\ensuremath{\mathbf{Y}}} diff --git a/figures/schemes/base_plan.pdf b/figures/schemes/base_plan.pdf index 04d165f..b5c74c0 100644 Binary files a/figures/schemes/base_plan.pdf and b/figures/schemes/base_plan.pdf differ diff --git a/figures/schemes/coordinates_systems.pdf b/figures/schemes/coordinates_systems.pdf index c015e8a..c9def7d 100644 Binary files a/figures/schemes/coordinates_systems.pdf and b/figures/schemes/coordinates_systems.pdf differ diff --git a/figures/schemes/ordinary_least_squares.pdf b/figures/schemes/ordinary_least_squares.pdf index b0d0bc1..db4a55a 100644 Binary files a/figures/schemes/ordinary_least_squares.pdf and b/figures/schemes/ordinary_least_squares.pdf differ diff --git a/figures/schemes/orthogonal_projection.pdf b/figures/schemes/orthogonal_projection.pdf index 1259746..7ca63e0 100644 Binary files a/figures/schemes/orthogonal_projection.pdf and b/figures/schemes/orthogonal_projection.pdf differ diff --git a/figures/schemes/orthogonal_projection.tex b/figures/schemes/orthogonal_projection.tex index 690cc07..6135b94 100644 --- a/figures/schemes/orthogonal_projection.tex +++ b/figures/schemes/orthogonal_projection.tex @@ -12,21 +12,31 @@ \tkzDefParallelogram(A,B,C) \tkzGetPoint{D} \tkzDrawPolygon[fill=gray!25!white](A,B,C,D) - \draw[decorate,decoration={brace, - amplitude=8pt},xshift=0pt,very thin,gray] (2,0) -- ++(-1,-0.5) node [black,midway,xshift=0.5em,yshift=-1em] {\color{blue}$a$}; + \end{scope} - \begin{scope}[canvas is xz plane at y=0] + % Draw the rectangle triangle scheme + \begin{scope}[canvas is xz plane at y=1] \draw[thick,fill=white,fill opacity=0.7,nodes={opacity=1}] - (2,0) node[bullet,label=below right:{$\mathbf{X}$}] {} - -- (0,0) node[bullet] {} - -- (0,3) node[bullet,label=above:{$\mathbf{Y}$}] {} -- cycle; - \draw (0.25,0) -- (0.25,0.25) -- (0,0.25); + (2,0) node[bullet,label=right:{$\bar{\mathbf{Y}}$}] (Y_bar) {} + -- (0,-0.5) node (B) {} + -- (0,3) node[label=above:{$\mathbf{Y}$}] (Y) {} -- cycle; + % Right angle annotation + \tkzPicRightAngle[draw, + angle eccentricity=.5,angle radius=2mm](Y,B,Y_bar) + % epsilon: Y - X \hat{\beta} curly brackets annotations \draw[decorate,decoration={brace, - amplitude=8pt},xshift=0pt,very thin,gray] (0,0) -- (0,3) node [black,midway,xshift=-1.25em,yshift=0em] {\color{blue}$b$}; + amplitude=8pt},xshift=0pt,very thin,gray] (B) -- (Y) node [black,midway,xshift=-1.25em,yshift=0em] {\color{blue}$b$}; + % X\hat{\beta} - \hat{Y} + \draw[decorate,decoration={brace, + amplitude=8pt},xshift=0pt,very thin,gray] (Y_bar) -- (B) node [black,midway,xshift=0.5em,yshift=-1em] {\color{blue}$a$}; + % + \draw[decorate,decoration={brace, + amplitude=8pt},xshift=0pt,very thin,gray] (Y) -- (Y_bar) node [black,midway,xshift=1em,yshift=1em] {\color{blue}$c$}; \end{scope} + % Coordinate system \begin{scope}[canvas is xy plane at z=0] - \draw[->] (2,0) -- ++(-0.75,0.75) node [left] {$\mathbf{1}$}; - \draw[->] (2,0) -- ++(-1,-0.5); + \draw[->] (2,1) -- node [above] {$\mathbf{1}$} ++(-1,0) ; + \draw[->] (2,1) -- ++(-0.45,-1) node [right] {$X_1$}; \end{scope} \end{tikzpicture} \end{document} \ No newline at end of file diff --git a/figures/schemes/regression_plan_3D.pdf b/figures/schemes/regression_plan_3D.pdf index 16f34d6..07bffa0 100644 Binary files a/figures/schemes/regression_plan_3D.pdf and b/figures/schemes/regression_plan_3D.pdf differ diff --git a/figures/schemes/vector_orthogonality.pdf b/figures/schemes/vector_orthogonality.pdf index 5ff62b4..71ef526 100644 Binary files a/figures/schemes/vector_orthogonality.pdf and b/figures/schemes/vector_orthogonality.pdf differ diff --git a/main.pdf b/main.pdf index 23ad1f1..101c25b 100644 --- a/main.pdf +++ b/main.pdf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:023ad4642b51357283dc5bef3cc844e826aa35c4d1757d7f41f603610d5fee92 -size 254796 +oid sha256:27d9599621738b732087974ecd8141b18411aac1ef77ff31c2ea2464ac443eb8 +size 308493 diff --git a/main.tex b/main.tex index 2b23f63..5069408 100644 --- a/main.tex +++ b/main.tex @@ -28,9 +28,6 @@ \definecolor{myblue}{HTML}{5654fa} \colorlet{primary}{myblue} -\input{definitions} -\input{preamble} - \hypersetup{ pdftitle={Course - Multivariate Statistics}, pdfauthor={Samuel Ortion}, @@ -51,6 +48,7 @@ \input{glossary} \input{definitions} + \makeindex% \makeglossary% \begin{document}