From 5a33ed3c8e0eef2d34d321607da2b398bf7f8bc3 Mon Sep 17 00:00:00 2001 From: Tobias Arndt Date: Tue, 21 Apr 2020 18:43:33 +0200 Subject: [PATCH] added poincare type inequality and neuron diagram --- TeX/introduction_nn.tex | 181 ++++++++++++++++++++++++++++++++-------- TeX/main.tex | 4 +- TeX/theo_3_8.tex | 69 ++++++++++++++- 3 files changed, 216 insertions(+), 38 deletions(-) diff --git a/TeX/introduction_nn.tex b/TeX/introduction_nn.tex index cb4c168..f5472d4 100644 --- a/TeX/introduction_nn.tex +++ b/TeX/introduction_nn.tex @@ -1,8 +1,4 @@ -%%% Local Variables: -%%% mode: latex -%%% TeX-master: "main" -%%% End: \section{Introduction to Neural Networks} Neural Networks (NN) are a mathematical construct inspired by the @@ -34,7 +30,9 @@ except for the input layer, which recieves the components of the input. \resizebox{\textwidth}{!}{% \begin{tikzpicture}[x=1.75cm, y=1.75cm, >=stealth] - + \tikzset{myptr/.style={decoration={markings,mark=at position 1 with % + {\arrow[scale=1.5,>=stealth]{>}}},postaction={decorate}}} + \foreach \m/\l [count=\y] in {1,2,3,missing,4} \node [every neuron/.try, neuron \m/.try] (input-\m) at (0,2.5-\y) {}; @@ -48,7 +46,7 @@ except for the input layer, which recieves the components of the input. \node [every neuron/.try, neuron \m/.try ] (output-\m) at (7,1.5-\y) {}; \foreach \l [count=\i] in {1,2,3,d_i} - \draw [<-] (input-\i) -- ++(-1,0) + \draw [myptr] (input-\i)+(-1,0) -- (input-\i) node [above, midway] {$x_{\l}$}; \foreach \l [count=\i] in {1,n_1} @@ -58,20 +56,20 @@ except for the input layer, which recieves the components of the input. \node [above] at (hidden2-\i.north) {$\mathcal{N}_{l,\l}$}; \foreach \l [count=\i] in {1,d_o} - \draw [->] (output-\i) -- ++(1,0) + \draw [myptr] (output-\i) -- ++(1,0) node [above, midway] {$O_{\l}$}; \foreach \i in {1,...,4} \foreach \j in {1,...,2} - \draw [->] (input-\i) -- (hidden1-\j); + \draw [myptr] (input-\i) -- (hidden1-\j); \foreach \i in {1,...,2} \foreach \j in {1,...,2} - \draw [->] (hidden1-\i) -- (hidden2-\j); + \draw [myptr] (hidden1-\i) -- (hidden2-\j); \foreach \i in {1,...,2} \foreach \j in {1,...,2} - \draw [->] (hidden2-\i) -- (output-\j); + \draw [myptr] (hidden2-\i) -- (output-\j); \node [align=center, above] at (0,2) {Input\\layer}; \node [align=center, above] at (2,2) {Hidden \\layer $1$}; @@ -79,46 +77,157 @@ except for the input layer, which recieves the components of the input. \node [align=center, above] at (7,2) {Output \\layer}; \node[fill=white,scale=1.5,inner xsep=10pt,inner ysep=10mm] at ($(hidden1-1)!.5!(hidden2-2)$) {$\dots$}; - + \end{tikzpicture}}} \caption{test} \end{figure} -\begin{tikzpicture}[x=1.5cm, y=1.5cm, >=stealth] - -\foreach \m/\l [count=\y] in {1} - \node [every neuron/.try, neuron \m/.try] (input-\m) at (0,0.5-\y) {}; +\begin{figure} + \begin{tikzpicture}[x=1.5cm, y=1.5cm] + \tikzset{myptr/.style={decoration={markings,mark=at position 1 with % + {\arrow[scale=1.5,>=stealth]{>}}},postaction={decorate}}} + + \foreach \m/\l [count=\y] in {1} + \node [every neuron/.try, neuron \m/.try] (input-\m) at (0,0.5-\y) {}; -\foreach \m [count=\y] in {1,2,missing,3,4} - \node [every neuron/.try, neuron \m/.try ] (hidden-\m) at (1.25,3.25-\y*1.25) {}; + \foreach \m [count=\y] in {1,2,missing,3,4} + \node [every neuron/.try, neuron \m/.try ] (hidden-\m) at (1.25,3.25-\y*1.25) {}; -\foreach \m [count=\y] in {1} - \node [every neuron/.try, neuron \m/.try ] (output-\m) at (2.5,0.5-\y) {}; + \foreach \m [count=\y] in {1} + \node [every neuron/.try, neuron \m/.try ] (output-\m) at (2.5,0.5-\y) {}; -\foreach \l [count=\i] in {1} - \draw [<-] (input-\i) -- ++(-1,0) + \foreach \l [count=\i] in {1} + \draw [myptr] (input-\i)+(-1,0) -- (input-\i) node [above, midway] {$x$}; -\foreach \l [count=\i] in {1,2,n-1,n} - \node [above] at (hidden-\i.north) {$\mathcal{N}_{\l}$}; + \foreach \l [count=\i] in {1,2,n-1,n} + \node [above] at (hidden-\i.north) {$\mathcal{N}_{\l}$}; -\foreach \l [count=\i] in {1,n_l} - \node [above] at (output-\i.north) {}; + \foreach \l [count=\i] in {1,n_l} + \node [above] at (output-\i.north) {}; -\foreach \l [count=\i] in {1} - \draw [->] (output-\i) -- ++(1,0) + \foreach \l [count=\i] in {1} + \draw [myptr, >=stealth] (output-\i) -- ++(1,0) node [above, midway] {$y$}; + + + \foreach \i in {1} + \foreach \j in {1,2,...,3,4} + \draw [myptr, >=stealth] (input-\i) -- (hidden-\j); + + \foreach \i in {1,2,...,3,4} + \foreach \j in {1} + \draw [myptr, >=stealth] (hidden-\i) -- (output-\j); + + \node [align=center, above] at (0,1) {Input \\layer}; + \node [align=center, above] at (1.25,3) {Hidden layer}; + \node [align=center, above] at (2.5,1) {Output \\layer}; + + \end{tikzpicture} + \caption{Shallow Neural Network with input- and output-dimension of \(d + = 1\)} +\end{figure} + + +\begin{figure} + \begin{tikzpicture}[x=1.5cm, y=1.5cm, >=stealth] + + + \tikzset{myptr/.style={decoration={markings,mark=at position 1 with % + {\arrow[scale=1.5,>=stealth]{>}}},postaction={decorate}}} + + \node [circle, draw, fill=black, inner sep = 0pt, minimum size = + 1.5mm, left] (i_1) at (0, 2.5) {}; + \node [align=left, left] at (-0.125, 2.5) {\(i_1\)}; + \node [circle, draw, fill=black, inner sep = 0pt, minimum size = + 1.5mm] (i_2) at (0, 1.25) {}; + \node [align=left, left] at (-0.125, 1.25) {\(i_2\)}; + \node [neuron missing] (i_3) at (0, 0) {}; + \node [circle, draw, fill=black, inner sep = 0pt, minimum size = + 1.5mm] (i_4) at (0, -1.25) {}; + \node [align=left, left] at (-0.125, -1.25) {\(i_m\)}; + \draw[decoration={calligraphic brace,amplitude=5pt, mirror}, decorate, line width=1.25pt] + (-0.6,2.7) -- (-0.6,-1.45) node [black, midway, xshift=-0.6cm, left] {Input}; + + \node [align = center, above] at (1.25, 3) {Synaptic\\weights}; + \node [every neuron] (w_1) at (1.25, 2.5) {\(w_{k, 1}\)}; + \node [every neuron] (w_2) at (1.25, 1.25) {\(w_{k, 2}\)}; + \node [neuron missing] (w_3) at (1.25, 0) {}; + \node [every neuron] (w_4) at (1.25, -1.25) {\(w_{k, m}\)}; + + \node [circle, draw] (sig) at (3, 0.625) {\Large\(\sum\)}; + \node [align = center, below] at (3, 0) {Summing \\junction}; + + \node [draw, minimum size = 1.25cm] (act) at (4.5, 0.625) + {\(\psi(.)\)}; + \node [align = center, above] at (4.5, 1.25) {Activation \\function}; -\foreach \i in {1} - \foreach \j in {1,2,...,3,4} - \draw [->] (input-\i) -- (hidden-\j); + \node [circle, draw, fill=black, inner sep = 0pt, minimum size = + 1.5mm] (b) at (3, 2.5) {}; + \node [align = center, above] at (3, 2.75) {Bias \\\(b_k\)}; -\foreach \i in {1,2,...,3,4} - \foreach \j in {1} - \draw [->] (hidden-\i) -- (output-\j); + \node [align = center] (out) at (6, 0.625) {Output \\\(o_k\)}; -\node [align=center, above] at (0,1) {Input\\layer}; -\node [align=center, above] at (1.25,3) {Hidden layer}; -\node [align=center, above] at (2.5,1) {Output\\layer}; + \draw [myptr] (i_1) -- (w_1); + \draw [myptr] (i_2) -- (w_2); + \draw [myptr] (i_4) -- (w_4); + + \draw [myptr] (w_1) -- (sig); + \draw [myptr] (w_2) -- (sig); + \draw [myptr] (w_4) -- (sig); + + \draw [myptr] (b) -- (sig); + + \draw [myptr] (sig) -- (act); + + \draw [myptr] (act) -- (out); + + % \foreach \m [count=\y] in {1,2,missing,3,4} + % \node [every neuron/.try, neuron \m/.try ] (hidden-\m) at (1.25,3.25-\y*1.25) {\(w_{k,\y}\)}; + + % \foreach \m [count=\y] in {1} + % \node [every neuron/.try, neuron \m/.try ] (output-\m) at (2.5,0.5-\y) {}; + + % \foreach \l [count=\i] in {1} + % \draw [<-] (input-\i) -- ++(-1,0) + % node [above, midway] {$x$}; + + % \foreach \l [count=\i] in {1,2,n-1,n} + % \node [above] at (hidden-\i.north) {$\mathcal{N}_{\l}$}; + + % \foreach \l [count=\i] in {1,n_l} + % \node [above] at (output-\i.north) {}; + + % \foreach \l [count=\i] in {1} + % \draw [->] (output-\i) -- ++(1,0) + % node [above, midway] {$y$}; + + % \foreach \i in {1} + % \foreach \j in {1,2,...,3,4} + % \draw [->] (input-\i) -- (hidden-\j); + + % \foreach \i in {1,2,...,3,4} + % \foreach \j in {1} + % \draw [->] (hidden-\i) -- (output-\j); + + \end{tikzpicture} + \caption{Structure of a single neuron} +\end{figure} + +\begin{tikzpicture} +\tikzset{myptr/.style={decoration={markings,mark=at position 1 with % + {\arrow[scale=2,>=stealth]{>}}},postaction={decorate}}} +%1 +\draw [->,>=stealth] (0,.5) -- (2,.5); +%2 +\draw [myptr] (0,0) -- (2,0); \end{tikzpicture} + + + + +%%% Local Variables: +%%% mode: latex +%%% TeX-master: "main" +%%% End: diff --git a/TeX/main.tex b/TeX/main.tex index e52d3c9..3438a5f 100644 --- a/TeX/main.tex +++ b/TeX/main.tex @@ -26,9 +26,10 @@ \usepackage{makecell} \usepackage{dsfont} \usepackage{tikz} +\usepackage{nicefrac} \usetikzlibrary{matrix,chains,positioning,decorations.pathreplacing,arrows} -\usetikzlibrary{positioning,calc} +\usetikzlibrary{positioning,calc,calligraphy} \usepackage{pgfplots} \usepgfplotslibrary{colorbrewer} @@ -55,6 +56,7 @@ \thispagestyle{plain} \newtheorem{Theorem}{Theorem}[section] \newtheorem{Definition}[Theorem]{Definition} +\newtheorem{Lemma}[Theorem]{Lemma} \newtheorem{Algorithm}[Theorem]{Algorithm} \newtheorem{Example}[Theorem]{Example} diff --git a/TeX/theo_3_8.tex b/TeX/theo_3_8.tex index 5b29926..41d82a2 100644 --- a/TeX/theo_3_8.tex +++ b/TeX/theo_3_8.tex @@ -25,7 +25,74 @@ limes of RN as the amount of nodes is increased. \in \mathbb{R} \end{align*} and \(RN^{*, \tilde{\lambda}}\), \(f^{*,\tilde{\lambda}}_{g, \pm}\) -as defined in ??? and ??? respectively. + as defined in ??? and ??? respectively. \end{Theorem} In order to proof Theo~\ref{theo:main1} we need to proof a number of auxilary Lemmata first. + +\begin{Definition}[Sobolev Norm] + \label{def:sobonorm} + The natural norm of the sobolev space is given by + \[ + \norm{f}_{W^{k,p}(K)} = + \begin{cases} + \left(\sum_{\abs{\alpha} \leq k} + \norm{f^{(\alpha)}}^p_{L^p}\right)^{\nicefrac{1}{p}},& + \text{for } 1 \leq p < \infty \\ + max_{\abs{\alpha} \leq k}\left\{f^{(\alpha)}\right\},& \text{for + } p = \infty + \end{cases} + . + \] +\end{Definition} + +\begin{Lemma}[Poincar\'e typed inequality] + Let \(f:\mathbb{R} \to \mathbb{R}\) differentiable with \(f' : + \mathbb{R} \to \mathbb{R}\) Lesbeque integrable. Then for \(K=[a,b] + \subset \mathbb{R}\) with \(f(a)=0\) it holds that + \begin{equation} + \label{eq:pti1} + \exists C_K^{\infty} \in \mathbb{R}_{>0} : + \norm{f}_{w^{1,\infty}(K)} \leq C_K^{\infty} + \norm{f'}_{L^{\infty}(K)}. + \end{equation} + If additionaly \(f'\) is differentiable with \(f'': \mathbb{R} \to + \mathbb{R}\) Lesbeque integrable then additionally + \begin{equation} + \label{eq:pti2} + \exists C_K^2 \in \mathbb{R}_{>0} : \norm{f}_{W^{1,\infty}(K)} \leq + C_K^2 \norm{f''}_{L^2(K)}. + \end{equation} + \proof + With the fundamental theorem of calculus, if + \(\norm{f}_{L^{\infty}(K)}<\infty\) we get + \begin{equation} + \label{eq:f_f'} + \norm{f}_{L^{\infty}(K)} = \sup_{x \in K}\abs{\int_a^x f'(s) ds} \leq + \sup_{x \in K}\abs{\int_a^x \sup_{y \in K} \abs{f'(y)} ds} \leq \abs{b-a} + \sup_{y \in K}\abs{f'(y)}. + \end{equation} + Using this we can bound \(\norm{f}_{w^{1,\infty}(K)}\) by + \[ + \norm{f}_{w^{1,\infty}(K)} \stackrel{\text{Def~\ref{def:sobonorm}}}{=} + \max\left\{\norm{f}_{L^{\infty}(K)}, + \norm{f'}_{L^{\infty}(K)}\right\} + \stackrel{(\ref{eq:f_f'})}{\leq} max\left\{\abs{b-a}, + 1\right\}\norm{f'}_{L^{\infty}(K)}. + \] + With \(C_k^{\infty} \coloneqq max\left\{\abs{b-a}, 1\right\}\) we + get (\ref{eq:pti1}). + By using the Hölder inequality, we can proof the second claim. + \begin{align*} + \norm{f'}_{L^{\infty}(K)} &= \sup_{x \in K} \abs{\int_a^bf''(y) + \mathds{1}_{[a,x]}(y)dy} \leq \sup_{x \in + K}\norm{f''\mathds{1}_{[a,x]}}_{L^1(K)}\\ + &\hspace{-6pt} \stackrel{\text{Hölder}}{\leq} sup_{x + \in + K}\norm{f''}_{L^2(K)}\norm{\mathds{1}_{[a,x]}}_{L^2(K)} + = \abs{b-a}\norm{f''}_{L^2(K)}. + \end{align*} + Thus (\ref{eq:pti2}) follows with \(C_K^2 \coloneqq + \abs{b-a}C_K^{\infty}\). + \qed +\end{Lemma} \ No newline at end of file