\section{Introduction to Neural Networks}

Neural Networks (NN) are a mathematical construct inspired by the
connection of neurons in nature. It consists of an input and output
layer with an arbitrary amount of hidden layers between them. Each
layer consits of a numer of neurons (nodes) with the number of nodes
in the in-/output layers corresponding to the dimensions of the
in-/output.\par
Each neuron recieves the output of all layers in the previous layers,
except for the input layer, which recieves the components of the input.

\tikzset{%
  every neuron/.style={
    circle,
    draw,
    minimum size=1cm
  },
  neuron missing/.style={
    draw=none, 
    scale=1.5,
    text height=0.333cm,
    execute at begin node=\color{black}$\vdots$
  },
}
\begin{figure}[h!]
  \center
  
  \fbox{
    
    \resizebox{\textwidth}{!}{%
      \begin{tikzpicture}[x=1.75cm, y=1.75cm, >=stealth]
    \tikzset{myptr/.style={decoration={markings,mark=at position 1 with %
      {\arrow[scale=1.5,>=stealth]{>}}},postaction={decorate}}}
        
        \foreach \m/\l [count=\y] in {1,2,3,missing,4}
        \node [every neuron/.try, neuron \m/.try] (input-\m) at (0,2.5-\y) {};
        
        \foreach \m [count=\y] in {1,missing,2}
        \node [every neuron/.try, neuron \m/.try ] (hidden1-\m) at (2,2-\y*1.25) {};
        
        \foreach \m [count=\y] in {1,missing,2}
        \node [every neuron/.try, neuron \m/.try ] (hidden2-\m) at (5,2-\y*1.25) {};
        
        \foreach \m [count=\y] in {1,missing,2}
        \node [every neuron/.try, neuron \m/.try ] (output-\m) at (7,1.5-\y) {};
        
        \foreach \l [count=\i] in {1,2,3,d_i}
        \draw [myptr] (input-\i)+(-1,0) -- (input-\i)
        node [above, midway] {$x_{\l}$};
        
        \foreach \l [count=\i] in {1,n_1}
        \node [above] at (hidden1-\i.north) {$\mathcal{N}_{1,\l}$};
        
        \foreach \l [count=\i] in {1,n_l}
        \node [above] at (hidden2-\i.north) {$\mathcal{N}_{l,\l}$};
        
        \foreach \l [count=\i] in {1,d_o}
        \draw [myptr] (output-\i) -- ++(1,0)
        node [above, midway] {$O_{\l}$};
        
        \foreach \i in {1,...,4}
        \foreach \j in {1,...,2}
        \draw [myptr] (input-\i) -- (hidden1-\j);
        
        \foreach \i in {1,...,2}
        \foreach \j in {1,...,2}
        \draw [myptr] (hidden1-\i) -- (hidden2-\j);
        
        \foreach \i in {1,...,2}
        \foreach \j in {1,...,2}
        \draw [myptr] (hidden2-\i) -- (output-\j);
        
        \node [align=center, above] at (0,2) {Input\\layer};
        \node [align=center, above] at (2,2) {Hidden \\layer $1$};
        \node [align=center, above] at (5,2) {Hidden \\layer $l$};
        \node [align=center, above] at (7,2) {Output \\layer};
        
        \node[fill=white,scale=1.5,inner xsep=10pt,inner ysep=10mm] at ($(hidden1-1)!.5!(hidden2-2)$) {$\dots$};
        
      \end{tikzpicture}}}
  \caption{test}
\end{figure}

\begin{figure}
  \begin{tikzpicture}[x=1.5cm, y=1.5cm]
    \tikzset{myptr/.style={decoration={markings,mark=at position 1 with %
      {\arrow[scale=1.5,>=stealth]{>}}},postaction={decorate}}}
    
    \foreach \m/\l [count=\y] in {1}
    \node [every neuron/.try, neuron \m/.try] (input-\m) at (0,0.5-\y) {};

    \foreach \m [count=\y] in {1,2,missing,3,4}
    \node [every neuron/.try, neuron \m/.try ] (hidden-\m) at (1.25,3.25-\y*1.25) {};

    \foreach \m [count=\y] in {1}
    \node [every neuron/.try, neuron \m/.try ] (output-\m) at (2.5,0.5-\y) {};

    \foreach \l [count=\i] in {1}
    \draw [myptr] (input-\i)+(-1,0) -- (input-\i)
    node [above, midway] {$x$};

    \foreach \l [count=\i] in {1,2,n-1,n}
    \node [above] at (hidden-\i.north) {$\mathcal{N}_{\l}$};

    \foreach \l [count=\i] in {1,n_l}
    \node [above] at (output-\i.north) {};

    \foreach \l [count=\i] in {1}
    \draw [myptr, >=stealth] (output-\i) -- ++(1,0)
    node [above, midway] {$y$};
    

    \foreach \i in {1}
    \foreach \j in {1,2,...,3,4}
    \draw [myptr, >=stealth] (input-\i) -- (hidden-\j);

    \foreach \i in {1,2,...,3,4}
    \foreach \j in {1}
    \draw [myptr, >=stealth] (hidden-\i) -- (output-\j);

    \node [align=center, above] at (0,1) {Input    \\layer};
    \node [align=center, above] at (1.25,3) {Hidden layer};
    \node [align=center, above] at (2.5,1) {Output \\layer};

  \end{tikzpicture}
  \caption{Shallow Neural Network with input- and output-dimension of \(d
    = 1\)}
\end{figure}


\begin{figure}
  \begin{tikzpicture}[x=1.5cm, y=1.5cm, >=stealth]

    
    \tikzset{myptr/.style={decoration={markings,mark=at position 1 with %
      {\arrow[scale=1.5,>=stealth]{>}}},postaction={decorate}}}
    
    \node [circle, draw, fill=black, inner sep = 0pt, minimum size =
    1.5mm, left] (i_1) at (0, 2.5) {};
    \node [align=left, left] at (-0.125, 2.5) {\(i_1\)};
    \node [circle, draw, fill=black, inner sep = 0pt, minimum size =
    1.5mm] (i_2) at (0, 1.25) {};
    \node [align=left, left] at (-0.125, 1.25) {\(i_2\)};
    \node [neuron missing] (i_3) at (0, 0) {};
    \node [circle, draw, fill=black, inner sep = 0pt, minimum size =
    1.5mm] (i_4) at (0, -1.25) {};
    \node [align=left, left] at (-0.125, -1.25) {\(i_m\)};
    \draw[decoration={calligraphic brace,amplitude=5pt, mirror}, decorate, line width=1.25pt]
    (-0.6,2.7) -- (-0.6,-1.45) node [black, midway, xshift=-0.6cm, left] {Inputs};

    \node [align = center, above] at (1.25, 3) {Synaptic\\weights};
    \node [every neuron] (w_1) at (1.25, 2.5) {\(w_{k, 1}\)};
    \node [every neuron] (w_2) at (1.25, 1.25) {\(w_{k, 2}\)};
    \node [neuron missing] (w_3) at (1.25, 0) {};
    \node [every neuron] (w_4) at (1.25, -1.25) {\(w_{k, m}\)};

    \node [circle, draw] (sig) at (3, 0.625) {\Large\(\sum\)};
    \node [align = center, below] at (3, 0) {Summing \\junction};

    \node [draw, minimum size = 1.25cm] (act) at (4.5, 0.625)
    {\(\psi(.)\)};
    \node [align = center, above] at (4.5, 1.25) {Activation \\function}; 

    \node [circle, draw, fill=black, inner sep = 0pt, minimum size =
    1.5mm] (b) at (3, 2.5) {};
    \node [align = center, above] at (3, 2.75) {Bias \\\(b_k\)};

    \node [align = center] (out) at (6, 0.625) {Output \\\(o_k\)};


    \draw [myptr] (i_1) -- (w_1);
    \draw [myptr] (i_2) -- (w_2);
    \draw [myptr] (i_4) -- (w_4);

    \draw [myptr] (w_1) -- (sig);
    \draw [myptr] (w_2) -- (sig);
    \draw [myptr] (w_4) -- (sig);

    \draw [myptr] (b) -- (sig);

    \draw [myptr] (sig) -- (act);

    \draw [myptr] (act) -- (out);

    % \foreach \m [count=\y] in {1,2,missing,3,4}
    % \node [every neuron/.try, neuron \m/.try ] (hidden-\m) at (1.25,3.25-\y*1.25) {\(w_{k,\y}\)};

    % \foreach \m [count=\y] in {1}
    % \node [every neuron/.try, neuron \m/.try ] (output-\m) at (2.5,0.5-\y) {};

    % \foreach \l [count=\i] in {1}
    % \draw [<-] (input-\i) -- ++(-1,0)
    % node [above, midway] {$x$};

    % \foreach \l [count=\i] in {1,2,n-1,n}
    % \node [above] at (hidden-\i.north) {$\mathcal{N}_{\l}$};

    % \foreach \l [count=\i] in {1,n_l}
    % \node [above] at (output-\i.north) {};

    % \foreach \l [count=\i] in {1}
    % \draw [->] (output-\i) -- ++(1,0)
    % node [above, midway] {$y$};

    % \foreach \i in {1}
    % \foreach \j in {1,2,...,3,4}
    % \draw [->] (input-\i) -- (hidden-\j);

    % \foreach \i in {1,2,...,3,4}
    % \foreach \j in {1}
    % \draw [->] (hidden-\i) -- (output-\j);

  \end{tikzpicture}
  \caption{Structure of a single neuron}
\end{figure}

\begin{tikzpicture}
\tikzset{myptr/.style={decoration={markings,mark=at position 1 with %
    {\arrow[scale=2,>=stealth]{>}}},postaction={decorate}}}
%1
\draw [->,>=stealth] (0,.5) -- (2,.5);
%2
\draw [myptr] (0,0) -- (2,0);
\end{tikzpicture}




%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End: