Masterarbeit/TeX/appendixA.tex

\newpage
\begin{appendices}
  \counterwithin{lstfloat}{section}
  \section{Proofs for sone Lemmata in ...}
  In the following there will be proofs for some important Lemmata in
  Section~\ref{sec:theo38}. Further proofs not discussed here can be
  found in \textcite{heiss2019}
  The proves in this section are based on \textcite{heiss2019}. Slight
  alterations have been made to accommodate for not splitting $f$ into
  $f_+$ and $f_-$.
  \begin{Theorem}[Proof of Lemma~\ref{theo38}]
  \end{Theorem}
  
  \begin{Lemma}[$\frac{w^{*,\tilde{\lambda}}_k}{v_k}\approx\mathcal{O}(\frac{1}{n})$]
    For any $\lambda > 0$ and training data $(x_i^{\text{train}},
    y_i^{\text{train}}) \in \mathbb{R}^2, \, i \in
    \left\{1,\dots,N\right\}$, we have
    \[
      \max_{k \in \left\{1,\dots,n\right\}} \frac{w^{*,
          \tilde{\lambda}}_k}{v_k} = \po_{n\to\infty}
    \]
  
    
  \end{Lemma}

  \begin{Proof}[Proof of Lemma~\ref{lem:s3}]
    \[
  \sum_{k \in \kappa : \xi_k < T} \varphi(\xi_k, v_k)
  h_{k,n} = \sum_{\substack{l \in \mathbb{Z} \\ [\delta l, \delta
      (l+1)) \in [C_{g_\xi}^l,\min\{C_{g_\xi}^u, T \}]}}
  \left(\sum_{\substack{k \in \kappa \\ \xi_k \in
        [\delta l , \delta(l+1))}} \varphi(\xi_k, v_k) 
    h_{k,n}\right) \approx
\]
\[
  \approx \sum_{\substack{l \in \mathbb{Z} \\ [\delta l, \delta
      (l+1)) \in [C_{g_\xi}^l,\min\{C_{g_\xi}^u, T \}]}}
  \left(\sum_{\substack{k \in \kappa \\ \xi_k \in
        [\delta l , \delta(l+1))}} \left(\varphi(\delta l, v_k)
      \frac{1}{n g_\xi (\delta l)} \pm \frac{\varepsilon}{n}\right) 
    \frac{\abs{\left\{m \in \kappa : \xi_m \in [\delta l,
          \delta(l+1))\right\}}}{\abs{\left\{m \in \kappa : \xi_m
          \in [\delta l, \delta(l+1))\right\}}}\right)
\]
\[
  \approx \sum_{\substack{l \in \mathbb{Z} \\ [\delta l, \delta
      (l+1)) \in [C_{g_\xi}^l,\min\{C_{g_\xi}^u, T \}]}}
  \left(\frac{\sum_{\substack{k \in \kappa \\ \xi_k \in
          [\delta l , \delta(l+1))}}\varphi(\delta l,
      v_k)}{\abs{\left\{m \in \kappa : \xi_m
          \in [\delta l, \delta(l+1))\right\}}} 
    \frac{\abs{\left\{m \in \kappa : \xi_m \in [\delta l,
          \delta(l+1))\right\}}}{n g_\xi (\delta l)}\right) \pm \varepsilon
\]
The amount of kinks in a given interval of length $\delta$ follows a
binomial distribution,
\[
  \mathbb{E} \left[\abs{\left\{m \in \kappa : \xi_m \in [\delta l,
        \delta(l+1))\right\}}\right] = n \int_{\delta
    l}^{\delta(l+1)}g_\xi (x) dx \approx n (\delta g_\xi(\delta l)
  \pm \delta \tilde{\varepsilon}),
\]
for any $\delta \leq \delta(\varepsilon, \tilde{\varepsilon})$, since $g_\xi$ is uniformly continuous on its
support by Assumption..
As the distribution of $v$ is continuous as well we get that
$\mathcal{L}(v_k) = \mathcal{L} v| \xi = \delta l) \forall k \in
\kappa : \xi_k \in [\delta l, \delta(l+1))$ for $\delta \leq
\delta(\varepsilon, \tilde{\varepsilon})$. Thus we get with the law of
large numbers
\begin{align*}
  &\sum_{k \in \kappa : \xi_k < T} \varphi(\xi_k, v_k)
    h_{k,n} \approx\\  
  &\approx \sum_{\substack{l \in \mathbb{Z} \\ [\delta l, \delta
  (l+1)) \in [C_{g_\xi}^l,\min\{C_{g_\xi}^u, T
  \}]}}\left(\mathbb{E}[\phi(\xi, v)|\xi=\delta l]
  \stackrel{\mathbb{P}}{\pm}\right) \delta \left(1 \pm
  \frac{\tilde{\varepsilon}}{g_\xi(\delta l)}\right) \pm \varepsilon
  \\
  &\approx \left(\sum_{\substack{l \in \mathbb{Z} \\ [\delta
  l, \delta 
  (l+1)) \in [C_{g_\xi}^l,\min\{C_{g_\xi}^u, T
  \}]}}\mathbb{E}[\phi(\xi, v)|\xi=\delta l] \delta
  \stackrel{\mathbb{P}}{\pm}\tilde{\tilde{\varepsilon}}
  \abs{C_{g_\xi}^u - C_{g_\xi}^l}
  \right)\\
  &\phantom{\approx}\cdot \left(1 \pm
    \frac{\tilde{\varepsilon}}{g_\xi(\delta l)}\right) \pm \varepsilon
\end{align*}
\end{Proof}

\begin{Lemma}[($L(f_n) \to L(f)$), Heiss, Teichmann, and
Wutte (2019, Lemma A.11)]
  For any data $(x_i^{\text{train}}, y_i^{\text{train}}) \in
  \mathbb{R}^2, i \in \left\{1,\dots,N\right\}$, let $(f_n)_{n \in
    \mathbb{N}}$ be a sequence of functions that converges point-wise
  in probability to a function $f : \mathbb{R}\to\mathbb{R}$, then the
  loss $L$ of $f_n$ converges is probability to $L(f)$ as $n$ tends to
  infinity,
  \[
    \plimn L(f_n) = L(f).
  \]
  \proof Vgl. ...
\end{Lemma}

\begin{Proof}[Step 2]
  We start by showing that
  \[
    \plimn \tilde{\lambda} \norm{\tilde{w}}_2^2 = \lambda g(0)
    \left(\int \frac{\left(f_g^{*,\lambda''}\right)^2}{g(x)} dx\right)
  \]
  With the definitions of $\tilde{w}$, $\tilde{\lambda}$ and
  $h$ we have
  \begin{align*}
    \tilde{\lambda} \norm{\tilde{w}}_2^2
    &= \tilde{\lambda} \sum_{k \in
      \kappa}\left(f_g^{*,\lambda''}(\xi_k) \frac{h_k
      v_k}{\mathbb{E}v^2|\xi = \xi_k]}\right)^2\\
    &= \tilde{\lambda} \sum_{k \in
      \kappa}\left(\left(f_g^{*,\lambda''}\right)^2(\xi_k) \frac{h_k
      v_k^2}{\mathbb{E}v^2|\xi = \xi_k]}\right) h_k\\
    & = \lambda g(0) \sum_{k \in
      \kappa}\left(\left(f_g^{*,\lambda''}\right)^2(\xi_k)\frac{v_k^2}{g_\xi(\xi_k)\mathbb{E}
      [v^2|\xi=\xi_k]}\right)h_k.
  \end{align*}
  By using Lemma~\ref{lem} with $\phi(x,y) =
  \left(f_g^{*,\lambda''}\right)^2(x)\frac{y^2}{g_\xi(\xi)\mathbb{E}[v^2|\xi=y]}$
  this converges to
  \begin{align*}
    &\plimn \tilde{\lambda}\norm{\tilde{w}}_2^2 = \\
    &=\lambda
      g_\xi(0)\mathbb{E}[v^2|\xi=0]\int_{\supp{g_\xi}}\mathbb{E}\left[
      \left(f_g^{*,\lambda''}\right)^2(\xi)\frac{v^2}{
      g_\xi(\xi)\mathbb{E}[v^2|\xi=x]^2}\Big{|} \xi = x\right]dx\\
    &=\lambda g_\xi(0) \mathbb{E}[v^2|\xi=0] \int_{\supp{g_xi}}
      \frac{\left(f_g^{*,\lambda''}\right)^2 (x)}{g_\xi(x)
      \mathbb{E}[v^2|\xi=x]} dx \\
    &=\lambda g(0) \int_{\supp{g_\xi}} \frac{\left(f_g^{*,\lambda''}\right)^2}{g(x)}dx.
  \end{align*}
\end{Proof}

\begin{Lemma}[Heiss, Teichmann, and
  Wutte (2019, Lemma A.13)]
  Using the notation of Definition .. and ... the following statement
  holds:
  $\forall \varepsilon \in \mathbb{R}_{>0} : \exists \delta \in
  \mathbb{R}_{>0} : \forall \omega \in \Omega : \forall l, l' \in
  \left\{1,\dots,N\right\} : \forall n \in \mathbb{N}$
  \[
    \left(\abs{\xi_l(\omega) - \xi_{l'}(\omega)} < \delta \wedge
      \text{sign}(v_l(\omega)) = \text{sign}(v_{l'}(\omega))\right)
    \implies \abs{\frac{w_l^{*, \tilde{\lambda}}(\omega)}{v_l(\omega)}
      - \frac{w_{l'}^{*, \tilde{\lambda}}(\omega)}{v_{l'}(\omega)}} <
    \frac{\varepsilon}{n},
  \]
  if we assume that $v_k$ is never zero.
  \proof given in ..
\end{Lemma}

\begin{Lemma}[$\frac{w^{*,\tilde{\lambda}}}{v} \approx
    \mathcal{O}(\frac{1}{n})$, Heiss, Teichmann, and
    Wutte (2019, Lemma A.14)]
    For any $\lambda > 0$ and data $(x_i^{\text{train}},
    y_i^{\text{train}}) \in \mathbb{R}^2, i\in
    \left\{1,\dots,\right\}$, we have
    \[
      \forall P \in (0,1) : \exists C \in \mathbb{R}_{>0} : \exists
      n_0 \in \mathbb{N} : \forall n > n_0 : \mathbb{P}
      \left[\max_{k\in \left\{1,\dots,n\right\}}
        \frac{w_k^{*,\tilde{\lambda}}}{v_k} < C
        \frac{1}{n}\right] > P
      % \max_{k\in \left\{1,\dots,n\right\}}
      % \frac{w_k^{*,\tilde{\lambda}}}{v_k} = \plimn
    \]
    \proof
    Let $k^*_+ \in \argmax_{k\in
      \left\{1,\dots,n\right\}}\frac{w^{*,\tilde{\lambda}}}{v_k} : v_k
    > 0$ and $k^*_- \in \argmax_{k\in
      \left\{1,\dots,n\right\}}\frac{w^{*,\tilde{\lambda}}}{v_k} : v_k
    < 0$. W.l.o.g. assume $\frac{w_{k_+^*}^2}{v_{k_+^*}^2} \geq
    \frac{w_{k_-^*}^2}{v_{k_-^*}^2}$
    \begin{align*}
      \frac{F^{\lambda,
      g}\left(f^{*,\lambda}_g\right)}{\tilde{\lambda}}
      \makebox[2cm][c]{$\stackrel{\mathbb{P}}{\geq}$}
      & \frac{1}{2 \tilde{\lambda}}
        F_n^{\tilde{\lambda}}\left(\mathcal{RN}^{*,\tilde{\lambda}}\right)
        = \frac{1}{2 \tilde{\lambda}}\left[\sum ... + \tilde{\lambda} \norm{w}_2^2\right]
      \\
      \makebox[2cm][c]{$\geq$}
      & \frac{1}{2}\left( \sum_{\substack{k: v_k
        > 0 \\\xi_k\in(\xi_{k^*}, \xi_{k^*}
      + \delta)}} \left(w_k^{*,\tilde{\lambda}}\right)^2 +
      \sum_{\substack{k: v_k < 0 \\\xi_k\in(\xi_{k^*}, \xi_{k^*}
      + \delta)}} \left(w_k^{*,\tilde{\lambda}}\right)^2\right) \\
      \makebox[2cm][c]{$\overset{\text{Lem. A.6}}{\underset{\delta \text{
      small enough}}{\geq}} $}
      &
        \frac{1}{4}\left(\left(\frac{w_{k_+^*}^{*,\tilde{\lambda}}}
        {v_{k_+^*}}\right)^2\sum_{\substack{k:
        v_k > 0 \\\xi_k\in(\xi_{k^*}, \xi_{k^*} + \delta)}}v_k^2 +
      \left(\frac{w_{k_-^*}^{*,\tilde{\lambda}}}{v_{k_-^*}}\right)^2
      \sum_{\substack{k:  
        v_k < 0 \\\xi_k\in(\xi_{k^*}, \xi_{k^*} +
      \delta)}}v_k^2\right)\\
      \makebox[2cm][c]{$\stackrel{\mathbb{P}}{\geq}$}
      & \frac{1}{8}
        \left(\frac{w_{k_+^*}^{*,\tilde{\lambda}}}{v_{k^*}}\right)^2
        n \delta g_\xi(\xi_{k_+^*}) \mathbb{P}(v_k
        >0)\mathbb{E}[v_k^2|\xi_k = \xi_{k^*_+}]
    \end{align*}
  
\end{Lemma}

\input{Appendix_code.tex}

\end{appendices}






%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End: