Masterarbeit/TeX/Appendix_code.tex

\section{Implementations}
In this section the implementations of the models used are given.
The randomized shallow neural network used in Section~\ref{sec:conv} is
implemented in Scala. No pre-existing frameworks were used to ensure
the implementation was according to the definitions used in Theorem~\ref{theo:main1}.

The neural networks used in Section~\ref{sec:cnn} are implemented in Python using
the Keras framework given in TensorFlow. TensorFlow is a library
containing highly efficient GPU implementations of a wide variety of
tensor operations and algorithms
for training neural networks.% (computing derivatives, updating parameters).

\vspace*{-0.5cm}
\begin{lstfloat}
\begin{lstlisting}[language=iPython]
import breeze.stats.distributions.Uniform
import breeze.stats.distributions.Gaussian
import scala.language.postfixOps

object Activation {
  def apply(x: Double): Double = math.max(0, x)

  def d(x: Double): Double = if (x > 0) 1 else 0
}

class RSNN(val n: Int, val gamma: Double = 0.001) {
    val g_unif = Uniform(-10, 10)
    val g_gauss = Gaussian(0, 5)

    val xis = g_unif.sample(n)
    val vs = g_gauss.sample(n)
    val bs = xis zip vs map {case(xi, v) => xi * v}

    def computeL1(x: Double) = (bs zip vs) map {
        case (b, v) => Activation(b + v * x) }
  
    def computeL2(l1: Seq[Double], ws: Seq[Double]): Double =
        (l1 zip ws) map { case (l, w) => w * l } sum
  
    def output(ws: Seq[Double])(x: Double): Double =
        computeL2(computeL1(x), ws)
  
    def learn(data: Seq[(Double, Double)], ws: Seq[Double],
        lamb: Double, gamma: Double): Seq[Double] = {
    
        lazy val deltas = data.map {
            case (x, y) =>
                val l1 = computeL1(x)
                val out = computeL2(l1, ws)
                (l1 zip ws) map {case (l1, w) => (l1 * 2 * (out - y) +
                    lam * 2 * w)  * gamma * -1}
        }
  
        deltas.foldRight(ws)(
            (delta, ws) => ws zip (delta) map { case (w, d) => w + d })  
    }

    def train(data: Seq[(Double, Double)], iter: Int, lam: Double,
        gamma: Double = gamma): (Seq[Double], Double => Double) = {
      
        val ws = (1 to iter).foldRight((1 to n).map(
            _ => 0.0) :Seq[Double])((i, w) => {
            println(s"Training iteration $i")
            println(w.sum/w.length)
            learn(data, w, lam, gamma / 10)
        })
        (ws, output(ws))
    }
}
\end{lstlisting}
  \caption{Scala code used to build and train the ridge penalized
    randomized shallow neural network in Section~\ref{sec:rsnn_sim}.}
  % The parameter \textit{lam}
  %   in the train function represents the $\lambda$ parameter in the error
  %   function. The parameters \textit{n} and \textit{gamma} set the number
  %   of hidden nodes and the stepsize for training.}
  \label{lst:rsnn}
\end{lstfloat}
\clearpage
\begin{lstfloat}
\begin{lstlisting}[language=iPython]
import tensorflow as tf
import numpy as np
from tensorflow.keras.callbacks import CSVLogger
from tensorflow.keras.preprocessing.image import ImageDataGenerator

mnist = tf.keras.datasets.mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_train = x_train / 255.0
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
x_test = x_test / 255.0

y_train = tf.keras.utils.to_categorical(y_train)
y_test = tf.keras.utils.to_categorical(y_test)

model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(24,kernel_size=5,padding='same',
    activation='relu',input_shape=(28,28,1)))
model.add(tf.keras.layers.MaxPool2D())
model.add(tf.keras.layers.Conv2D(64,kernel_size=5,padding='same',
    activation='relu'))
model.add(tf.keras.layers.MaxPool2D(padding='same'))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss="categorical_crossentropy",
    metrics=["accuracy"])

datagen = ImageDataGenerator(
    rotation_range = 30,
    zoom_range = 0.15,
    width_shift_range=2,
    height_shift_range=2,
    shear_range = 1)

csv_logger = CSVLogger(<Target File>)
    
history = model.fit(datagen.flow(x_train, y_train, batch_size=50),
validation_data=(x_test, y_test),
    epochs=125, callbacks=[csv_logger],
    steps_per_epoch = x_train.shape[0]//50) 

\end{lstlisting}
  \caption{Python code used to build the network modeling the MNIST
    handwritten digits data set.}
  \label{lst:handwriting}
\end{lstfloat}
\clearpage
\begin{lstfloat}
\begin{lstlisting}[language=iPython]
import tensorflow as tf
import numpy as np
from tensorflow.keras.callbacks import CSVLogger
from tensorflow.keras.preprocessing.image import ImageDataGenerator
mnist = tf.keras.datasets.fashion_mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
x_train, x_test = x_train / 255.0, x_test / 255.0

y_train = tf.keras.utils.to_categorical(y_train)
y_test = tf.keras.utils.to_categorical(y_test)

model = tf.keras.Sequential()
model.add(tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3),
    activation='relu', input_shape = (28, 28, 1), padding='same'))
model.add(tf.keras.layers.Conv2D(filters = 32, kernel_size = (2, 2), activation='relu', padding = 'same'))
model.add(tf.keras.layers.MaxPool2D(strides=(2,2)))
model.add(tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), activation='relu', padding='same'))
model.add(tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), activation='relu', padding='same'))
model.add(tf.keras.layers.MaxPool2D(strides=(2,2)))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Dense(10, activation='softmax'))

model.compile(optimizer=tf.keras.optimizers.Adam(lr = 1e-3), loss="categorical_crossentropy", metrics=["accuracy"])

datagen = ImageDataGenerator(
    rotation_range = 6,
    zoom_range = 0.15,
    width_shift_range=2,
    height_shift_range=2,
    shear_range = 0.15,
    fill_mode = 'constant',
    cval = 0)
        
    csv_logger = CSVLogger(<Target File>)
    
    history = model.fit(datagen.flow(x_train, y_train, batch_size=30),
        steps_per_epoch=x_train.shape[0]//30,
        validation_data=(x_test, y_test),
        epochs=125, callbacks=[csv_logger],
        shuffle=True)

\end{lstlisting}
  \caption[Python Code for fashion MNIST]{Python code
    used to build the network modeling the fashion MNIST data set.}
  \label{lst:fashion}
\end{lstfloat}
\clearpage
\begin{lstfloat}
\begin{lstlisting}[language=iPython]
def get_random_sample(a, b, number_of_samples=10):
    x = []
    y = []
    for category_number in range(0,10):
    # get all samples of a category
        train_data_category = a[b==category_number]
        # pick a number of random samples from the category
        train_data_category = train_data_category[np.random.randint(
            train_data_category.shape[0], size=number_of_samples), :]
        x.extend(train_data_category)
        y.append([category_number]*number_of_samples)

        return (np.asarray(x).reshape(-1, 28, 28, 1),
            np.asarray(y).reshape(10*number_of_samples,1))
\end{lstlisting}
  \caption{Python code used to generate the data sets containing a
    certain amount of random data points per class.}
\end{lstfloat}

\section{Additional Comparisons}
\label{app:comp}
In this section, comparisons of cross entropy loss and training
accuracy for the models trained in Section~\ref{sec:smalldata} are given.
\begin{figure}[h]
  \centering
  \small
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = \textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch},ylabel = {Test Loss}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_1.mean};  
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_02_1.mean};
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_1.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_02_1.mean};

        
        \addlegendentry{\footnotesize{Default}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G. + D. 0.2}}
        \addlegendentry{\footnotesize{D. 0.4}}
        \addlegendentry{\footnotesize{Default}}
      \end{axis}
    \end{tikzpicture}
    \caption{1 Sample per Class}
    \vspace{0.25cm}
  \end{subfigure}
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = \textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch},ylabel = {Test Loss}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_00_10.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_02_10.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_00_10.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_02_10.mean};


        \addlegendentry{\footnotesize{Default.}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G + D. 0.2}}
      \end{axis}
    \end{tikzpicture}
    \caption{10 Samples per Class}
  \end{subfigure}
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = 0.9875\textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch}, ylabel = {Test Loss}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_00_100.mean};
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_02_100.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_00_100.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_02_100.mean}; 
        
        \addlegendentry{\footnotesize{Default.}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G + D. 0.2}}
      \end{axis}
    \end{tikzpicture}
    \caption{100 Samples per Class}
    \vspace{.25cm}
  \end{subfigure}
  \caption[Mean Test Loss for Subsets of MNIST Handwritten
  Digits]{Mean test cross entropy loss of the models fitting the
    sampled subsets of MNIST 
    handwritten digits over the 125 epochs of training.}
\end{figure}

\begin{figure}[h]
  \centering
  \small
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style =
                   {draw = none}, width = \textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch},ylabel = {Test Loss}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_0_1.mean};  
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_2_1.mean};
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_0_1.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_2_1.mean};

        
        \addlegendentry{\footnotesize{Default}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G. + D. 0.2}}
        \addlegendentry{\footnotesize{D. 0.4}}
      \end{axis}
    \end{tikzpicture}
    \caption{1 Sample per Class}
    \vspace{0.25cm}
  \end{subfigure}
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = \textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch},ylabel = {Test Loss}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}, ymin = {0.62}]
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_0_10.mean};  
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_2_10.mean};
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_0_10.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_2_10.mean};

        
        \addlegendentry{\footnotesize{Default.}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G + D. 0.2}}
      \end{axis}
    \end{tikzpicture}
    \caption{10 Samples per Class}
  \end{subfigure}
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = 0.9875\textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch}, ylabel = {Test Loss}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_0_100.mean};  
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_2_100.mean};
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_0_100.mean}; 
        \addplot table
        [x=epoch, y=val_loss, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_2_100.mean};
        
        \addlegendentry{\footnotesize{Default.}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G + D. 0.2}}
      \end{axis}
    \end{tikzpicture}
    \caption{100 Samples per Class}
    \vspace{.25cm}
  \end{subfigure}
  \caption[Mean Test Accuracies for Subsets of Fashion MNIST]{Mean
    test cross entropy loss of the models fitting the sampled subsets
    of fashion MNIST
    over the 125 epochs of training.}
\end{figure}

\begin{figure}[h]
  \centering
  \small
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = \textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch},ylabel = {Training Accuracy}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_1.mean};  
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_02_1.mean};
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_1.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_02_1.mean};

        
        \addlegendentry{\footnotesize{Default}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G. + D. 0.2}}
        \addlegendentry{\footnotesize{D. 0.4}}
        \addlegendentry{\footnotesize{Default}}
      \end{axis}
    \end{tikzpicture}
    \caption{1 Sample per Class}
    \vspace{0.25cm}
  \end{subfigure}
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = \textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch},ylabel = {Test Accuracy}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_00_10.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_02_10.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_00_10.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_02_10.mean};


        \addlegendentry{\footnotesize{Default.}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G + D. 0.2}}
      \end{axis}
    \end{tikzpicture}
    \caption{10 Samples per Class}
  \end{subfigure}
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = 0.9875\textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch}, ylabel = {Training Accuracy}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}, ymin = {0.92}]
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_00_100.mean};
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_dropout_02_100.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_00_100.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/adam_datagen_dropout_02_100.mean}; 
        
        \addlegendentry{\footnotesize{Default.}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G + D. 0.2}}
      \end{axis}
    \end{tikzpicture}
    \caption{100 Samples per Class}
    \vspace{.25cm}
  \end{subfigure}
  \caption[Mean Training Accuracies for Subsets of MNIST Handwritten
  Digits]{Mean training accuracies of the models fitting the sampled
    subsets of MNIST 
    handwritten digits over the 125 epochs of training.}
\end{figure}

\begin{figure}[h]
  \centering
  \small
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style =
                   {draw = none}, width = \textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch},ylabel = {Training Accuracy}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_0_1.mean};  
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_2_1.mean};
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_0_1.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_2_1.mean};

        
        \addlegendentry{\footnotesize{Default}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G. + D. 0.2}}
        \addlegendentry{\footnotesize{D. 0.4}}
      \end{axis}
    \end{tikzpicture}
    \caption{1 Sample per Class}
    \vspace{0.25cm}
  \end{subfigure}
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = \textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch},ylabel = {Training Accuracy}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}, ymin = {0.62}]
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_0_10.mean};  
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_2_10.mean};
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_0_10.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_2_10.mean};

        
        \addlegendentry{\footnotesize{Default.}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G + D. 0.2}}
      \end{axis}
    \end{tikzpicture}
    \caption{10 Samples per Class}
  \end{subfigure}
  \begin{subfigure}[h]{\textwidth}
    \begin{tikzpicture}
      \begin{axis}[legend cell align={left},yticklabel style={/pgf/number format/fixed,
                     /pgf/number format/precision=3},tick style = {draw = none}, width = 0.9875\textwidth,
        height = 0.4\textwidth, legend style={at={(0.9825,0.0175)},anchor=south east},
        xlabel = {Epoch}, ylabel = {Training Accuracy}, cycle
        list/Dark2, every axis plot/.append style={line width
          =1.25pt}]
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_0_100.mean};  
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_dropout_2_100.mean};
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_0_100.mean}; 
        \addplot table
        [x=epoch, y=accuracy, col sep=comma, mark = none]
        {Figures/Data/fashion_datagen_dropout_2_100.mean};
        
        \addlegendentry{\footnotesize{Default.}}
        \addlegendentry{\footnotesize{D. 0.2}}
        \addlegendentry{\footnotesize{G.}}
        \addlegendentry{\footnotesize{G + D. 0.2}}
      \end{axis}
    \end{tikzpicture}
    \caption{100 Samples per Class}
    \vspace{.25cm}
  \end{subfigure}
  \caption[Mean Training Accuracies for Subsets of Fashion MNIST]{Mean
    training accuracies of the models fitting the sampled subsets of fashion MNIST 
    over the 125 epochs of training.}
\end{figure}

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End: