\section{Code...}
In this ... the implementations of the models used in ... are
given. The randomized shallow neural network used in CHAPTER... are
implemented in Scala from ground up to ensure the model is exactly to
... of Theorem~\ref{theo:main1}.

The neural networks used in CHAPTER are implemented in python using
the Keras framework given in Tensorflow. Tensorflow is a library
containing highly efficient GPU implementations of most important
tensor operations, such as convolution as well as efficient algorithms
for training neural networks (computing derivatives, updating parameters).
\begin{itemize}
  \item Code for randomized shallow neural network
  \item Code for keras
\end{itemize}

\begin{lstfloat}
\begin{lstlisting}[language=iPython]
import breeze.stats.distributions.Uniform
import breeze.stats.distributions.Gaussian
import scala.language.postfixOps

object Activation {
  def apply(x: Double): Double = math.max(0, x)

  def d(x: Double): Double = if (x > 0) 1 else 0
}

class RSNN(val n: Int, val gamma: Double = 0.001) {
    val g_unif = Uniform(-10, 10)
    val g_gauss = Gaussian(0, 5)

    val xis = g_unif.sample(n)
    val vs = g_gauss.sample(n)
    val bs = xis zip vs map {case(xi, v) => xi * v}

    def computeL1(x: Double) = (bs zip vs) map {
        case (b, v) => Activation(b + v * x) }
  
    def computeL2(l1: Seq[Double], ws: Seq[Double]): Double =
        (l1 zip ws) map { case (l, w) => w * l } sum
  
    def output(ws: Seq[Double])(x: Double): Double =
        computeL2(computeL1(x), ws)
  
    def learn(data: Seq[(Double, Double)], ws: Seq[Double],
        lamb: Double, gamma: Double): Seq[Double] = {
    
        lazy val deltas = data.map {
            case (x, y) =>
                val l1 = computeL1(x)
                val out = computeL2(l1, ws)
                (l1 zip ws) map {case (l1, w) => (l1 * 2 * (out - y) +
                    lam * 2 * w)  * gamma * -1}
        }
  
        deltas.foldRight(ws)(
            (delta, ws) => ws zip (delta) map { case (w, d) => w + d })  
    }

    def train(data: Seq[(Double, Double)], iter: Int, lam: Double,
        gamma: Double = gamma): (Seq[Double], Double => Double) = {
      
        val ws = (1 to iter).foldRight((1 to n).map(
            _ => 0.0) :Seq[Double])((i, w) => {
            println(s"Training iteration $i")
            println(w.sum/w.length)
            learn(data, w, lam, gamma / 10)
        })
        (ws, output(ws))
    }
}
\end{lstlisting}
  \caption{Scala code used to build and train the ridge penalized
    randomized shallow neural network in .... The parameter \textit{lam}
    in the train function represents the $\lambda$ parameter in the error
    function. The parameters \textit{n} and \textit{gamma} set the number
    of hidden nodes and the stepsize for training.}
  \label{lst:rsnn}
\end{lstfloat}
\clearpage
\begin{lstfloat}
\begin{lstlisting}[language=iPython]
import tensorflow as tf
import numpy as np
from tensorflow.keras.callbacks import CSVLogger
from tensorflow.keras.preprocessing.image import ImageDataGenerator

mnist = tf.keras.datasets.mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_train = x_train / 255.0
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
x_test = x_test / 255.0

y_train = tf.keras.utils.to_categorical(y_train)
y_test = tf.keras.utils.to_categorical(y_test)

model = tf.keras.models.Sequential()
model.add(tf.keras.layers.Conv2D(24,kernel_size=5,padding='same',
    activation='relu',input_shape=(28,28,1)))
model.add(tf.keras.layers.MaxPool2D())
model.add(tf.keras.layers.Conv2D(64,kernel_size=5,padding='same',
    activation='relu'))
model.add(tf.keras.layers.MaxPool2D(padding='same'))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
model.compile(optimizer='adam', loss="categorical_crossentropy",
    metrics=["accuracy"])

datagen = ImageDataGenerator(
    rotation_range = 30,
    zoom_range = 0.15,
    width_shift_range=2,
    height_shift_range=2,
    shear_range = 1)

csv_logger = CSVLogger(<Target File>)
    
history = model.fit(datagen.flow(x_train, y_train, batch_size=50),
validation_data=(x_test, y_test),
    epochs=125, callbacks=[csv_logger],
    steps_per_epoch = x_train.shape[0]//50) 

\end{lstlisting}
  \caption{Python code for the model used... the MNIST handwritten digits
    dataset.}
  \label{lst:handwriting}
\end{lstfloat}
\clearpage
\begin{lstfloat}
\begin{lstlisting}[language=iPython]
import tensorflow as tf
import numpy as np
from tensorflow.keras.callbacks import CSVLogger
from tensorflow.keras.preprocessing.image import ImageDataGenerator
mnist = tf.keras.datasets.fashion_mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.reshape(x_train.shape[0], 28, 28, 1)
x_test = x_test.reshape(x_test.shape[0], 28, 28, 1)
x_train, x_test = x_train / 255.0, x_test / 255.0

y_train = tf.keras.utils.to_categorical(y_train)
y_test = tf.keras.utils.to_categorical(y_test)

model = tf.keras.Sequential()
model.add(tf.keras.layers.Conv2D(filters = 32, kernel_size = (3, 3),
    activation='relu', input_shape = (28, 28, 1), padding='same'))
model.add(tf.keras.layers.Conv2D(filters = 32, kernel_size = (2, 2), activation='relu', padding = 'same'))
model.add(tf.keras.layers.MaxPool2D(strides=(2,2)))
model.add(tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), activation='relu', padding='same'))
model.add(tf.keras.layers.Conv2D(filters = 64, kernel_size = (3, 3), activation='relu', padding='same'))
model.add(tf.keras.layers.MaxPool2D(strides=(2,2)))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.2))
model.add(tf.keras.layers.Dense(10, activation='softmax'))

model.compile(optimizer=tf.keras.optimizers.Adam(lr = 1e-3), loss="categorical_crossentropy", metrics=["accuracy"])

datagen = ImageDataGenerator(
    rotation_range = 15,
    zoom_range = 0.1,
    width_shift_range=2,
    height_shift_range=2,
    shear_range = 0.5,
    fill_mode = 'constant',
    cval = 0)
        
    csv_logger = CSVLogger(<Target File>)
    
    history = model.fit(datagen.flow(x_train, y_train, batch_size=30),
        steps_per_epoch=x_train.shape[0]//30,
        validation_data=(x_test, y_test),
        epochs=125, callbacks=[csv_logger],
        shuffle=True)

\end{lstlisting}
  \caption{Python code for the model used... the fashion MNIST
    dataset.}
  \label{lst:fashion}
\end{lstfloat}
\clearpage
\begin{lstfloat}
\begin{lstlisting}[language=iPython]
def get_random_sample(a, b, number_of_samples=10):
    x = []
    y = []
    for category_number in range(0,10):
    # get all samples of a category
        train_data_category = a[b==category_number]
        # pick a number of random samples from the category
        train_data_category = train_data_category[np.random.randint(
            train_data_category.shape[0], size=number_of_samples), :]
        x.extend(train_data_category)
        y.append([category_number]*number_of_samples)

        return (np.asarray(x).reshape(-1, 28, 28, 1),
            np.asarray(y).reshape(10*number_of_samples,1))
\end{lstlisting}
  \caption{Python code used to generate the datasets containing a
    certain amount of random datapoints per class.}
\end{lstfloat}
%%% Local Variables:
%%% mode: latex
%%% TeX-master: "main"
%%% End: