Update spec and scripts

scripts/generate_batchnorm_test.py

@@ -0,0 +1,43 @@
+import numpy as np
+from mlp.layers import BatchNormalizationLayer
+import argparse
+
+parser = argparse.ArgumentParser(description='Welcome to GAN-Shot-Learning script')
+
+parser.add_argument('--student_id', nargs="?", type=str, help='Your student id in the format "sxxxxxxx"')
+
+args = parser.parse_args()
+
+student_id = args.student_id
+
+def generate_inputs(student_id):
+    student_number = student_id
+    tests = np.arange(96).reshape((2, 3, 4, 4))
+    tests[:, 0, :, :] = float(student_number[1:3]) / 10 - 5
+    tests[:, :, 1, :] = float(student_number[3:5]) / 10 - 5
+    tests[:, 2, :, :] = float(student_number[5:7]) / 10 - 5
+    tests[0, 1, :, :] = float(student_number[7]) / 10 - 5
+    return tests
+
+test_inputs = generate_inputs(student_id)
+test_inputs = np.reshape(test_inputs, newshape=(2, -1))
+test_grads_wrt_outputs = np.arange(-48, 48).reshape((2, -1))
+
+#produce BatchNorm Layer fprop and bprop
+activation_layer = BatchNormalizationLayer(input_dim=48)
+
+beta = np.array(48*[0.3])
+gamma = np.array(48*[0.8])
+
+activation_layer.params = [gamma, beta]
+BN_fprop = activation_layer.fprop(test_inputs)
+BN_bprop = activation_layer.bprop(
+    test_inputs, BN_fprop, test_grads_wrt_outputs)
+BN_grads_wrt_params = activation_layer.grads_wrt_params(
+    test_inputs, test_grads_wrt_outputs)
+
+test_output = "BatchNormalization:\nFprop: {}\nBprop: {}\nGrads_wrt_params: {}\n"\
+    .format(BN_fprop, BN_bprop, BN_grads_wrt_params)
+
+with open("{}_batchnorm_test_file.txt".format(student_id), "w+") as out_file:
+    out_file.write(test_output)

scripts/generate_conv_test.py

@@ -0,0 +1,46 @@
+import numpy as np
+from mlp.layers import ConvolutionalLayer
+import argparse
+
+parser = argparse.ArgumentParser(description='Welcome to GAN-Shot-Learning script')
+
+parser.add_argument('--student_id', nargs="?", type=str, help='Your student id in the format "sxxxxxxx"')
+
+args = parser.parse_args()
+
+student_id = args.student_id
+
+def generate_inputs(student_id):
+    student_number = student_id
+    tests = np.arange(96).reshape((2, 3, 4, 4))
+    tests[:, 0, :, :] = float(student_number[1:3]) / 10 - 5
+    tests[:, :, 1, :] = float(student_number[3:5]) / 10 - 5
+    tests[:, 2, :, :] = float(student_number[5:7]) / 10 - 5
+    tests[0, 1, :, :] = float(student_number[7]) / 10 - 5
+    return tests
+
+
+
+test_inputs = generate_inputs(student_id)
+test_grads_wrt_outputs = np.arange(-20, 16).reshape((2, 2, 3, 3))
+inputs = np.arange(96).reshape((2, 3, 4, 4))
+kernels = np.arange(-12, 12).reshape((2, 3, 2, 2))
+biases = np.arange(2)
+
+#produce ConvolutionalLayer fprop, bprop and grads_wrt_params
+activation_layer = ConvolutionalLayer(num_input_channels=3, num_output_channels=2, input_dim_1=4, input_dim_2=4,
+                                      kernel_dim_1=2, kernel_dim_2=2)
+activation_layer.params = [kernels, biases]
+conv_fprop = activation_layer.fprop(test_inputs)
+conv_bprop = activation_layer.bprop(
+    test_inputs, conv_fprop, test_grads_wrt_outputs)
+conv_grads_wrt_params = activation_layer.grads_wrt_params(test_inputs,
+                                                          test_grads_wrt_outputs)
+
+test_output = "ConvolutionalLayer:\nFprop: {}\nBprop: {}\n" \
+              "Grads_wrt_params: {}\n".format(conv_fprop,
+            conv_bprop,
+            conv_grads_wrt_params)
+
+with open("{}_conv_test_file.txt".format(student_id), "w+") as out_file:
+    out_file.write(test_output)

spec/coursework2.tex

@@ -186,9 +186,9 @@ In part A of the coursework you will focus on using deep neural networks on EMNI
   \item Perform baseline experiments using DNNs trained on EMNIST.  Obviously there are a lot things that could be explored including hidden unit activation functions, network architectures, training hyperparameters, and the use of regularisation and dropout.   You cannot explore everything and is best to carefully investigate a few things in depth.
   \item Implement the RMSProp \citep{tieleman2012rmsprop} and Adam \citep{kingma2015adam} learning rules, by defining new classes inheriting from \texttt{GradientDescendLearningRule} in the \texttt{mlp/learning\_rules.py} module. The \texttt{MomentumLearningRule} class is an example of how to define  a learning rules which uses an additional state variable to calculate the updates to the parameters.
   \item Perform experiments to compare stochastic gradient descent, RMSProp, and Adam for deep neural network training on EMNIST, building on your earlier baseline experiments.  
-  \item Implement batch normalisation \citep{ioffe2015batch} as a class \verb+BatchNormLayer+.  You need to implement \texttt{fprop}, \texttt{bprop} and \texttt{grads\_wrt\_params} methods for this class. 
-  \item Verify the correctness of your implementation using the supplied unit tests in \verb+Batchnorm_tests.ipynb+.
-  \item Automatically create a test file \verb+sXXXXXXX_batchnorm_test.txt+, by running the provided program \verb+generate_conv_test.py+ which uses your \verb+BatchnormLayer+ class methods on a unique test vector generated using your student ID number.
+  \item Implement batch normalisation \citep{ioffe2015batch} as a class \verb+BatchNormalizationLayer+.  You need to implement \texttt{fprop}, \texttt{bprop} and \texttt{grads\_wrt\_params} methods for this class. 
+  \item Verify the correctness of your implementation using the supplied unit tests in\\\verb+BatchNormalizationLayer_tests.ipynb+.
+  \item Automatically create a test file \verb+sXXXXXXX_batchnorm_test.txt+, by running the provided program \verb+generate_batchnorm_test.py+ which uses your \verb+BatchNormalizationLayer+ class methods on a unique test vector generated using your student ID number.
   \item Perform experiments on EMNIST to investigate the impact of using batch normalisation in deep neural networks, building on your earlier experiments.  
 \end{enumerate}
 In the above experiments you should use the validation set to assess accuracy.  Use the test set at the end to assess the accuracy of the deep neural network architecture and training setup that you judge to be the best.
@@ -199,7 +199,7 @@ In the above experiments you should use the validation set to assess accuracy.
 In part B of the coursework you should implement convolutional  and max-pooling layers, and carry out experiments using a convolutional networks with one and two convolutional layers.  
 \begin{enumerate}
   \item Implement a convolutional layer as a class \verb+ConvolutionalLayer+.  You need to implement \texttt{fprop}, \texttt{bprop} and \texttt{grads\_wrt\_params} methods for this class. 
-  \item Verify the correctness of your implementation using the supplied unit tests in \verb+Convolutional_layer_tests.ipynb+.
+  \item Verify the correctness of your implementation using the supplied unit tests in\\\verb+ConvolutionalLayer_tests.ipynb+.
   \item Automatically create a test file \verb+sXXXXXXX_conv_test.txt+, by running the provided program \verb+generate_conv_test.py+ which uses your \verb+ConvolutionalLayer+ class methods on a unique test vector generated using your student ID number.
   \item Implement a max-pooling layer. Non-overlapping pooling (which was assumed in the lecture presentation) is required. You may also implement a more generic solution with striding as well. 
   \item Construct and train networks containing one and two convolutional layers, and max-pooling layers, using the Balanced EMNIST data, reporting your experimental results.  As a default use convolutional kernels of dimension 5x5 (stride 1) and pooling regions of 2x2 (stride 2, hence non-overlapping).  As a default convolutional networks with two convolutional layers,  investigate a network with two convolutional+maxpooling layers with 5 feature maps in the first convolutional layer, and 10 feature maps in the second convolutional layer.