changes

.gitignore

@@ -86,3 +86,4 @@ report/mlp-cw2-template.bbl
 report/mlp-cw2-template.blg
 
 venv
+saved_models

pytorch_mlp_framework/experiment_builder.py

@@ -79,6 +79,7 @@ class ExperimentBuilder(nn.Module):
         print("Total number of conv layers", num_conv_layers)
         print("Total number of linear layers", num_linear_layers)
 
+        print(f"Learning rate: {learning_rate}")
         self.optimizer = optim.Adam(
             self.parameters(),
             amsgrad=False,

report/VGG38_BN/result_outputs/summary.csv

@@ -0,0 +1,101 @@
+train_acc,train_loss,val_acc,val_loss
+0.027410526315789472,4.440032,0.0368,4.238186
+0.0440842105263158,4.1909122,0.0644,4.1239405
+0.05604210526315791,4.0817885,0.0368,4.495799
+0.0685263157894737,3.984858,0.0964,3.8527937
+0.08345263157894738,3.8947835,0.09080000000000002,3.8306112
+0.09391578947368423,3.8246264,0.10399999999999998,3.7504945
+0.10189473684210527,3.760145,0.1124,3.6439042
+0.11197894736842108,3.704831,0.0992,3.962508
+0.12534736842105265,3.6408415,0.1404,3.516474
+0.1385894736842105,3.5672796,0.1444,3.5242612
+0.14873684210526317,3.5145628,0.12960000000000002,3.5745378
+0.16103157894736844,3.4476008,0.1852,3.3353982
+0.16846315789473681,3.399858,0.15600000000000003,3.453797
+0.1760210526315789,3.3611393,0.1464,3.5799885
+0.18625263157894736,3.3005812,0.196,3.201007
+0.19233684210526317,3.26565,0.17439999999999997,3.397586
+0.19625263157894737,3.2346153,0.212,3.169959
+0.20717894736842105,3.174345,0.2132,3.0981174
+0.2136,3.1425776,0.2036,3.2191591
+0.2217684210526316,3.094137,0.236,3.0018876
+0.23069473684210529,3.0539455,0.20440000000000003,3.1800296
+0.23395789473684211,3.0338168,0.22599999999999998,3.0360818
+0.24463157894736842,2.9761615,0.2588,2.8876188
+0.25311578947368424,2.931479,0.2,3.242481
+0.25795789473684216,2.900163,0.28320000000000006,2.830947
+0.26789473684210524,2.8484874,0.2768,2.8190458
+0.2709263157894737,2.833472,0.2352,3.0098538
+0.2816421052631579,2.7842317,0.29560000000000003,2.7288156
+0.28764210526315787,2.745757,0.2648,2.8955112
+0.2930315789473684,2.7276495,0.27680000000000005,2.8336413
+0.3001263157894737,2.6826382,0.316,2.6245823
+0.3068421052631579,2.658441,0.27,2.9279957
+0.30909473684210526,2.638565,0.31160000000000004,2.637653
+0.3213263157894737,2.5939283,0.31799999999999995,2.627816
+0.3211157894736843,2.579544,0.25079999999999997,2.9502957
+0.3259999999999999,2.5540712,0.3332,2.569941
+0.3336421052631579,2.5239582,0.278,2.7676308
+0.3371368421052632,2.5109046,0.2916,2.725589
+0.34404210526315787,2.4714804,0.34120000000000006,2.4782379
+0.3500631578947368,2.4545348,0.30600000000000005,2.6625924
+0.34976842105263156,2.4408882,0.342,2.5351026
+0.3586315789473684,2.4116046,0.3452,2.450749
+0.3568421052631579,2.4133172,0.3288,2.5647113
+0.3630947368421052,2.3772728,0.36519999999999997,2.388074
+0.37069473684210524,2.3505116,0.324,2.5489926
+0.37132631578947367,2.352426,0.33680000000000004,2.5370462
+0.37606315789473677,2.319005,0.3712,2.3507965
+0.3800210526315789,2.3045664,0.33,2.6327293
+0.38185263157894733,2.2965574,0.3764,2.364877
+0.38785263157894734,2.269467,0.37799999999999995,2.330837
+0.3889684210526316,2.26941,0.3559999999999999,2.513778
+0.3951789473684211,2.2413251,0.3888,2.2839465
+0.3944421052631579,2.2319226,0.35919999999999996,2.4310353
+0.4,2.220305,0.3732,2.348543
+0.4051157894736842,2.1891508,0.39440000000000003,2.2730627
+0.40581052631578945,2.1873925,0.33399999999999996,2.5648093
+0.4067789473684211,2.1817088,0.4044,2.2244952
+0.41555789473684207,2.1543047,0.39759999999999995,2.220972
+0.4170526315789474,2.14905,0.33399999999999996,2.6612198
+0.41762105263157895,2.1321266,0.3932,2.2343464
+0.42341052631578946,2.1131704,0.37800000000000006,2.327929
+0.4212842105263158,2.112597,0.376,2.3302126
+0.4295157894736842,2.0925663,0.4100000000000001,2.175698
+0.4299368421052632,2.0846903,0.3772,2.3750577
+0.43134736842105265,2.075184,0.4044,2.1888158
+0.43829473684210524,2.045202,0.41239999999999993,2.1673117
+0.43534736842105265,2.0590534,0.37440000000000007,2.3269994
+0.4417684210526316,2.0356588,0.42,2.1668334
+0.4442736842105263,2.028207,0.41239999999999993,2.2346516
+0.44581052631578943,2.021492,0.40519999999999995,2.2030904
+0.44884210526315793,2.0058675,0.4296,2.0948715
+0.45071578947368424,1.993417,0.39,2.2856123
+0.45130526315789476,1.9970801,0.43599999999999994,2.110219
+0.45686315789473686,1.9651922,0.4244,2.1253593
+0.4557263157894737,1.9701725,0.3704,2.4576838
+0.4609684210526315,1.956996,0.4412,2.0626938
+0.4639789473684211,1.9407912,0.398,2.3076272
+0.46311578947368426,1.9410807,0.4056,2.2181008
+0.4686736842105263,1.918824,0.45080000000000003,2.030652
+0.4650315789473684,1.924879,0.3948,2.2926931
+0.46964210526315786,1.9188553,0.43599999999999994,2.107239
+0.47357894736842104,1.8991861,0.43119999999999997,2.067097
+0.47212631578947367,1.8987728,0.41359999999999997,2.1667569
+0.4773263157894737,1.8892545,0.46,2.0283196
+0.4802526315789474,1.8736148,0.41960000000000003,2.1698954
+0.47406315789473685,1.8849738,0.43399999999999994,2.1001608
+0.48627368421052636,1.8492608,0.45520000000000005,1.9936249
+0.48589473684210527,1.8534511,0.38439999999999996,2.354954
+0.48667368421052637,1.8421199,0.44120000000000004,2.0467849
+0.4902736842105263,1.8265136,0.45519999999999994,2.0044358
+0.4879789473684211,1.838593,0.3984,2.3019247
+0.49204210526315795,1.8199797,0.4656,1.9858631
+0.4945894736842105,1.805858,0.436,2.1293921
+0.4939578947368421,1.8174701,0.4388,2.0611947
+0.4961684210526316,1.7953233,0.4612,1.9728945
+0.49610526315789477,1.7908033,0.42440000000000005,2.1648548
+0.4996,1.7908286,0.4664,1.9897026
+0.5070105263157895,1.7658812,0.452,2.0411723
+0.5027368421052631,1.7692825,0.4136000000000001,2.280331
+0.5062315789473685,1.7649119,0.4768,1.9493303

report/VGG38_BN/result_outputs/test_summary.csv

@@ -0,0 +1,2 @@
+test_acc,test_loss
+0.46970000000000006,1.9579598

report/VGG38_BN_RC/result_outputs/summary.csv

@@ -0,0 +1,101 @@
+train_acc,train_loss,val_acc,val_loss
+0.04040000000000001,4.2986817,0.07600000000000001,3.9793916
+0.07663157894736841,3.948711,0.09840000000000002,3.8271046
+0.1072842105263158,3.7670445,0.0908,3.8834984
+0.14671578947368422,3.544252,0.1784,3.3180876
+0.18690526315789474,3.3382895,0.1672,3.4958847
+0.2185684210526316,3.1613564,0.23240000000000002,3.0646808
+0.2584,2.9509778,0.2904,2.7620668
+0.2886736842105263,2.7674758,0.2504,3.083242
+0.3186736842105263,2.6191177,0.34600000000000003,2.5320892
+0.3488421052631579,2.4735146,0.3556,2.463249
+0.36701052631578945,2.3815694,0.32480000000000003,2.6590502
+0.39258947368421054,2.2661598,0.41200000000000003,2.215237
+0.40985263157894736,2.1811035,0.3644,2.4625826
+0.42557894736842106,2.1193688,0.3896,2.2802749
+0.4452,2.0338347,0.45080000000000003,2.0216491
+0.45298947368421055,1.9886738,0.3768,2.4903286
+0.4690105263157895,1.9385177,0.46519999999999995,1.9589043
+0.48627368421052636,1.8654134,0.46199999999999997,1.9572229
+0.4910947368421053,1.836772,0.3947999999999999,2.371203
+0.5033052631578947,1.7882212,0.4864,1.8270072
+0.515578947368421,1.7451773,0.418,2.2281988
+0.5166526315789474,1.7310464,0.4744,1.9468222
+0.532,1.6639497,0.5176,1.7627875
+0.534821052631579,1.6504371,0.426,2.2908173
+0.5399578947368422,1.6263881,0.5092,1.7892419
+0.5538105263157893,1.5786182,0.5184,1.7781507
+0.5530526315789474,1.5743873,0.45480000000000004,2.052206
+0.5610526315789474,1.5367776,0.5404000000000001,1.6886607
+0.5709263157894736,1.508275,0.5072000000000001,1.8317349
+0.5693894736842106,1.5026951,0.49760000000000004,1.9268813
+0.5827368421052632,1.4614111,0.5484,1.6791071
+0.583557894736842,1.4580216,0.4744,2.084504
+0.5856842105263159,1.4402864,0.5468,1.6674811
+0.5958105263157895,1.4054152,0.5468,1.7081916
+0.5964631578947368,1.4043275,0.4988,1.8901508
+0.6044631578947368,1.3692447,0.548,1.6456038
+0.6065473684210526,1.3562685,0.5448,1.7725601
+0.6055578947368421,1.3638091,0.52,1.803752
+0.6169684210526316,1.3224502,0.5688,1.6048553
+0.6184421052631579,1.3228824,0.4772,2.0309162
+0.6193894736842105,1.312684,0.5496,1.6357917
+0.6287368421052631,1.2758818,0.5552,1.7120187
+0.6270105263157894,1.2829372,0.4872000000000001,1.9630791
+0.6313473684210527,1.2609128,0.5632,1.6049384
+0.6374736842105263,1.2429903,0.5516,1.7101723
+0.6342947368421055,1.2540665,0.5272,1.8112053
+0.642778947368421,1.2098345,0.5692,1.5996393
+0.6447368421052632,1.217454,0.5056,2.087292
+0.6437052631578949,1.2123955,0.5660000000000001,1.6426488
+0.6533263157894735,1.1804259,0.5672,1.6429158
+0.6521052631578947,1.1856273,0.5316000000000001,1.8833923
+0.658021052631579,1.1663536,0.5652,1.6239171
+0.6622947368421054,1.1522906,0.5376000000000001,1.8352613
+0.6543789473684212,1.1700194,0.5539999999999999,1.7920883
+0.6664,1.1246897,0.5828,1.5657492
+0.6645473684210526,1.1307288,0.5296,1.8285477
+0.6647157894736843,1.1294464,0.5852,1.59438
+0.6713473684210526,1.1020554,0.5647999999999999,1.6256377
+0.6691368421052631,1.1129124,0.5224,1.9497899
+0.6737684210526315,1.0941163,0.5708,1.5900868
+0.6765473684210527,1.0844595,0.55,1.7522817
+0.6762947368421053,1.0832069,0.5428000000000001,1.8020345
+0.6799789473684209,1.0637755,0.5864,1.5690281
+0.6808421052631578,1.066873,0.5168,1.9964217
+0.6843157894736842,1.0618489,0.5720000000000001,1.6391727
+0.6866736842105262,1.0432214,0.5731999999999999,1.6571078
+0.6877684210526315,1.0442319,0.5192,2.0341485
+0.6890105263157895,1.0338738,0.5836,1.5887364
+0.693642105263158,1.0206536,0.5456,1.8537303
+0.6905894736842106,1.0271776,0.5548000000000001,1.8022745
+0.6981263157894737,1.001102,0.5852,1.5923084
+0.6986105263157896,1.0052379,0.512,2.011443
+0.698042105263158,0.9990784,0.5744,1.638558
+0.7031578947368421,0.977477,0.5816,1.5790274
+0.7013473684210526,0.98766434,0.5448000000000001,1.8414693
+0.7069684210526315,0.9691622,0.59,1.5866013
+0.7061894736842105,0.9620083,0.55,1.7695292
+0.7050526315789474,0.9689725,0.5408,1.8329593
+0.7101052631578948,0.95279986,0.5852,1.5835829
+0.7122315789473684,0.9483001,0.5224,1.9749893
+0.7115157894736842,0.94911486,0.5808,1.6965445
+0.7166315789473684,0.9338312,0.5788,1.6249495
+0.7120631578947368,0.9428737,0.5224,1.9721117
+0.7197263157894737,0.92057914,0.5960000000000001,1.6235417
+0.7258315789473684,0.9071854,0.528,2.0651033
+0.7186947368421053,0.922529,0.5628,1.7508049
+0.7257684210526316,0.9007169,0.5980000000000001,1.5797865
+0.7254105263157896,0.89657074,0.5472,1.8673587
+0.7229263157894736,0.90324384,0.5771999999999999,1.6998875
+0.7308842105263157,0.8757633,0.5856,1.6750972
+0.7254947368421052,0.8956531,0.5479999999999999,1.9809356
+0.7302105263157894,0.8803156,0.5960000000000001,1.6343199
+0.7353473684210525,0.8630421,0.56,1.9686066
+0.732021052631579,0.8823739,0.5632,1.8139118
+0.7324631578947367,0.8676047,0.5952000000000001,1.6235788
+0.7366526315789473,0.85581774,0.5392,1.9346147
+0.7340210526315789,0.8636227,0.5868,1.6743768
+0.7416631578947368,0.84529686,0.5836,1.6691054
+0.734757894736842,0.85352796,0.516,2.227477
+0.7435368421052632,0.83374214,0.582,1.697568

report/VGG38_BN_RC/result_outputs/test_summary.csv

@@ -0,0 +1,2 @@
+test_acc,test_loss
+0.6018000000000001,1.5933747

report/mlp-cw2-questions.tex

@@ -44,20 +44,25 @@ Our results suggest that the increased capacity of VGG38 does not translate into
 }
 
 %% Question 3:
+% In this coursework, we didn't incorporate residual connections to the downsampling layers. Explain and justify what would need to be changed in order to add residual connections to the downsampling layers. Give and explain 2 ways of incorporating these changes and discuss pros and cons of each.
 \newcommand{\questionThree} {
-\youranswer{Question 3 - In this coursework, we didn't incorporate residual connections to the downsampling layers. Explain and justify what would need to be changed in order to add residual connections to the downsampling layers. Give and explain 2 ways of incorporating these changes and discuss pros and cons of each.
+\youranswer{
+Our work does not incorporate residual connections across the downsampling layers, as this creates a dimensional mismatch between the input and output feature maps due to the reduction in spatial dimensions. To add residual connections, one approach is to use a convolutional layer with a kernel size of $1\times 1$, stride, and padding matched to the downsampling operation to transform the input to the same shape as the output. Another approach would be to use average pooling or max pooling directly on the residual connection to downsample the input feature map, matching its spatial dimensions to the output, followed by a linear transformation to align the channel dimensions.
+
+The difference between these two methods is that the first approach using a $1\times 1$ convolution provides more flexibility by learning the transformation, which can enhance model expressiveness but increases computational cost, whereas the second approach with pooling is computationally cheaper and simpler but may lose fine-grained information due to the fixed, non-learnable nature of pooling operations.
 }
 }
 
 %% Question 4:
+% Question 4 - Present and discuss the experiment results (all of the results and not just the ones you had to fill in) in Table 1 and Figures 4 and 5 (you may use any of the other Figures if you think they are relevant to your analysis). You will have to determine what data are relevant to the discussion, and what information can be extracted from it. Also, discuss what further experiments you would have ran on any combination of VGG08, VGG38, BN, RC in order to
+% \begin{itemize}
+%     \item Improve performance of the model trained (explain why you expect your suggested experiments will help with this).
+%     \item Learn more about the behaviour of BN and RC (explain what you are trying to learn and how).
+% \end{itemize}
+% 
+% The average length for an answer to this question is approximately 1 of the columns in a 2-column page
 \newcommand{\questionFour} {
-\youranswer{Question 4 - Present and discuss the experiment results (all of the results and not just the ones you had to fill in) in Table 1 and Figures 4 and 5 (you may use any of the other Figures if you think they are relevant to your analysis). You will have to determine what data are relevant to the discussion, and what information can be extracted from it. Also, discuss what further experiments you would have ran on any combination of VGG08, VGG38, BN, RC in order to
+\youranswer{test1
-\begin{itemize}
-    \item Improve performance of the model trained (explain why you expect your suggested experiments will help with this).
-    \item Learn more about the behaviour of BN and RC (explain what you are trying to learn and how).
-\end{itemize}
-
-The average length for an answer to this question is approximately 1 of the columns in a 2-column page
 }
 }
 
@@ -80,13 +85,12 @@ The length of this question description is indicative of the average length of a
 
 %% Question Figure 3:
 \newcommand{\questionFigureThree} {
-\youranswer{Question Figure 3 - Replace this image with a figure depicting the average gradient across layers, for the VGG38 model.
+% Question Figure 3 - Replace this image with a figure depicting the average gradient across layers, for the VGG38 model.
-
+%\textit{(The provided figure is correct, and can be used in your analysis. It is partially obscured so you can get credit for producing your own copy).}
-\textit{(The provided figure is correct, and can be used in your analysis. It is partially obscured so you can get credit for producing your own copy).}
+\youranswer{
-%
 \begin{figure}[t]
     \centering
-    \includegraphics[width=\linewidth]{figures/gradplot_38_watermarked.pdf}
+    \includegraphics[width=\linewidth]{figures/gradplot_38.pdf}
     \caption{Gradient Flow on VGG38}
     \label{fig:avg_grad_flow_38}
 \end{figure}
@@ -94,26 +98,26 @@ The length of this question description is indicative of the average length of a
 }
 
 %% Question Figure 4:
+% Question Figure 4 - Replace this image with a figure depicting the training curves for the model with the best performance \textit{across experiments you have available (you don't need to run the experiments for the models we already give you results for)}. Edit the caption so that it clearly identifies the model and what is depicted.
 \newcommand{\questionFigureFour} {
-\youranswer{Question Figure 4 - Replace this image with a figure depicting the training curves for the model with the best performance \textit{across experiments you have available (you don't need to run the experiments for the models we already give you results for)}. Edit the caption so that it clearly identifies the model and what is depicted.
+\youranswer{
-%
 \begin{figure}[t]
     \centering
-    \includegraphics[width=\linewidth]{example-image-duck}
+    \includegraphics[width=\linewidth]{figures/VGG38_BN_RC_accuracy_performance.pdf}
-    \caption{Training curves for ? ? ?}
+    \caption{Training curves for 38 layer CNN with batch normalisation and residual connections, trained with LR of $0.01$}
     \label{fig:training_curves_bestModel}
 \end{figure}
 }
 }
 
 %% Question Figure 5:
+% Question Figure 5 - Replace this image with a figure depicting the average gradient across layers, for the model with the best performance \textit{across experiments you have available (you don't need to run the experiments for the models we already give you results for)}. Edit the caption so that it clearly identifies the model and what is depicted.
 \newcommand{\questionFigureFive} {
-\youranswer{Question Figure 5 - Replace this image with a figure depicting the average gradient across layers, for the model with the best performance \textit{across experiments you have available (you don't need to run the experiments for the models we already give you results for)}. Edit the caption so that it clearly identifies the model and what is depicted.
+\youranswer{
-%
 \begin{figure}[t]
     \centering
-    \includegraphics[width=\linewidth]{example-image-duck}
+    \includegraphics[width=\linewidth]{figures/gradplot_38_bn_rc.pdf}
-    \caption{Gradient Flow on ? ? ?}
+    \caption{Gradient Flow for 38 layer CNN with batch normalisation and residual connections, trained with LR of $0.01$}
     \label{fig:avg_grad_flow_bestModel}
 \end{figure}
 }
@@ -122,13 +126,13 @@ The length of this question description is indicative of the average length of a
 %% - - - - - - - - - - - - TABLES - - - - - - - - - - - - 
 
 %% Question Table 1:
+% Question Table 1 - Fill in Table 1 with the results from your experiments on 
+% \begin{enumerate}
+%     \item \textit{VGG38 BN (LR 1e-3)}, and 
+%     \item \textit{VGG38 BN + RC (LR 1e-2)}.
+% \end{enumerate}
 \newcommand{\questionTableOne} {
 \youranswer{
-Question Table 1 - Fill in Table 1 with the results from your experiments on 
-\begin{enumerate}
-    \item \textit{VGG38 BN (LR 1e-3)}, and 
-    \item \textit{VGG38 BN + RC (LR 1e-2)}.
-\end{enumerate}
 %
 \begin{table*}[t]
     \centering
@@ -138,11 +142,11 @@ Question Table 1 - Fill in Table 1 with the results from your experiments on
     \midrule
         VGG08                   & 1e-3 & 60 K      &  1.74      & 51.59     & 1.95     & 46.84 \\
         VGG38                   & 1e-3 & 336 K     &  4.61      & 00.01     & 4.61     & 00.01 \\
-        VGG38 BN                & 1e-3 &     ?     &     ?      &     ?     &    ?     &     ? \\
+        VGG38 BN                & 1e-3 & 339 K     &  1.76      & 50.62     & 1.95     & 47.68 \\
         VGG38 RC                & 1e-3 & 336 K     &  1.33      & 61.52     & 1.84     & 52.32 \\
         VGG38 BN + RC           & 1e-3 & 339 K     &  1.26      & 62.99     & 1.73     & 53.76 \\
         VGG38 BN                & 1e-2 & 339 K     &  1.70      & 52.28     & 1.99     & 46.72 \\
-        VGG38 BN + RC           & 1e-2 &     ?     &     ?      &     ?     &    ?     &     ? \\
+        VGG38 BN + RC           & 1e-2 & 339 K     &  0.83      & 74.35     & 1.70     & 58.20 \\
     \bottomrule
     \end{tabular}
     \caption{Experiment results (number of model parameters, Training and Validation loss and accuracy) for different combinations of VGG08, VGG38, Batch Normalisation (BN), and Residual Connections (RC), LR is learning rate.}

run_vgg_38_bn.sh

@@ -1 +1 @@
-python pytorch_mlp_framework/train_evaluate_image_classification_system.py --batch_size 100 --seed 0 --num_filters 32 --num_stages 3 --num_blocks_per_stage 5 --experiment_name VGG_38_experiment --use_gpu True --num_classes 100 --block_type 'conv_bn' --continue_from_epoch -1
+python pytorch_mlp_framework/train_evaluate_image_classification_system.py --batch_size 100 --seed 0 --num_filters 32 --num_stages 3 --num_blocks_per_stage 5 --experiment_name VGG38_BN --use_gpu True --num_classes 100 --block_type 'conv_bn' --continue_from_epoch -1

run_vgg_38_bn_rc.sh

@@ -1 +1 @@
-python pytorch_mlp_framework/train_evaluate_image_classification_system.py --batch_size 100 --seed 0 --num_filters 32 --num_stages 3 --num_blocks_per_stage 5 --experiment_name VGG_38_experiment --use_gpu True --num_classes 100 --block_type 'conv_bn_rc' --continue_from_epoch -1 --learning-rate 0.01
+python pytorch_mlp_framework/train_evaluate_image_classification_system.py --batch_size 100 --seed 0 --num_filters 32 --num_stages 3 --num_blocks_per_stage 5 --experiment_name VGG38_BN_RC --use_gpu True --num_classes 100 --block_type 'conv_bn_rc' --continue_from_epoch -1 --learning-rate 0.01