anton/mlpractical
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
Visual Computing (VICO) Group 2024-11-11 09:57:57 +00:00
parent 9b9a7d50fa
commit be1f124dff
27 changed files with 1472 additions and 6537 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -9,7 +9,7 @@ The code in this repository is split into:
* a Python package `mlp`, a [NumPy](http://www.numpy.org/) based neural network package designed specifically for the course that students will implement parts of and extend during the course labs and assignments, * a Python package `mlp`, a [NumPy](http://www.numpy.org/) based neural network package designed specifically for the course that students will implement parts of and extend during the course labs and assignments,
* a series of [Jupyter](http://jupyter.org/) notebooks in the `notebooks` directory containing explanatory material and coding exercises to be completed during the course labs. * a series of [Jupyter](http://jupyter.org/) notebooks in the `notebooks` directory containing explanatory material and coding exercises to be completed during the course labs.
## Coursework 1 ## Coursework 2
This branch contains the python code and latex files of the first coursework. The code follows the same structure as the labs, in particular the mlp package, and a specific notebook is provided to help you run experiments. This branch contains the python code and latex files of the first coursework. The code follows the same structure as the labs, in particular the mlp package, and a specific notebook is provided to help you run experiments.
* Detailed instructions are given in MLP2024_25_CW1_Spec.pdf (see Learn, Assessment, CW1). * Detailed instructions are given in MLP2024_25_CW2_Spec.pdf (see Learn, Assessment, CW2).
* The [report directory](https://github.com/VICO-UoE/mlpractical/tree/mlp2024-25/coursework1/report) contains the latex files that you will use to create your report. * The [report directory](https://github.com/VICO-UoE/mlpractical/tree/mlp2024-25/coursework2/report) contains the latex files that you will use to create your report.

2
mlp/__init__.py
View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
"""Machine Learning Practical package.""" """Machine Learning Practical package."""
__authors__ = ['Pawel Swietojanski', 'Steve Renals', 'Matt Graham'] __authors__ = ['Pawel Swietojanski', 'Steve Renals', 'Matt Graham', 'Antreas Antoniou']
DEFAULT_SEED = 123456 # Default random number generator seed if none provided. DEFAULT_SEED = 123456 # Default random number generator seed if none provided.

390
mlp/data_providers.py
View File

@ -7,8 +7,17 @@ data points.
import pickle import pickle
import gzip import gzip
import sys
import numpy as np import numpy as np
import os import os
from PIL import Image
from torch.utils import data
from torch.utils.data import Dataset
from torchvision import transforms
from torchvision.datasets.utils import download_url, check_integrity
from mlp import DEFAULT_SEED from mlp import DEFAULT_SEED
@ -16,7 +25,7 @@ class DataProvider(object):
"""Generic data provider.""" """Generic data provider."""
def __init__(self, inputs, targets, batch_size, max_num_batches=-1, def __init__(self, inputs, targets, batch_size, max_num_batches=-1,
shuffle_order=True, rng=None, smooth_labels=False): shuffle_order=True, rng=None):
"""Create a new data provider object. """Create a new data provider object.
Args: Args:
@ -32,7 +41,6 @@ class DataProvider(object):
shuffle_order (bool): Whether to randomly permute the order of shuffle_order (bool): Whether to randomly permute the order of
the data before each epoch. the data before each epoch.
rng (RandomState): A seeded random number generator. rng (RandomState): A seeded random number generator.
smooth_labels (bool): turn on label smoothing
""" """
self.inputs = inputs self.inputs = inputs
self.targets = targets self.targets = targets
@ -44,12 +52,10 @@ class DataProvider(object):
self._max_num_batches = max_num_batches self._max_num_batches = max_num_batches
self._update_num_batches() self._update_num_batches()
self.shuffle_order = shuffle_order self.shuffle_order = shuffle_order
self._current_order = np.arange(inputs.shape[0]) self._current_order = np.arange(inputs.shape[0])
if rng is None: if rng is None:
rng = np.random.RandomState(DEFAULT_SEED) rng = np.random.RandomState(DEFAULT_SEED)
self.rng = rng self.rng = rng
self.smooth_labels = smooth_labels
self.new_epoch() self.new_epoch()
@property @property
@ -140,7 +146,7 @@ class MNISTDataProvider(DataProvider):
"""Data provider for MNIST handwritten digit images.""" """Data provider for MNIST handwritten digit images."""
def __init__(self, which_set='train', batch_size=100, max_num_batches=-1, def __init__(self, which_set='train', batch_size=100, max_num_batches=-1,
shuffle_order=True, rng=None, smooth_labels=False): shuffle_order=True, rng=None):
"""Create a new MNIST data provider object. """Create a new MNIST data provider object.
Args: Args:
@ -154,7 +160,6 @@ class MNISTDataProvider(DataProvider):
shuffle_order (bool): Whether to randomly permute the order of shuffle_order (bool): Whether to randomly permute the order of
the data before each epoch. the data before each epoch.
rng (RandomState): A seeded random number generator. rng (RandomState): A seeded random number generator.
smooth_labels (bool): enable/disable label smoothing
""" """
# check a valid which_set was provided # check a valid which_set was provided
assert which_set in ['train', 'valid', 'test'], ( assert which_set in ['train', 'valid', 'test'], (
@ -177,7 +182,7 @@ class MNISTDataProvider(DataProvider):
inputs = inputs.astype(np.float32) inputs = inputs.astype(np.float32)
# pass the loaded data to the parent class __init__ # pass the loaded data to the parent class __init__
super(MNISTDataProvider, self).__init__( super(MNISTDataProvider, self).__init__(
inputs, targets, batch_size, max_num_batches, shuffle_order, rng, smooth_labels) inputs, targets, batch_size, max_num_batches, shuffle_order, rng)
def next(self): def next(self):
"""Returns next data batch or raises `StopIteration` if at end.""" """Returns next data batch or raises `StopIteration` if at end."""
@ -207,7 +212,7 @@ class EMNISTDataProvider(DataProvider):
"""Data provider for EMNIST handwritten digit images.""" """Data provider for EMNIST handwritten digit images."""
def __init__(self, which_set='train', batch_size=100, max_num_batches=-1, def __init__(self, which_set='train', batch_size=100, max_num_batches=-1,
shuffle_order=True, rng=None, smooth_labels=False): shuffle_order=True, rng=None, flatten=False):
"""Create a new EMNIST data provider object. """Create a new EMNIST data provider object.
Args: Args:
@ -221,7 +226,6 @@ class EMNISTDataProvider(DataProvider):
shuffle_order (bool): Whether to randomly permute the order of shuffle_order (bool): Whether to randomly permute the order of
the data before each epoch. the data before each epoch.
rng (RandomState): A seeded random number generator. rng (RandomState): A seeded random number generator.
smooth_labels (bool): enable/disable label smoothing
""" """
# check a valid which_set was provided # check a valid which_set was provided
assert which_set in ['train', 'valid', 'test'], ( assert which_set in ['train', 'valid', 'test'], (
@ -243,23 +247,20 @@ class EMNISTDataProvider(DataProvider):
print(loaded.keys()) print(loaded.keys())
inputs, targets = loaded['inputs'], loaded['targets'] inputs, targets = loaded['inputs'], loaded['targets']
inputs = inputs.astype(np.float32) inputs = inputs.astype(np.float32)
targets = targets.astype(np.int)
if flatten:
inputs = np.reshape(inputs, newshape=(-1, 28*28)) inputs = np.reshape(inputs, newshape=(-1, 28*28))
else:
inputs = np.reshape(inputs, newshape=(-1, 28, 28, 1))
inputs = inputs / 255.0 inputs = inputs / 255.0
# pass the loaded data to the parent class __init__ # pass the loaded data to the parent class __init__
super(EMNISTDataProvider, self).__init__( super(EMNISTDataProvider, self).__init__(
inputs, targets, batch_size, max_num_batches, shuffle_order, rng, smooth_labels) inputs, targets, batch_size, max_num_batches, shuffle_order, rng)
def next(self): def next(self):
"""Returns next data batch or raises `StopIteration` if at end.""" """Returns next data batch or raises `StopIteration` if at end."""
inputs_batch, targets_batch = super(EMNISTDataProvider, self).next() inputs_batch, targets_batch = super(EMNISTDataProvider, self).next()
return inputs_batch, self.to_one_of_k(targets_batch)
if self.smooth_labels:
targets_batch_mat = self.label_smoothing(targets_batch)
else:
targets_batch_mat = self.to_one_of_k(targets_batch)
return inputs_batch, targets_batch_mat
def to_one_of_k(self, int_targets): def to_one_of_k(self, int_targets):
"""Converts integer coded class target to 1 of K coded targets. """Converts integer coded class target to 1 of K coded targets.
@ -280,26 +281,6 @@ class EMNISTDataProvider(DataProvider):
one_of_k_targets[range(int_targets.shape[0]), int_targets] = 1 one_of_k_targets[range(int_targets.shape[0]), int_targets] = 1
return one_of_k_targets return one_of_k_targets
def label_smoothing(self, int_targets, alpha=0.1):
"""Converts integer coded class target to 1 of K coded targets with label smoothing.
Args:
int_targets (ndarray): Array of integer coded class targets (i.e.
where an integer from 0 to `num_classes` - 1 is used to
indicate which is the correct class). This should be of shape
(num_data,).
alpha (float): Smoothing factor.
Returns:
Array of 1 of K coded targets with label smoothing i.e. an array of shape
(num_data, num_classes)
"""
raise NotImplementedError
class MetOfficeDataProvider(DataProvider): class MetOfficeDataProvider(DataProvider):
"""South Scotland Met Office weather data provider.""" """South Scotland Met Office weather data provider."""
@ -393,6 +374,21 @@ class CCPPDataProvider(DataProvider):
super(CCPPDataProvider, self).__init__( super(CCPPDataProvider, self).__init__(
inputs, targets, batch_size, max_num_batches, shuffle_order, rng) inputs, targets, batch_size, max_num_batches, shuffle_order, rng)
class EMNISTPytorchDataProvider(Dataset):
def __init__(self, which_set='train', batch_size=100, max_num_batches=-1,
shuffle_order=True, rng=None, flatten=False, transforms=None):
self.numpy_data_provider = EMNISTDataProvider(which_set=which_set, batch_size=batch_size, max_num_batches=max_num_batches,
shuffle_order=shuffle_order, rng=rng, flatten=flatten)
self.transforms = transforms
def __getitem__(self, item):
x = self.numpy_data_provider.inputs[item]
for augmentation in self.transforms:
x = augmentation(x)
return x, int(self.numpy_data_provider.targets[item])
def __len__(self):
return len(self.numpy_data_provider.targets)
class AugmentedMNISTDataProvider(MNISTDataProvider): class AugmentedMNISTDataProvider(MNISTDataProvider):
"""Data provider for MNIST dataset which randomly transforms images.""" """Data provider for MNIST dataset which randomly transforms images."""
@ -430,3 +426,321 @@ class AugmentedMNISTDataProvider(MNISTDataProvider):
AugmentedMNISTDataProvider, self).next() AugmentedMNISTDataProvider, self).next()
transformed_inputs_batch = self.transformer(inputs_batch, self.rng) transformed_inputs_batch = self.transformer(inputs_batch, self.rng)
return transformed_inputs_batch, targets_batch return transformed_inputs_batch, targets_batch
class Omniglot(data.Dataset):
"""`CIFAR10 <https://www.cs.toronto.edu/~kriz/cifar.html>`_ Dataset.
Args:
root (string): Root directory of dataset where directory
``cifar-10-batches-py`` exists or will be saved to if download is set to True.
train (bool, optional): If True, creates dataset from training set, otherwise
creates from test set.
transform (callable, optional): A function/transform that takes in an PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
target_transform (callable, optional): A function/transform that takes in the
target and transforms it.
download (bool, optional): If true, downloads the dataset from the internet and
puts it in root directory. If dataset is already downloaded, it is not
downloaded again.
"""
def collect_data_paths(self, root):
data_dict = dict()
print(root)
for subdir, dir, files in os.walk(root):
for file in files:
if file.endswith('.png'):
filepath = os.path.join(subdir, file)
class_label = '_'.join(subdir.split("/")[-2:])
if class_label in data_dict:
data_dict[class_label].append(filepath)
else:
data_dict[class_label] = [filepath]
return data_dict
def __init__(self, root, set_name,
transform=None, target_transform=None,
download=False):
self.root = os.path.expanduser(root)
self.root = os.path.abspath(os.path.join(self.root, 'omniglot_dataset'))
self.transform = transform
self.target_transform = target_transform
self.set_name = set_name # training set or test set
self.data_dict = self.collect_data_paths(root=self.root)
x = []
label_to_idx = {label: idx for idx, label in enumerate(self.data_dict.keys())}
y = []
for key, value in self.data_dict.items():
x.extend(value)
y.extend(len(value) * [label_to_idx[key]])
y = np.array(y)
rng = np.random.RandomState(seed=0)
idx = np.arange(len(x))
rng.shuffle(idx)
x = [x[current_idx] for current_idx in idx]
y = y[idx]
train_sample_idx = rng.choice(a=[i for i in range(len(x))], size=int(len(x) * 0.80), replace=False)
evaluation_sample_idx = [i for i in range(len(x)) if i not in train_sample_idx]
validation_sample_idx = rng.choice(a=[i for i in range(len(evaluation_sample_idx))], size=int(len(evaluation_sample_idx) * 0.40), replace=False)
test_sample_idx = [i for i in range(len(evaluation_sample_idx)) if i not in evaluation_sample_idx]
if self.set_name=='train':
self.data = [item for idx, item in enumerate(x) if idx in train_sample_idx]
self.labels = y[train_sample_idx]
elif self.set_name=='val':
self.data = [item for idx, item in enumerate(x) if idx in validation_sample_idx]
self.labels = y[validation_sample_idx]
else:
self.data = [item for idx, item in enumerate(x) if idx in test_sample_idx]
self.labels = y[test_sample_idx]
def __getitem__(self, index):
"""
Args:
index (int): Index
Returns:
tuple: (image, target) where target is index of the target class.
"""
img, target = self.data[index], self.labels[index]
img = Image.open(img)
img.show()
if self.transform is not None:
img = self.transform(img)
if self.target_transform is not None:
target = self.target_transform(target)
return img, target
def __len__(self):
return len(self.data)
def __repr__(self):
fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
fmt_str += ' Number of datapoints: {}\n'.format(self.__len__())
tmp = self.set_name
fmt_str += ' Split: {}\n'.format(tmp)
fmt_str += ' Root Location: {}\n'.format(self.root)
tmp = ' Transforms (if any): '
fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
tmp = ' Target Transforms (if any): '
fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
return fmt_str
class CIFAR10(data.Dataset):
"""`CIFAR10 <https://www.cs.toronto.edu/~kriz/cifar.html>`_ Dataset.
Args:
root (string): Root directory of dataset where directory
``cifar-10-batches-py`` exists or will be saved to if download is set to True.
train (bool, optional): If True, creates dataset from training set, otherwise
creates from test set.
transform (callable, optional): A function/transform that takes in an PIL image
and returns a transformed version. E.g, ``transforms.RandomCrop``
target_transform (callable, optional): A function/transform that takes in the
target and transforms it.
download (bool, optional): If true, downloads the dataset from the internet and
puts it in root directory. If dataset is already downloaded, it is not
downloaded again.
"""
base_folder = 'cifar-10-batches-py'
url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
filename = "cifar-10-python.tar.gz"
tgz_md5 = 'c58f30108f718f92721af3b95e74349a'
train_list = [
['data_batch_1', 'c99cafc152244af753f735de768cd75f'],
['data_batch_2', 'd4bba439e000b95fd0a9bffe97cbabec'],
['data_batch_3', '54ebc095f3ab1f0389bbae665268c751'],
['data_batch_4', '634d18415352ddfa80567beed471001a'],
['data_batch_5', '482c414d41f54cd18b22e5b47cb7c3cb'],
]
test_list = [
['test_batch', '40351d587109b95175f43aff81a1287e'],
]
def __init__(self, root, set_name,
transform=None, target_transform=None,
download=False):
self.root = os.path.expanduser(root)
self.transform = transform
self.target_transform = target_transform
self.set_name = set_name # training set or test set
if download:
self.download()
if not self._check_integrity():
raise RuntimeError('Dataset not found or corrupted.' +
' You can use download=True to download it')
# now load the picked numpy arrays
rng = np.random.RandomState(seed=0)
train_sample_idx = rng.choice(a=[i for i in range(50000)], size=47500, replace=False)
val_sample_idx = [i for i in range(50000) if i not in train_sample_idx]
if self.set_name=='train':
self.data = []
self.labels = []
for fentry in self.train_list:
f = fentry[0]
file = os.path.join(self.root, self.base_folder, f)
fo = open(file, 'rb')
if sys.version_info[0] == 2:
entry = pickle.load(fo)
else:
entry = pickle.load(fo, encoding='latin1')
self.data.append(entry['data'])
if 'labels' in entry:
self.labels += entry['labels']
else:
self.labels += entry['fine_labels']
fo.close()
self.data = np.concatenate(self.data)
self.data = self.data.reshape((50000, 3, 32, 32))
self.data = self.data.transpose((0, 2, 3, 1)) # convert to HWC
self.data = self.data[train_sample_idx]
self.labels = np.array(self.labels)[train_sample_idx]
print(set_name, self.data.shape)
print(set_name, self.labels.shape)
elif self.set_name=='val':
self.data = []
self.labels = []
for fentry in self.train_list:
f = fentry[0]
file = os.path.join(self.root, self.base_folder, f)
fo = open(file, 'rb')
if sys.version_info[0] == 2:
entry = pickle.load(fo)
else:
entry = pickle.load(fo, encoding='latin1')
self.data.append(entry['data'])
if 'labels' in entry:
self.labels += entry['labels']
else:
self.labels += entry['fine_labels']
fo.close()
self.data = np.concatenate(self.data)
self.data = self.data.reshape((50000, 3, 32, 32))
self.data = self.data.transpose((0, 2, 3, 1)) # convert to HWC
self.data = self.data[val_sample_idx]
self.labels = np.array(self.labels)[val_sample_idx]
print(set_name, self.data.shape)
print(set_name, self.labels.shape)
else:
f = self.test_list[0][0]
file = os.path.join(self.root, self.base_folder, f)
fo = open(file, 'rb')
if sys.version_info[0] == 2:
entry = pickle.load(fo)
else:
entry = pickle.load(fo, encoding='latin1')
self.data = entry['data']
if 'labels' in entry:
self.labels = entry['labels']
else:
self.labels = entry['fine_labels']
fo.close()
self.data = self.data.reshape((10000, 3, 32, 32))
self.data = self.data.transpose((0, 2, 3, 1)) # convert to HWC
self.labels = np.array(self.labels)
print(set_name, self.data.shape)
print(set_name, self.labels.shape)
def __getitem__(self, index):
"""
Args:
index (int): Index
Returns:
tuple: (image, target) where target is index of the target class.
"""
img, target = self.data[index], self.labels[index]
# doing this so that it is consistent with all other datasets
# to return a PIL Image
img = Image.fromarray(img)
if self.transform is not None:
img = self.transform(img)
if self.target_transform is not None:
target = self.target_transform(target)
return img, target
def __len__(self):
return len(self.data)
def _check_integrity(self):
root = self.root
for fentry in (self.train_list + self.test_list):
filename, md5 = fentry[0], fentry[1]
fpath = os.path.join(root, self.base_folder, filename)
if not check_integrity(fpath, md5):
return False
return True
def download(self):
import tarfile
if self._check_integrity():
print('Files already downloaded and verified')
return
root = self.root
download_url(self.url, root, self.filename, self.tgz_md5)
# extract file
cwd = os.getcwd()
tar = tarfile.open(os.path.join(root, self.filename), "r:gz")
os.chdir(root)
tar.extractall()
tar.close()
os.chdir(cwd)
def __repr__(self):
fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
fmt_str += ' Number of datapoints: {}\n'.format(self.__len__())
tmp = self.set_name
fmt_str += ' Split: {}\n'.format(tmp)
fmt_str += ' Root Location: {}\n'.format(self.root)
tmp = ' Transforms (if any): '
fmt_str += '{0}{1}\n'.format(tmp, self.transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
tmp = ' Target Transforms (if any): '
fmt_str += '{0}{1}'.format(tmp, self.target_transform.__repr__().replace('\n', '\n' + ' ' * len(tmp)))
return fmt_str
class CIFAR100(CIFAR10):
"""`CIFAR100 <https://www.cs.toronto.edu/~kriz/cifar.html>`_ Dataset.
This is a subclass of the `CIFAR10` Dataset.
"""
base_folder = 'cifar-100-python'
url = "https://www.cs.toronto.edu/~kriz/cifar-100-python.tar.gz"
filename = "cifar-100-python.tar.gz"
tgz_md5 = 'eb9058c3a382ffc7106e4002c42a8d85'
train_list = [
['train', '16019d7e3df5f24257cddd939b257f8d'],
]
test_list = [
['test', 'f0ef6b0ae62326f3e7ffdfab6717acfc'],
]

242
mlp/layers.py
View File

@ -16,6 +16,7 @@ import numpy as np
import mlp.initialisers as init import mlp.initialisers as init
from mlp import DEFAULT_SEED from mlp import DEFAULT_SEED
class Layer(object): class Layer(object):
"""Abstract class defining the interface for a layer.""" """Abstract class defining the interface for a layer."""
@ -95,6 +96,7 @@ class LayerWithParameters(Layer):
""" """
raise NotImplementedError() raise NotImplementedError()
class StochasticLayerWithParameters(Layer): class StochasticLayerWithParameters(Layer):
"""Specialised layer which uses a stochastic forward propagation.""" """Specialised layer which uses a stochastic forward propagation."""
@ -124,6 +126,7 @@ class StochasticLayerWithParameters(Layer):
outputs: Array of layer outputs of shape (batch_size, output_dim). outputs: Array of layer outputs of shape (batch_size, output_dim).
""" """
raise NotImplementedError() raise NotImplementedError()
def grads_wrt_params(self, inputs, grads_wrt_outputs): def grads_wrt_params(self, inputs, grads_wrt_outputs):
"""Calculates gradients with respect to layer parameters. """Calculates gradients with respect to layer parameters.
@ -166,6 +169,7 @@ class StochasticLayerWithParameters(Layer):
""" """
raise NotImplementedError() raise NotImplementedError()
class StochasticLayer(Layer): class StochasticLayer(Layer):
"""Specialised layer which uses a stochastic forward propagation.""" """Specialised layer which uses a stochastic forward propagation."""
@ -329,6 +333,7 @@ class AffineLayer(LayerWithParameters):
return 'AffineLayer(input_dim={0}, output_dim={1})'.format( return 'AffineLayer(input_dim={0}, output_dim={1})'.format(
self.input_dim, self.output_dim) self.input_dim, self.output_dim)
class SigmoidLayer(Layer): class SigmoidLayer(Layer):
"""Layer implementing an element-wise logistic sigmoid transformation.""" """Layer implementing an element-wise logistic sigmoid transformation."""
@ -368,6 +373,152 @@ class SigmoidLayer(Layer):
def __repr__(self): def __repr__(self):
return 'SigmoidLayer' return 'SigmoidLayer'
class ConvolutionalLayer(LayerWithParameters):
"""Layer implementing a 2D convolution-based transformation of its inputs.
The layer is parameterised by a set of 2D convolutional kernels, a four
dimensional array of shape
(num_output_channels, num_input_channels, kernel_height, kernel_dim_2)
and a bias vector, a one dimensional array of shape
(num_output_channels,)
i.e. one shared bias per output channel.
Assuming no-padding is applied to the inputs so that outputs are only
calculated for positions where the kernel filters fully overlap with the
inputs, and that unit strides are used the outputs will have spatial extent
output_height = input_height - kernel_height + 1
output_width = input_width - kernel_width + 1
"""
def __init__(self, num_input_channels, num_output_channels,
input_height, input_width,
kernel_height, kernel_width,
kernels_init=init.UniformInit(-0.01, 0.01),
biases_init=init.ConstantInit(0.),
kernels_penalty=None, biases_penalty=None):
"""Initialises a parameterised convolutional layer.
Args:
num_input_channels (int): Number of channels in inputs to
layer (this may be number of colour channels in the input
images if used as the first layer in a model, or the
number of output channels, a.k.a. feature maps, from a
a previous convolutional layer).
num_output_channels (int): Number of channels in outputs
from the layer, a.k.a. number of feature maps.
input_height (int): Size of first input dimension of each 2D
channel of inputs.
input_width (int): Size of second input dimension of each 2D
channel of inputs.
kernel_height (int): Size of first dimension of each 2D channel of
kernels.
kernel_width (int): Size of second dimension of each 2D channel of
kernels.
kernels_intialiser: Initialiser for the kernel parameters.
biases_initialiser: Initialiser for the bias parameters.
kernels_penalty: Kernel-dependent penalty term (regulariser) or
None if no regularisation is to be applied to the kernels.
biases_penalty: Biases-dependent penalty term (regulariser) or
None if no regularisation is to be applied to the biases.
"""
self.num_input_channels = num_input_channels
self.num_output_channels = num_output_channels
self.input_height = input_height
self.input_width = input_width
self.kernel_height = kernel_height
self.kernel_width = kernel_width
self.kernels_init = kernels_init
self.biases_init = biases_init
self.kernels_shape = (
num_output_channels, num_input_channels, kernel_height, kernel_width
)
self.inputs_shape = (
None, num_input_channels, input_height, input_width
)
self.kernels = self.kernels_init(self.kernels_shape)
self.biases = self.biases_init(num_output_channels)
self.kernels_penalty = kernels_penalty
self.biases_penalty = biases_penalty
self.cache = None
def fprop(self, inputs):
"""Forward propagates activations through the layer transformation.
For inputs `x`, outputs `y`, kernels `K` and biases `b` the layer
corresponds to `y = conv2d(x, K) + b`.
Args:
inputs: Array of layer inputs of shape (batch_size, num_input_channels, image_height, image_width).
Returns:
outputs: Array of layer outputs of shape (batch_size, num_output_channels, output_height, output_width).
"""
raise NotImplementedError
def bprop(self, inputs, outputs, grads_wrt_outputs):
"""Back propagates gradients through a layer.
Given gradients with respect to the outputs of the layer calculates the
gradients with respect to the layer inputs.
Args:
inputs: Array of layer inputs of shape
(batch_size, num_input_channels, input_height, input_width).
outputs: Array of layer outputs calculated in forward pass of
shape
(batch_size, num_output_channels, output_height, output_width).
grads_wrt_outputs: Array of gradients with respect to the layer
outputs of shape
(batch_size, num_output_channels, output_height, output_width).
Returns:
Array of gradients with respect to the layer inputs of shape
(batch_size, num_input_channels, input_height, input_width).
"""
# Pad the grads_wrt_outputs
raise NotImplementedError
def grads_wrt_params(self, inputs, grads_wrt_outputs):
"""Calculates gradients with respect to layer parameters.
Args:
inputs: array of inputs to layer of shape (batch_size, input_dim)
grads_wrt_to_outputs: array of gradients with respect to the layer
outputs of shape
(batch_size, num_output_channels, output_height, output_width).
Returns:
list of arrays of gradients with respect to the layer parameters
`[grads_wrt_kernels, grads_wrt_biases]`.
"""
# Get inputs_col from previous fprop
raise NotImplementedError
def params_penalty(self):
"""Returns the parameter dependent penalty term for this layer.
If no parameter-dependent penalty terms are set this returns zero.
"""
params_penalty = 0
if self.kernels_penalty is not None:
params_penalty += self.kernels_penalty(self.kernels)
if self.biases_penalty is not None:
params_penalty += self.biases_penalty(self.biases)
return params_penalty
@property
def params(self):
"""A list of layer parameter values: `[kernels, biases]`."""
return [self.kernels, self.biases]
@params.setter
def params(self, values):
self.kernels = values[0]
self.biases = values[1]
def __repr__(self):
return (
'ConvolutionalLayer(\n'
' num_input_channels={0}, num_output_channels={1},\n'
' input_height={2}, input_width={3},\n'
' kernel_height={4}, kernel_width={5}\n'
')'
.format(self.num_input_channels, self.num_output_channels,
self.input_height, self.input_width, self.kernel_height,
self.kernel_width)
)
class ReluLayer(Layer): class ReluLayer(Layer):
"""Layer implementing an element-wise rectified linear transformation.""" """Layer implementing an element-wise rectified linear transformation."""
@ -406,84 +557,6 @@ class ReluLayer(Layer):
def __repr__(self): def __repr__(self):
return 'ReluLayer' return 'ReluLayer'
class LeakyReluLayer(Layer):
"""Layer implementing an element-wise leaky rectified linear transformation."""
def __init__(self, alpha=0.01):
self.alpha = alpha
def fprop(self, inputs):
"""Forward propagates activations through the layer transformation.
For inputs `x` and outputs `y` this corresponds to `y = ..., else`.
"""
raise NotImplementedError
def bprop(self, inputs, outputs, grads_wrt_outputs):
"""Back propagates gradients through a layer.
Given gradients with respect to the outputs of the layer calculates the
gradients with respect to the layer inputs.
"""
raise NotImplementedError
def __repr__(self):
return 'LeakyReluLayer'
class ParametricReluLayer(LayerWithParameters):
"""Layer implementing an element-wise parametric rectified linear transformation."""
def __init__(self, alpha=0.25):
self.alpha = np.array([alpha])
@property
def params(self):
"""A list of layer parameter values: `[weights, biases]`."""
return [self.alpha]
def fprop(self, inputs):
"""Forward propagates activations through the layer transformation.
For inputs `x` and outputs `y` this corresponds to `y = ..., else`.
"""
raise NotImplementedError
def bprop(self, inputs, outputs, grads_wrt_outputs):
"""Back propagates gradients through a layer.
Given gradients with respect to the outputs of the layer calculates the
gradients with respect to the layer inputs.
"""
raise NotImplementedError
def grads_wrt_params(self, inputs, grads_wrt_outputs):
"""Calculates gradients with respect to layer parameters.
Args:
inputs: array of inputs to layer of shape (batch_size, input_dim)
grads_wrt_to_outputs: array of gradients with respect to the layer
outputs of shape (batch_size, output_dim)
Returns:
list of arrays of gradients with respect to the layer parameters
`[grads_wrt_params]`. Where params is the alpha parameter.
"""
raise NotImplementedError
@property
def params(self):
"""A list of layer parameter values: `[weights, biases]`."""
return [self.alpha]
@params.setter
def params(self, values):
self.alpha = values[0]
def __repr__(self):
return 'ParametricReluLayer'
class TanhLayer(Layer): class TanhLayer(Layer):
"""Layer implementing an element-wise hyperbolic tangent transformation.""" """Layer implementing an element-wise hyperbolic tangent transformation."""
@ -523,6 +596,7 @@ class TanhLayer(Layer):
def __repr__(self): def __repr__(self):
return 'TanhLayer' return 'TanhLayer'
class SoftmaxLayer(Layer): class SoftmaxLayer(Layer):
"""Layer implementing a softmax transformation.""" """Layer implementing a softmax transformation."""
@ -567,6 +641,7 @@ class SoftmaxLayer(Layer):
def __repr__(self): def __repr__(self):
return 'SoftmaxLayer' return 'SoftmaxLayer'
class RadialBasisFunctionLayer(Layer): class RadialBasisFunctionLayer(Layer):
"""Layer implementing projection to a grid of radial basis functions.""" """Layer implementing projection to a grid of radial basis functions."""
@ -627,6 +702,7 @@ class RadialBasisFunctionLayer(Layer):
def __repr__(self): def __repr__(self):
return 'RadialBasisFunctionLayer(grid_dim={0})'.format(self.grid_dim) return 'RadialBasisFunctionLayer(grid_dim={0})'.format(self.grid_dim)
class DropoutLayer(StochasticLayer): class DropoutLayer(StochasticLayer):
"""Layer which stochastically drops input dimensions in its output.""" """Layer which stochastically drops input dimensions in its output."""
@ -661,7 +737,12 @@ class DropoutLayer(StochasticLayer):
Returns: Returns:
outputs: Array of layer outputs of shape (batch_size, output_dim). outputs: Array of layer outputs of shape (batch_size, output_dim).
""" """
raise NotImplementedError if stochastic:
mask_shape = (1,) + inputs.shape[1:] if self.share_across_batch else inputs.shape
self._mask = (self.rng.uniform(size=mask_shape) < self.incl_prob)
return inputs * self._mask
else:
return inputs * self.incl_prob
def bprop(self, inputs, outputs, grads_wrt_outputs): def bprop(self, inputs, outputs, grads_wrt_outputs):
"""Back propagates gradients through a layer. """Back propagates gradients through a layer.
@ -681,11 +762,12 @@ class DropoutLayer(StochasticLayer):
Array of gradients with respect to the layer inputs of shape Array of gradients with respect to the layer inputs of shape
(batch_size, input_dim). (batch_size, input_dim).
""" """
raise NotImplementedError return grads_wrt_outputs * self._mask
def __repr__(self): def __repr__(self):
return 'DropoutLayer(incl_prob={0:.1f})'.format(self.incl_prob) return 'DropoutLayer(incl_prob={0:.1f})'.format(self.incl_prob)
class ReshapeLayer(Layer): class ReshapeLayer(Layer):
"""Layer which reshapes dimensions of inputs.""" """Layer which reshapes dimensions of inputs."""

71
mlp/learning_rules.py
View File

@ -315,3 +315,74 @@ class AdaGradLearningRule(GradientDescentLearningRule):
param -= (self.learning_rate * grad / param -= (self.learning_rate * grad /
(sum_sq_grad + self.epsilon) ** 0.5) (sum_sq_grad + self.epsilon) ** 0.5)
class RMSPropLearningRule(GradientDescentLearningRule):
"""Root mean squared gradient normalised learning rule (RMSProp).
First-order gradient-descent based learning rule which normalises gradient
updates by a exponentially smoothed estimate of the gradient second
moments.
References:
[1]: Neural Networks for Machine Learning: Lecture 6a slides
University of Toronto,Computer Science Course CSC321
http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf
"""
def __init__(self, learning_rate=1e-3, beta=0.9, epsilon=1e-8):
"""Creates a new learning rule object.
Args:
learning_rate: A postive scalar to scale gradient updates to the
parameters by. This needs to be carefully set - if too large
the learning dynamic will be unstable and may diverge, while
if set too small learning will proceed very slowly.
beta: Exponential decay rate for gradient second moment
estimates. This should be a scalar value in [0, 1]. The running
gradient second moment estimate is calculated using
`m_2 = beta * m_2_prev + (1 - beta) * g**2`
where `m_2_prev` is the previous estimate and `g` the current
parameter gradients.
epsilon: 'Softening' parameter to stop updates diverging when
gradient second moment estimates are close to zero. Should be
set to a small positive value.
"""
super(RMSPropLearningRule, self).__init__(learning_rate)
assert beta >= 0. and beta <= 1., 'beta should be in [0, 1].'
assert epsilon > 0., 'epsilon should be > 0.'
self.beta = beta
self.epsilon = epsilon
def initialise(self, params):
"""Initialises the state of the learning rule for a set or parameters.
This must be called before `update_params` is first called.
Args:
params: A list of the parameters to be optimised. Note these will
be updated *in-place* to avoid reallocating arrays on each
update.
"""
super(RMSPropLearningRule, self).initialise(params)
self.moms_2 = []
for param in self.params:
self.moms_2.append(np.zeros_like(param))
def reset(self):
"""Resets any additional state variables to their initial values.
For this learning rule this corresponds to zeroing all gradient
second moment estimates.
"""
for mom_2 in self.moms_2:
mom_2 *= 0.
def update_params(self, grads_wrt_params):
"""Applies a single update to all parameters.
All parameter updates are performed using in-place operations and so
nothing is returned.
Args:
grads_wrt_params: A list of gradients of the scalar loss function
with respect to each of the parameters passed to `initialise`
previously, with this list expected to be in the same order.
"""
for param, mom_2, grad in zip(
self.params, self.moms_2, grads_wrt_params):
mom_2 *= self.beta
mom_2 += (1. - self.beta) * grad ** 2
param -= (self.learning_rate * grad /
(mom_2 + self.epsilon) ** 0.5)

2
mlp/models.py
View File

@ -27,7 +27,7 @@ class SingleLayerModel(object):
"""A list of all of the parameters of the model.""" """A list of all of the parameters of the model."""
return self.layer.params return self.layer.params
def fprop(self, inputs): def fprop(self, inputs, evaluation=False):
"""Calculate the model outputs corresponding to a batch of inputs. """Calculate the model outputs corresponding to a batch of inputs.
Args: Args:

6
mlp/optimisers.py
View File

@ -58,7 +58,7 @@ class Optimiser(object):
parameters according to the learning rule. parameters according to the learning rule.
""" """
with self.tqdm_progress(total=self.train_dataset.num_batches) as train_progress_bar: with self.tqdm_progress(total=self.train_dataset.num_batches) as train_progress_bar:
train_progress_bar.set_description("Ep Prog") train_progress_bar.set_description("Epoch Progress")
for inputs_batch, targets_batch in self.train_dataset: for inputs_batch, targets_batch in self.train_dataset:
activations = self.model.fprop(inputs_batch) activations = self.model.fprop(inputs_batch)
grads_wrt_outputs = self.error.grad(activations[-1], targets_batch) grads_wrt_outputs = self.error.grad(activations[-1], targets_batch)
@ -112,7 +112,7 @@ class Optimiser(object):
""" """
logger.info('Epoch {0}: {1:.1f}s to complete\n {2}'.format( logger.info('Epoch {0}: {1:.1f}s to complete\n {2}'.format(
epoch, epoch_time, epoch, epoch_time,
', '.join(['{}={:.2e}'.format(k, v) for (k, v) in stats.items()]) ', '.join(['{0}={1:.2e}'.format(k, v) for (k, v) in stats.items()])
)) ))
def train(self, num_epochs, stats_interval=5): def train(self, num_epochs, stats_interval=5):
@ -132,7 +132,7 @@ class Optimiser(object):
start_train_time = time.time() start_train_time = time.time()
run_stats = [list(self.get_epoch_stats().values())] run_stats = [list(self.get_epoch_stats().values())]
with self.tqdm_progress(total=num_epochs) as progress_bar: with self.tqdm_progress(total=num_epochs) as progress_bar:
progress_bar.set_description("Exp Prog") progress_bar.set_description("Experiment Progress")
for epoch in range(1, num_epochs + 1): for epoch in range(1, num_epochs + 1):
start_time = time.time() start_time = time.time()
self.do_training_epoch() self.do_training_epoch()

8
mlp/penalties.py
View File

@ -29,7 +29,7 @@ class L1Penalty(object):
Returns: Returns:
Value of penalty term. Value of penalty term.
""" """
raise NotImplementedError return self.coefficient * abs(parameter).sum()
def grad(self, parameter): def grad(self, parameter):
"""Calculate the penalty gradient with respect to the parameter. """Calculate the penalty gradient with respect to the parameter.
@ -41,7 +41,7 @@ class L1Penalty(object):
Value of penalty gradient with respect to parameter. This Value of penalty gradient with respect to parameter. This
should be an array of the same shape as the parameter. should be an array of the same shape as the parameter.
""" """
raise NotImplementedError return self.coefficient * np.sign(parameter)
def __repr__(self): def __repr__(self):
return 'L1Penalty({0})'.format(self.coefficient) return 'L1Penalty({0})'.format(self.coefficient)
@ -72,7 +72,7 @@ class L2Penalty(object):
Returns: Returns:
Value of penalty term. Value of penalty term.
""" """
raise NotImplementedError return 0.5 * self.coefficient * (parameter ** 2).sum()
def grad(self, parameter): def grad(self, parameter):
"""Calculate the penalty gradient with respect to the parameter. """Calculate the penalty gradient with respect to the parameter.
@ -84,7 +84,7 @@ class L2Penalty(object):
Value of penalty gradient with respect to parameter. This Value of penalty gradient with respect to parameter. This
should be an array of the same shape as the parameter. should be an array of the same shape as the parameter.
""" """
raise NotImplementedError return self.coefficient * parameter
def __repr__(self): def __repr__(self):
return 'L2Penalty({0})'.format(self.coefficient) return 'L2Penalty({0})'.format(self.coefficient)

39
mlp/schedulers.py
View File

@ -32,42 +32,3 @@ class ConstantLearningRateScheduler(object):
epoch_number: Integer index of training epoch about to be run. epoch_number: Integer index of training epoch about to be run.
""" """
learning_rule.learning_rate = self.learning_rate learning_rule.learning_rate = self.learning_rate
class CosineAnnealingWithWarmRestarts(object):
"""Cosine annealing scheduler, implemented as in https://arxiv.org/pdf/1608.03983.pdf"""
def __init__(self, min_learning_rate, max_learning_rate, total_iters_per_period, max_learning_rate_discount_factor,
period_iteration_expansion_factor):
"""
Instantiates a new cosine annealing with warm restarts learning rate scheduler
:param min_learning_rate: The minimum learning rate the scheduler can assign
:param max_learning_rate: The maximum learning rate the scheduler can assign
:param total_epochs_per_period: The number of epochs in a period
:param max_learning_rate_discount_factor: The rate of discount for the maximum learning rate after each restart i.e. how many times smaller the max learning rate will be after a restart compared to the previous one
:param period_iteration_expansion_factor: The rate of expansion of the period epochs. e.g. if it's set to 1 then all periods have the same number of epochs, if it's larger than 1 then each subsequent period will have more epochs and vice versa.
"""
self.min_learning_rate = min_learning_rate
self.max_learning_rate = max_learning_rate
self.total_epochs_per_period = total_iters_per_period
self.max_learning_rate_discount_factor = max_learning_rate_discount_factor
self.period_iteration_expansion_factor = period_iteration_expansion_factor
def update_learning_rule(self, learning_rule, epoch_number):
"""Update the hyperparameters of the learning rule.
Run at the beginning of each epoch.
Args:
learning_rule: Learning rule object being used in training run,
any scheduled hyperparameters to be altered should be
attributes of this object.
epoch_number: Integer index of training epoch about to be run.
Returns:
effective_learning_rate at step 'epoch_number'
"""
raise NotImplementedError

80
mlp/test_methods.py
View File

@ -1,80 +0,0 @@
# from mlp.learning_rules import AdamLearningRuleWithWeightDecay
# from mlp.schedulers import CosineAnnealingWithWarmRestarts
from mlp.layers import DropoutLayer
from mlp.penalties import L1Penalty, L2Penalty
import numpy as np
import os
def test_dropout_layer():
# loaded = np.load("../data/correct_results.npz")
rng = np.random.RandomState(92019)
x = rng.normal(loc=0, scale=5.0, size=(50, 3, 64, 64))
correct_outputs = np.load(os.path.join(os.environ['MLP_DATA_DIR'], 'regularization_debug_pack.npy'), allow_pickle=True).item()
rng = np.random.RandomState(92019)
layer = DropoutLayer(rng=rng)
out = layer.fprop(x)
grads = layer.bprop(inputs=x, outputs=out, grads_wrt_outputs=np.ones(x.shape))
# correct_outputs = correct_outputs['dropout']
fprop_test = np.allclose(correct_outputs['DropoutLayer_fprop'], out)
bprop_test = np.allclose(correct_outputs['DropoutLayer_bprop'], grads)
return fprop_test, out, correct_outputs['DropoutLayer_fprop'], bprop_test, grads, correct_outputs['DropoutLayer_bprop']
def test_L1_Penalty():
rng = np.random.RandomState(92019)
x = rng.normal(loc=0, scale=5.0, size=(50, 3, 64, 64))
correct_outputs = np.load(os.path.join(os.environ['MLP_DATA_DIR'], 'regularization_debug_pack.npy'), allow_pickle=True).item()
layer = L1Penalty(1e-4)
out = layer(x)
grads = layer.grad(x)
# correct_outputs = correct_outputs['l1penalty']
__call__test = np.allclose(correct_outputs['L1Penalty___call__correct'], out)
grad_test = np.allclose(correct_outputs['L1Penalty_grad_correct'], grads)
return __call__test, out, correct_outputs['L1Penalty___call__correct'], grad_test, grads, correct_outputs['L1Penalty_grad_correct']
def test_L2_Penalty():
rng = np.random.RandomState(92019)
x = rng.normal(loc=0, scale=5.0, size=(50, 3, 64, 64))
correct_outputs = np.load(os.path.join(os.environ['MLP_DATA_DIR'], 'regularization_debug_pack.npy'), allow_pickle=True).item()
layer = L2Penalty(1e-4)
out = layer(x)
grads = layer.grad(x)
# correct_outputs = correct_outputs['l2penalty']
__call__test = np.allclose(correct_outputs['L2Penalty___call__correct'], out)
grad_test = np.allclose(correct_outputs['L2Penalty_grad_correct'], grads)
return __call__test, out, correct_outputs['L2Penalty___call__correct'], grad_test, grads, correct_outputs['L2Penalty_grad_correct']

242
notebooks/00_notebook.ipynb
View File

@ -1,242 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Introduction\n",
"\n",
"## Getting started with Jupyter notebooks\n",
"\n",
"The majority of your work in this course will be done using Jupyter notebooks so we will here introduce some of the basics of the notebook system. If you are already comfortable using notebooks or just would rather get on with some coding feel free to [skip straight to the exercises below](#Exercises).\n",
"\n",
"*Note: Jupyter notebooks are also known as IPython notebooks. The Jupyter system now supports languages other than Python [hence the name was changed to make it more language agnostic](https://ipython.org/#jupyter-and-the-future-of-ipython) however IPython notebook is still commonly used.*\n",
"\n",
"### Jupyter basics: the server, dashboard and kernels\n",
"\n",
"In launching this notebook you will have already come across two of the other key components of the Jupyter system - the notebook *server* and *dashboard* interface.\n",
"\n",
"We began by starting a notebook server instance in the terminal by running\n",
"\n",
"```\n",
"jupyter notebook\n",
"```\n",
"\n",
"This will have begun printing a series of log messages to terminal output similar to\n",
"\n",
"```\n",
"$ jupyter notebook\n",
"[I 08:58:24.417 NotebookApp] Serving notebooks from local directory: ~/mlpractical\n",
"[I 08:58:24.417 NotebookApp] 0 active kernels\n",
"[I 08:58:24.417 NotebookApp] The Jupyter Notebook is running at: http://localhost:8888/\n",
"```\n",
"\n",
"The last message included here indicates the URL the application is being served at. The default behaviour of the `jupyter notebook` command is to open a tab in a web browser pointing to this address after the server has started up. The server can be launched without opening a browser window by running `jupyter notebook --no-browser`. This can be useful for example when running a notebook server on a remote machine over SSH. Descriptions of various other command options can be found by displaying the command help page using\n",
"\n",
"```\n",
"jupyter notebook --help\n",
"```\n",
"\n",
"While the notebook server is running it will continue printing log messages to terminal it was started from. Unless you detach the process from the terminal session you will need to keep the session open to keep the notebook server alive. If you want to close down a running server instance from the terminal you can use `Ctrl+C` - this will bring up a confirmation message asking you to confirm you wish to shut the server down. You can either enter `y` or skip the confirmation by hitting `Ctrl+C` again.\n",
"\n",
"When the notebook application first opens in your browser you are taken to the notebook *dashboard*. This will appear something like this\n",
"\n",
"<img src='res/jupyter-dashboard.png' />\n",
"\n",
"The dashboard above is showing the `Files` tab, a list of files in the directory the notebook server was launched from. We can navigate in to a sub-directory by clicking on a directory name and back up to the parent directory by clicking the `..` link. An important point to note is that the top-most level that you will be able to navigate to is the directory you run the server from. This is a security feature and generally you should try to limit the access the server has by launching it in the highest level directory which gives you access to all the files you need to work with.\n",
"\n",
"As well as allowing you to launch existing notebooks, the `Files` tab of the dashboard also allows new notebooks to be created using the `New` drop-down on the right. It can also perform basic file-management tasks such as renaming and deleting files (select a file by checking the box alongside it to bring up a context menu toolbar).\n",
"\n",
"In addition to opening notebook files, we can also edit text files such as `.py` source files, directly in the browser by opening them from the dashboard. The in-built text-editor is less-featured than a full IDE but is useful for quick edits of source files and previewing data files.\n",
"\n",
"The `Running` tab of the dashboard gives a list of the currently running notebook instances. This can be useful to keep track of which notebooks are still running and to shutdown (or reopen) old notebook processes when the corresponding tab has been closed.\n",
"\n",
"### The notebook interface\n",
"\n",
"The top of your notebook window should appear something like this:\n",
"\n",
"<img src='res/jupyter-notebook-interface.png' />\n",
"\n",
"The name of the current notebook is displayed at the top of the page and can be edited by clicking on the text of the name. Displayed alongside this is an indication of the last manual *checkpoint* of the notebook file. On-going changes are auto-saved at regular intervals; the check-point mechanism is mainly meant as a way to recover an earlier version of a notebook after making unwanted changes. Note the default system only currently supports storing a single previous checkpoint despite the `Revert to checkpoint` dropdown under the `File` menu perhaps suggesting otherwise.\n",
"\n",
"As well as having options to save and revert to checkpoints, the `File` menu also allows new notebooks to be created in same directory as the current notebook, a copy of the current notebook to be made and the ability to export the current notebook to various formats.\n",
"\n",
"The `Edit` menu contains standard clipboard functions as well as options for reorganising notebook *cells*. Cells are the basic units of notebooks, and can contain formatted text like the one you are reading at the moment or runnable code as we will see below. The `Edit` and `Insert` drop down menus offer various options for moving cells around the notebook, merging and splitting cells and inserting new ones, while the `Cell` menu allow running of code cells and changing cell types.\n",
"\n",
"The `Kernel` menu offers some useful commands for managing the Python process (kernel) running in the notebook. In particular it provides options for interrupting a busy kernel (useful for example if you realise you have set a slow code cell running with incorrect parameters) and to restart the current kernel. This will cause all variables currently defined in the workspace to be lost but may be necessary to get the kernel back to a consistent state after polluting the namespace with lots of global variables or when trying to run code from an updated module and `reload` is failing to work. \n",
"\n",
"To the far right of the menu toolbar is a kernel status indicator. When a dark filled circle is shown this means the kernel is currently busy and any further code cell run commands will be queued to happen after the currently running cell has completed. An open status circle indicates the kernel is currently idle.\n",
"\n",
"The final row of the top notebook interface is the notebook toolbar which contains shortcut buttons to some common commands such as clipboard actions and cell / kernel management. If you are interested in learning more about the notebook user interface you may wish to run through the `User Interface Tour` under the `Help` menu drop down.\n",
"\n",
"### Markdown cells: easy text formatting\n",
"\n",
"This entire introduction has been written in what is termed a *Markdown* cell of a notebook. [Markdown](https://en.wikipedia.org/wiki/Markdown) is a lightweight markup language intended to be readable in plain-text. As you may wish to use Markdown cells to keep your own formatted notes in notebooks, a small sampling of the formatting syntax available is below (escaped mark-up on top and corresponding rendered output below that); there are many much more extensive syntax guides - for example [this cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet).\n",
"\n",
"---\n",
"\n",
"```\n",
"## Level 2 heading\n",
"### Level 3 heading\n",
"\n",
"*Italicised* and **bold** text.\n",
"\n",
" * bulleted\n",
" * lists\n",
" \n",
"and\n",
"\n",
" 1. enumerated\n",
" 2. lists\n",
"\n",
"Inline maths $y = mx + c$ using [MathJax](https://www.mathjax.org/) as well as display style\n",
"\n",
"$$ ax^2 + bx + c = 0 \\qquad \\Rightarrow \\qquad x = \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a} $$\n",
"```\n",
"---\n",
"\n",
"## Level 2 heading\n",
"### Level 3 heading\n",
"\n",
"*Italicised* and **bold** text.\n",
"\n",
" * bulleted\n",
" * lists\n",
" \n",
"and\n",
"\n",
" 1. enumerated\n",
" 2. lists\n",
"\n",
"Inline maths $y = mx + c$ using [MathJax]() as well as display maths\n",
"\n",
"$$ ax^2 + bx + c = 0 \\qquad \\Rightarrow \\qquad x = \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a} $$\n",
"\n",
"---\n",
"\n",
"We can also directly use HTML tags in Markdown cells to embed rich content such as images and videos.\n",
"\n",
"---\n",
"```\n",
"<img src=\"http://placehold.it/350x150\" />\n",
"```\n",
"---\n",
"\n",
"<img src=\"http://placehold.it/350x150\" />\n",
"\n",
"---\n",
"\n",
" \n",
"### Code cells: in browser code execution\n",
"\n",
"Up to now we have not seen any runnable code. An example of a executable code cell is below. To run it first click on the cell so that it is highlighted, then either click the <i class=\"fa-step-forward fa\"></i> button on the notebook toolbar, go to `Cell > Run Cells` or use the keyboard shortcut `Ctrl+Enter`."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from __future__ import print_function\n",
"import sys\n",
"\n",
"print('Hello world!')\n",
"print('Alarming hello!', file=sys.stderr)\n",
"print('Hello again!')\n",
"'And again!'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This example shows the three main components of a code cell.\n",
"\n",
"The most obvious is the input area. This (unsuprisingly) is used to enter the code to be run which will be automatically syntax highlighted.\n",
"\n",
"To the immediate left of the input area is the execution indicator / counter. Before a code cell is first run this will display `In [ ]:`. After the cell is run this is updated to `In [n]:` where `n` is a number corresponding to the current execution counter which is incremented whenever any code cell in the notebook is run. This can therefore be used to keep track of the relative order in which cells were last run. There is no fundamental requirement to run cells in the order they are organised in the notebook, though things will usually be more readable if you keep things in roughly in order!\n",
"\n",
"Immediately below the input area is the output area. This shows any output produced by the code in the cell. This is dealt with a little bit confusingly in the current Jupyter version. At the top any output to [`stdout`](https://en.wikipedia.org/wiki/Standard_streams#Standard_output_.28stdout.29) is displayed. Immediately below that output to [`stderr`](https://en.wikipedia.org/wiki/Standard_streams#Standard_error_.28stderr.29) is displayed. All of the output to `stdout` is displayed together even if there has been output to `stderr` between as shown by the suprising ordering in the output here. \n",
"\n",
"The final part of the output area is the *display* area. By default this will just display the returned output of the last Python statement as would usually be the case in a (I)Python interpreter run in a terminal. What is displayed for a particular object is by default determined by its special `__repr__` method e.g. for a string it is just the quote enclosed value of the string itself.\n",
"\n",
"### Useful keyboard shortcuts\n",
"\n",
"There are a wealth of keyboard shortcuts available in the notebook interface. For an exhaustive list see the `Keyboard Shortcuts` option under the `Help` menu. We will cover a few of those we find most useful below.\n",
"\n",
"Shortcuts come in two flavours: those applicable in *command mode*, active when no cell is currently being edited and indicated by a blue highlight around the current cell; those applicable in *edit mode* when the content of a cell is being edited, indicated by a green current cell highlight.\n",
"\n",
"In edit mode of a code cell, two of the more generically useful keyboard shortcuts are offered by the `Tab` key.\n",
"\n",
" * Pressing `Tab` a single time while editing code will bring up suggested completions of what you have typed so far. This is done in a scope aware manner so for example typing `a` + `[Tab]` in a code cell will come up with a list of objects beginning with `a` in the current global namespace, while typing `np.a` + `[Tab]` (assuming `import numpy as np` has been run already) will bring up a list of objects in the root NumPy namespace beginning with `a`.\n",
" * Pressing `Shift+Tab` once immediately after opening parenthesis of a function or method will cause a tool-tip to appear with the function signature (including argument names and defaults) and its docstring. Pressing `Shift+Tab` twice in succession will cause an expanded version of the same tooltip to appear, useful for longer docstrings. Pressing `Shift+Tab` four times in succession will cause the information to be instead displayed in a pager docked to bottom of the notebook interface which stays attached even when making further edits to the code cell and so can be useful for keeping documentation visible when editing e.g. to help remember the name of arguments to a function and their purposes.\n",
"\n",
"A series of useful shortcuts available in both command and edit mode are `[modifier]+Enter` where `[modifier]` is one of `Ctrl` (run selected cell), `Shift` (run selected cell and select next) or `Alt` (run selected cell and insert a new cell after).\n",
"\n",
"A useful command mode shortcut to know about is the ability to toggle line numbers on and off for a cell by pressing `L` which can be useful when trying to diagnose stack traces printed when an exception is raised or when referring someone else to a section of code.\n",
" \n",
"### Magics\n",
"\n",
"There are a range of *magic* commands in IPython notebooks, than provide helpful tools outside of the usual Python syntax. A full list of the inbuilt magic commands is given [here](http://ipython.readthedocs.io/en/stable/interactive/magics.html), however three that are particularly useful for this course:\n",
"\n",
" * [`%%timeit`](http://ipython.readthedocs.io/en/stable/interactive/magics.html?highlight=matplotlib#magic-timeit) Put at the beginning of a cell to time its execution and print the resulting timing statistics.\n",
" * [`%precision`](http://ipython.readthedocs.io/en/stable/interactive/magics.html?highlight=matplotlib#magic-precision) Set the precision for pretty printing of floating point values and NumPy arrays.\n",
" * [`%debug`](http://ipython.readthedocs.io/en/stable/interactive/magics.html?highlight=matplotlib#magic-debug) Activates the interactive debugger in a cell. Run after an exception has been occured to help diagnose the issue.\n",
" \n",
"### Plotting with `matplotlib`\n",
"\n",
"When setting up your environment one of the dependencies we asked you to install was `matplotlib`. This is an extensive plotting and data visualisation library which is tightly integrated with NumPy and Jupyter notebooks.\n",
"\n",
"When using `matplotlib` in a notebook you should first run the [magic command](http://ipython.readthedocs.io/en/stable/interactive/magics.html?highlight=matplotlib)\n",
"\n",
"```\n",
"%matplotlib inline\n",
"```\n",
"\n",
"This will cause all plots to be automatically displayed as images in the output area of the cell they are created in. Below we give a toy example of plotting two sinusoids using `matplotlib` to show case some of the basic plot options. To see the output produced select the cell and then run it."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"\n",
"# generate a pair of sinusoids\n",
"x = np.linspace(0., 2. * np.pi, 100)\n",
"y1 = np.sin(x)\n",
"y2 = np.cos(x)\n",
"\n",
"# produce a new figure object with a defined (width, height) in inches\n",
"fig = plt.figure(figsize=(8, 4))\n",
"# add a single axis to the figure\n",
"ax = fig.add_subplot(111)\n",
"# plot the two sinusoidal traces on the axis, adjusting the line width\n",
"# and adding LaTeX legend labels\n",
"ax.plot(x, y1, linewidth=2, label=r'$\\sin(x)$')\n",
"ax.plot(x, y2, linewidth=2, label=r'$\\cos(x)$')\n",
"# set the axis labels\n",
"ax.set_xlabel('$x$', fontsize=16)\n",
"ax.set_ylabel('$y$', fontsize=16)\n",
"# force the legend to be displayed\n",
"ax.legend()\n",
"# adjust the limits of the horizontal axis\n",
"ax.set_xlim(0., 2. * np.pi)\n",
"# make a grid be displayed in the axis background\n",
"ax.grid(True)"
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 2
}

412
notebooks/01_Introduction.ipynb
View File

File diff suppressed because one or more lines are too long

1322
notebooks/02_Single_layer_models.ipynb
View File

File diff suppressed because one or more lines are too long

3279
notebooks/03_Multiple_layer_models.ipynb
View File

File diff suppressed because one or more lines are too long

614
notebooks/04_Generalisation_and_overfitting.ipynb
View File

@ -1,614 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Generalisation and overfitting\n",
"\n",
"In this notebook we will explore the issue of overfitting and how we can measure how well the models we train generalise their predictions to unseen data. This will build upon the introduction to generalisation given in the fourth lecture."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Exercise 1: overfitting and model complexity in a 1D regression problem\n",
"\n",
"As an exercise we will consider a regression problem. In particular, given a fixed set of (noisy) observations of the underlying functional relationship between inputs and outputs, we will attempt to use a multiple layer network model to learn to predict output values from inputs. The aim of the exercise will be to visualise how increasing the complexity of the model we fit to the training data effects the ability of the model to make predictions across the input space.\n",
"\n",
"### Function\n",
"\n",
"To keep things simple we will consider a single input-output function defined by a fourth degree polynomial (quartic)\n",
"\n",
"$$ f(x) = 10 x^4 - 17 x^3 + 8 x^2 - x $$\n",
"\n",
"with the observed values being the function values plus zero-mean Gaussian noise\n",
"\n",
"$$ y = f(x) + 0.01 \\epsilon \\qquad \\epsilon \\sim \\mathcal{N}\\left(\\cdot;\\,0,\\,1\\right) $$\n",
"\n",
"The inputs will be drawn from the uniform distribution on $[0, 1]$.\n",
"\n",
"First import the necessary modules and seed the random number generator by running the cell below."
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"plt.style.use('ggplot')\n",
"seed = 17102016 \n",
"rng = np.random.RandomState(seed)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Your Tasks:**\n",
"\n",
"Write code in the cell below to calculate a polynomial function of one dimensional inputs. \n",
"\n",
"If $\\boldsymbol{c}$ is a length $P$ vector of coefficients corresponding to increasing powers in the polynomial (starting from the constant zero power term up to the $P-1^{\\textrm{th}}$ power) the function should correspond to the following\n",
"\n",
"\\begin{equation}\n",
" f_{\\textrm{polynomial}}(x,\\ \\boldsymbol{c}) = \\sum_{p=0}^{P-1} \\left( c_p x^p \\right)\n",
"\\end{equation}"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"def polynomial_function(inputs, coefficients):\n",
" \"\"\"Calculates polynomial with given coefficients of an array of inputs.\n",
" \n",
" Args:\n",
" inputs: One-dimensional array of input values of shape (num_inputs,)\n",
" coefficients: One-dimensional array of polynomial coefficient terms\n",
" with `coefficients[0]` corresponding to the coefficient for the\n",
" zero order term in the polynomial (constant) and `coefficients[-1]`\n",
" corresponding to the highest order term.\n",
" \n",
" Returns:\n",
" One dimensional array of output values of shape (num_inputs,)\n",
" \n",
" \"\"\"\n",
" raise NotImplementedError(\"TODO Implement this function\") "
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Run the cell below to test your implementation."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"test_coefficients = np.array([-1., 3., 4.])\n",
"test_inputs = np.array([0., 0.5, 1., 2.])\n",
"test_outputs = np.array([-1., 1.5, 6., 21.])\n",
"assert polynomial_function(test_inputs, test_coefficients).shape == (4,), (\n",
" 'Function gives wrong shape output.'\n",
")\n",
"assert np.allclose(polynomial_function(test_inputs, test_coefficients), test_outputs), (\n",
" 'Function gives incorrect output values.'\n",
")\n",
"print(\"Function is correct!\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We now need to use the random number generator to sample input values and calculate the corresponding target outputs using your polynomial implementation with the relevant coefficients for our function. Do this by running the cell below."
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
"coefficients = np.array([0, -1., 8., -17., 10.])\n",
"input_dim, output_dim = 1, 1\n",
"noise_std = 0.01\n",
"num_data = 80\n",
"inputs = rng.uniform(size=(num_data, input_dim))\n",
"epsilons = rng.normal(size=num_data)\n",
"targets = (polynomial_function(inputs[:, 0], coefficients) + \n",
" epsilons * noise_std)[:, None]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We will split the generated data points in to equal sized training and validation data sets and use these to create data provider objects which we can use to train models in our framework. As the dataset is small here we will use a batch size equal to the size of the data set. Run the cell below to split the data and set up the data provider objects."
]
},
{
"cell_type": "code",
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"from mlp.data_providers import DataProvider\n",
"num_train = num_data // 2\n",
"batch_size = num_train\n",
"inputs_train, targets_train = inputs[:num_train], targets[:num_train]\n",
"inputs_valid, targets_valid = inputs[num_train:], targets[num_train:]\n",
"train_data = DataProvider(inputs_train, targets_train, batch_size=batch_size, rng=rng)\n",
"valid_data = DataProvider(inputs_valid, targets_valid, batch_size=batch_size, rng=rng)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We can now visualise the data we will be modelling. Run the cell below to plot the target outputs against inputs for both the training and validation sets. Note the clear underlying smooth functional relationship evident in the noisy data."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(8, 4))\n",
"ax = fig.add_subplot(111)\n",
"ax.plot(inputs_train[:, 0], targets_train[:, 0], '.', label='training data')\n",
"ax.plot(inputs_valid[:, 0], targets_valid[:, 0], '.', label='validation data')\n",
"ax.set_xlabel('Inputs $x$', fontsize=14)\n",
"ax.set_ylabel('Ouputs $y$', fontsize=14)\n",
"ax.legend(loc='best')\n",
"fig.tight_layout()\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Model\n",
"\n",
"We will fit models with a varying number of parameters to the training data. As multi-layer logistic sigmoid models tend to perform poorly in regressions tasks like this we will instead use a [radial basis function (RBF) network](https://en.wikipedia.org/wiki/Radial_basis_function_network).\n",
"\n",
"This model predicts the output as the weighted sum of basis functions (here Gaussian like bumps) tiled across the input space. The cell below generates a random set of weights and bias for a RBF network and plots the modelled input-output function across inputs $[0, 1]$. Run the cell below for several different number of weight parameters (specified with `num_weights` variable) to get a feel for the sort of predictions the RBF network models produce."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"num_weights = 15\n",
"weights_scale = 1.\n",
"bias_scale = 1.\n",
"\n",
"def basis_function(x, centre, scale):\n",
" return np.exp(-(x - centre)**2 / scale**2)\n",
"\n",
"weights = rng.normal(size=num_weights) * weights_scale\n",
"bias = rng.normal() * bias_scale\n",
"\n",
"centres = np.linspace(0, 1, weights.shape[0])\n",
"scale = 1. / weights.shape[0]\n",
"\n",
"xs = np.linspace(0, 1, 200)\n",
"ys = np.zeros(xs.shape[0])\n",
"\n",
"fig = plt.figure(figsize=(12, 4))\n",
"ax = fig.add_subplot(1, 1, 1)\n",
"for weight, centre in zip(weights, centres):\n",
" ys += weight * basis_function(xs, centre, scale)\n",
"ys += bias\n",
"ax.plot(xs, ys)\n",
"ax.set_xlabel('Input', fontsize=14)\n",
"ax.set_ylabel('Output', fontsize=14)\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You do not need to study the details of how to implement this model. All of the additional code you need to fit RBF networks is provided in the `RadialBasisFunctionLayer` in the `mlp.layers` module. The `RadialBasisFunctionLayer` class has the same interface as the layer classes we encountered in the previous lab, defining both `fprop` and `bprop` methods, and we can therefore include it as a layer in a `MultipleLayerModel` as with any other layer. \n",
"\n",
"Here we will use the `RadialBasisFunctionLayer` as the first layer in a two layer model. This first layer calculates the basis function terms which are then be weighted and summed together in an `AffineLayer`, the second and final layer. This illustrates the advantage of using a modular framework - we can reuse the code we previously implemented to train a quite different model architecture just by defining a new layer class. \n",
"\n",
"Run the cell below which contains some necessary setup code."
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
"from mlp.models import MultipleLayerModel\n",
"from mlp.layers import AffineLayer, RadialBasisFunctionLayer\n",
"from mlp.errors import SumOfSquaredDiffsError\n",
"from mlp.initialisers import ConstantInit, UniformInit\n",
"from mlp.learning_rules import GradientDescentLearningRule\n",
"from mlp.optimisers import Optimiser\n",
"\n",
"# Regression problem therefore use sum of squared differences error\n",
"error = SumOfSquaredDiffsError()\n",
"# Use basic gradient descent learning rule with fixed learning rate\n",
"learning_rule = GradientDescentLearningRule(0.1)\n",
"# Initialise weights from uniform distribution and zero bias\n",
"weights_init = UniformInit(-0.1, 0.1)\n",
"biases_init = ConstantInit(0.)\n",
"# Train all models for 2000 epochs\n",
"num_epoch = 2000"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The next cell defines RBF network models with varying number of weight parameters (equal to the number of basis functions) and fits each to the training set, recording the final training and validation set errors for the fitted models. Run it now to fit the models and calculate the error values."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"num_weight_list = [2, 5, 10, 25, 50, 100]\n",
"models = []\n",
"train_errors = []\n",
"valid_errors = []\n",
"for num_weight in num_weight_list:\n",
" model = MultipleLayerModel([\n",
" RadialBasisFunctionLayer(num_weight),\n",
" AffineLayer(input_dim * num_weight, output_dim, \n",
" weights_init, biases_init)\n",
" ])\n",
" optimiser = Optimiser(model, error, learning_rule, \n",
" train_data, valid_data)\n",
" print('-' * 80)\n",
" print('Training model with {0} weights'.format(num_weight))\n",
" print('-' * 80)\n",
" _ = optimiser.train(num_epoch, -1)\n",
" outputs_train = model.fprop(inputs_train)[-1]\n",
" outputs_valid = model.fprop(inputs_valid)[-1]\n",
" models.append(model)\n",
" train_errors.append(error(outputs_train, targets_train))\n",
" valid_errors.append(error(outputs_valid, targets_valid))\n",
" print(' Final training set error: {0:.1e}'.format(train_errors[-1]))\n",
" print(' Final validation set error: {0:.1e}'.format(valid_errors[-1]))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"**Your Tasks**\n",
"\n",
"In the cell below write code to [plot bar charts](http://matplotlib.org/examples/api/barchart_demo.html) of the training and validation set errors for the different fitted models.\n",
"\n",
"Some questions to think about from the plots:\n",
"\n",
" * Do the models with more free parameters fit the training data better or worse?\n",
" * What does the validation set error value tell us about the models?\n",
" * Of the models fitted here which would you say seems like it is most likely to generalise well to unseen data? \n",
" * Do any of the models seem to be overfitting?"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#TODO plot the bar charts here"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now let's visualise what the fitted model's predictions look like across the whole input space compared to the 'true' function we were trying to fit. \n",
"\n",
"**Your Tasks:** \n",
"\n",
"In the cell below, for each of the fitted models stored in the `models` list above:\n",
" * Compute output predictions for the model across a linearly spaced series of 500 input points between 0 and 1 in the input space.\n",
" * Plot the computed predicted outputs and true function values at the corresponding inputs as line plots on the same axis (use a new axis for each model).\n",
" * On the same axis plot the training data sets input-target pairs as points."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"#TODO plot the graphs here"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You should be able to relate your answers to the questions above to what you see in these plots - ask a demonstrator if you are unsure what is going on. In particular for the models which appeared to be overfitting and generalising poorly you should now have an idea how this looks in terms of the model's predictions and how these relate to the training data points and true function values.\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# PyTorch\n",
"\n",
"As we have seen in the [previous lab](https://github.com/VICO-UoE/mlpractical/tree/mlp2023-24/lab3/notebooks/03_Multiple_layer_models.ipynb), our model shows signs of overfitting after $15$ epochs. \n",
"\n",
"Overfitting occurs when the model learns the training data too well, and fails to generalise to unseen data. In this case, the model learns the noise in the training data, and fails to learn the underlying function. \n",
"\n",
"The model may be too complex, and we can reduce the complexity by reducing the number of parameters. However, this may not be the best solution, as we may not be able to learn the underlying function with a simple model.\n",
"\n",
"In practice, a model that overfits the training data will have a low training error, but a high validation error.\n",
"\n",
"*What can we deduce if we observe a high training error and a high validation error?*\n",
"\n",
"Overfitting is a common problem in machine learning, and there are many techniques to avoid it. In this lab, we will explore one of them: *early stopping*."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"import torch.nn as nn\n",
"import torch.optim as optim\n",
"from torchvision import datasets,transforms\n",
"from torch.utils.data.sampler import SubsetRandomSampler\n",
"\n",
"torch.manual_seed(seed)"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"# Device configuration\n",
"device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
"\n",
"# Set training run hyperparameters\n",
"batch_size = 128 # number of data points in a batch\n",
"learning_rate = 0.001 # learning rate for gradient descent\n",
"num_epochs = 50 # number of training epochs to perform\n",
"stats_interval = 1 # epoch interval between recording and printing stats"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"transform=transforms.Compose([\n",
" transforms.ToTensor(),\n",
" transforms.Normalize((0.1307,), (0.3081,))\n",
" ])\n",
"\n",
"train_dataset = datasets.MNIST('../data', train=True, download=True, transform=transform)\n",
"test_dataset = datasets.MNIST('../data', train=False, download=True, transform=transform)\n",
"\n",
"valid_size=0.2 # Leave 20% of training set as validation set\n",
"num_train = len(train_dataset)\n",
"indices = list(range(num_train))\n",
"split = int(np.floor(valid_size * num_train))\n",
"np.random.shuffle(indices) # Shuffle indices in-place\n",
"train_idx, valid_idx = indices[split:], indices[:split] # Split indices into training and validation sets\n",
"train_sampler = SubsetRandomSampler(train_idx)\n",
"valid_sampler = SubsetRandomSampler(valid_idx)\n",
"\n",
"# Create the dataloaders\n",
"train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, sampler=train_sampler, pin_memory=True)\n",
"valid_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, sampler=valid_sampler, pin_memory=True)\n",
"test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, pin_memory=True)"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"class MultipleLayerModel(nn.Module):\n",
" \"\"\"Multiple layer model.\"\"\"\n",
" def __init__(self, input_dim, output_dim, hidden_dim):\n",
" super().__init__()\n",
" self.flatten = nn.Flatten()\n",
" self.linear_relu_stack = nn.Sequential(\n",
" nn.Linear(input_dim, hidden_dim),\n",
" nn.ReLU(),\n",
" nn.Linear(hidden_dim, hidden_dim),\n",
" nn.ReLU(),\n",
" nn.Linear(hidden_dim, output_dim),\n",
" )\n",
" \n",
" def forward(self, x):\n",
" x = self.flatten(x)\n",
" logits = self.linear_relu_stack(x)\n",
" return logits\n",
" \n",
"input_dim = 1*28*28\n",
"output_dim = 10\n",
"hidden_dim = 100\n",
"\n",
"model = MultipleLayerModel(input_dim, output_dim, hidden_dim).to(device)\n",
"\n",
"loss = nn.CrossEntropyLoss() # Cross-entropy loss function\n",
"optimizer = optim.Adam(model.parameters(), lr=learning_rate) # Adam optimiser"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Early stopping is a simple technique to avoid overfitting. The idea is to stop training when the validation error starts to increase. This is usually done by monitoring the validation error during training, and stopping when it has not decreased for a certain number of epochs.\n",
"\n",
"*Can we state that overfitting is ultimatelly inevitable given training over a very large number of epochs?*\n",
"\n",
"In this section, we will implement early stopping in PyTorch. We will use the same model as in the previous lab, but we will train it for $50$ epochs with an early stopping."
]
},
{
"cell_type": "code",
"execution_count": 38,
"metadata": {},
"outputs": [],
"source": [
"class EarlyStopping:\n",
" \"\"\"Early stops the training if validation loss doesn't improve after a given patience.\"\"\"\n",
" def __init__(self, patience=5, min_delta=0):\n",
"\n",
" self.patience = patience # Number of epochs with no improvement after which training will be stopped\n",
" self.min_delta = min_delta # Minimum change in the monitored quantity to qualify as an improvement\n",
" self.counter = 0\n",
" self.min_validation_loss = float('inf')\n",
" self.early_stop = False\n",
"\n",
" def __call__(self, validation_loss):\n",
" # If validation loss is lower than minimum validation loss so far,\n",
" # reset counter and set minimum validation loss to current validation loss\n",
" if validation_loss < self.min_validation_loss:\n",
" self.min_validation_loss = validation_loss\n",
" self.counter = 0\n",
" # If validation loss hasn't improved since minimum validation loss,\n",
" # increment counter\n",
" elif validation_loss > (self.min_validation_loss + self.min_delta):\n",
" self.counter += 1\n",
" # If counter has reached patience, set early_stop flag to True\n",
" if self.counter >= self.patience: \n",
" self.early_stop = True"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Initialise early stopping object\n",
"early_stopping = EarlyStopping(patience=5, min_delta=0.01)\n",
"\n",
"# Keep track of the loss values over training\n",
"train_loss = [] \n",
"valid_loss = []\n",
"\n",
"# Train model\n",
"for i in range(num_epochs+1):\n",
" # Training\n",
" model.train()\n",
" batch_loss = []\n",
" for batch_idx, (x, t) in enumerate(train_loader):\n",
" x = x.to(device)\n",
" t = t.to(device)\n",
" \n",
" # Forward pass\n",
" y = model(x)\n",
" E_value = loss(y, t)\n",
" \n",
" # Backward pass\n",
" optimizer.zero_grad()\n",
" E_value.backward()\n",
" optimizer.step()\n",
" \n",
" # Logging\n",
" batch_loss.append(E_value.item())\n",
" \n",
" train_loss.append(np.mean(batch_loss))\n",
"\n",
" # Validation\n",
" model.eval()\n",
" batch_loss = []\n",
" for batch_idx, (x, t) in enumerate(valid_loader):\n",
" x = x.to(device)\n",
" t = t.to(device)\n",
" \n",
" # Forward pass\n",
" y = model(x)\n",
" E_value = loss(y, t)\n",
" \n",
" # Logging\n",
" batch_loss.append(E_value.item())\n",
" \n",
" valid_loss.append(np.mean(batch_loss))\n",
"\n",
" if i % stats_interval == 0:\n",
" print('Epoch: {} \\tError(train): {:.6f} \\tError(valid): {:.6f} '.format(\n",
" i, train_loss[-1], valid_loss[-1]))\n",
" \n",
" # Check for early stopping\n",
" early_stopping(valid_loss[-1])\n",
"\n",
" if early_stopping.early_stop:\n",
" print(\"Early stopping\")\n",
" break # Stop training"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Plot the change in the validation and training set error over training.\n",
"fig_1 = plt.figure(figsize=(8, 4))\n",
"ax_1 = fig_1.add_subplot(111)\n",
"ax_1.plot(train_loss, label='Error(train)')\n",
"ax_1.plot(valid_loss, label='Error(valid)')\n",
"ax_1.legend(loc=0)\n",
"ax_1.set_xlabel('Epoch number')\n",
"plt.show()"
]
}
],
"metadata": {
"anaconda-cloud": {},
"kernelspec": {
"display_name": "mlp",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.5"
}
},
"nbformat": 4,
"nbformat_minor": 1
}

182
notebooks/Coursework_1.ipynb
View File

@ -1,182 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Coursework 1\n",
"\n",
"This notebook is intended to be used as a starting point for your experiments. The instructions can be found in the MLP2024_25_CW1_Spec.pdf (see Learn, Assignment Submission, Coursework 1). The methods provided here are just helper functions. If you want more complex graphs such as side by side comparisons of different experiments you should learn more about matplotlib and implement them. Before each experiment remember to re-initialize neural network weights and reset the data providers so you get a properly initialized experiment. For each experiment try to keep most hyperparameters the same except the one under investigation so you can understand what the effects of each are."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"plt.style.use('ggplot')\n",
"\n",
"def train_model_and_plot_stats(\n",
" model, error, learning_rule, train_data, valid_data, num_epochs, stats_interval, notebook=True):\n",
" \n",
" # As well as monitoring the error over training also monitor classification\n",
" # accuracy i.e. proportion of most-probable predicted classes being equal to targets\n",
" data_monitors={'acc': lambda y, t: (y.argmax(-1) == t.argmax(-1)).mean()}\n",
"\n",
" # Use the created objects to initialise a new Optimiser instance.\n",
" optimiser = Optimiser(\n",
" model, error, learning_rule, train_data, valid_data, data_monitors, notebook=notebook)\n",
"\n",
" # Run the optimiser for num_epochs epochs (full passes through the training set)\n",
" # printing statistics every epoch.\n",
" stats, keys, run_time = optimiser.train(num_epochs=num_epochs, stats_interval=stats_interval)\n",
"\n",
" # Plot the change in the validation and training set error over training.\n",
" fig_1 = plt.figure(figsize=(8, 4))\n",
" ax_1 = fig_1.add_subplot(111)\n",
" for k in ['error(train)', 'error(valid)']:\n",
" ax_1.plot(np.arange(1, stats.shape[0]) * stats_interval, \n",
" stats[1:, keys[k]], label=k)\n",
" ax_1.legend(loc=0)\n",
" ax_1.set_xlabel('Epoch number')\n",
" ax_1.set_ylabel('Error')\n",
"\n",
" # Plot the change in the validation and training set accuracy over training.\n",
" fig_2 = plt.figure(figsize=(8, 4))\n",
" ax_2 = fig_2.add_subplot(111)\n",
" for k in ['acc(train)', 'acc(valid)']:\n",
" ax_2.plot(np.arange(1, stats.shape[0]) * stats_interval, \n",
" stats[1:, keys[k]], label=k)\n",
" ax_2.legend(loc=0)\n",
" ax_2.set_xlabel('Epoch number')\n",
" ax_2.set_xlabel('Accuracy')\n",
" \n",
" return stats, keys, run_time, fig_1, ax_1, fig_2, ax_2"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# The below code will set up the data providers, random number\n",
"# generator and logger objects needed for training runs. As\n",
"# loading the data from file take a little while you generally\n",
"# will probably not want to reload the data providers on\n",
"# every training run. If you wish to reset their state you\n",
"# should instead use the .reset() method of the data providers.\n",
"import numpy as np\n",
"import logging\n",
"import sys\n",
"# sys.path.append('/path/to/mlpractical')\n",
"from mlp.data_providers import MNISTDataProvider, EMNISTDataProvider\n",
"\n",
"# Seed a random number generator\n",
"seed = 11102019 \n",
"rng = np.random.RandomState(seed)\n",
"batch_size = 100\n",
"# Set up a logger object to print info about the training run to stdout\n",
"logger = logging.getLogger()\n",
"logger.setLevel(logging.INFO)\n",
"logger.handlers = [logging.StreamHandler()]\n",
"\n",
"# Create data provider objects for the MNIST data set\n",
"train_data = EMNISTDataProvider('train', batch_size=batch_size, rng=rng)\n",
"valid_data = EMNISTDataProvider('valid', batch_size=batch_size, rng=rng)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# The model set up code below is provided as a starting point.\n",
"# You will probably want to add further code cells for the\n",
"# different experiments you run.\n",
"\n",
"%pip install tqdm\n",
"\n",
"from mlp.layers import AffineLayer, SoftmaxLayer, SigmoidLayer, ReluLayer\n",
"from mlp.errors import CrossEntropySoftmaxError\n",
"from mlp.models import MultipleLayerModel\n",
"from mlp.initialisers import ConstantInit, GlorotUniformInit\n",
"from mlp.learning_rules import AdamLearningRule\n",
"from mlp.optimisers import Optimiser\n",
"\n",
"# Setup hyperparameters\n",
"learning_rate = 0.001\n",
"num_epochs = 100\n",
"stats_interval = 1\n",
"input_dim, output_dim, hidden_dim = 784, 47, 128\n",
"\n",
"weights_init = GlorotUniformInit(rng=rng)\n",
"biases_init = ConstantInit(0.)\n",
"\n",
"# Create model with ONE hidden layer\n",
"model = MultipleLayerModel([\n",
" AffineLayer(input_dim, hidden_dim, weights_init, biases_init), # hidden layer\n",
" ReluLayer(),\n",
" AffineLayer(hidden_dim, output_dim, weights_init, biases_init) # output layer\n",
"])\n",
"\n",
"error = CrossEntropySoftmaxError()\n",
"# Use a Adam learning rule\n",
"learning_rule = AdamLearningRule(learning_rate=learning_rate)\n",
"\n",
"# Remember to use notebook=False when you write a script to be run in a terminal\n",
"_ = train_model_and_plot_stats(\n",
" model, error, learning_rule, train_data, valid_data, num_epochs, stats_interval, notebook=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Create model with TWO hidden layers\n",
"model = MultipleLayerModel([\n",
" AffineLayer(input_dim, hidden_dim, weights_init, biases_init), # first hidden layer\n",
" ReluLayer(),\n",
" AffineLayer(hidden_dim, hidden_dim, weights_init, biases_init), # second hidden layer\n",
" ReluLayer(),\n",
" AffineLayer(hidden_dim, output_dim, weights_init, biases_init) # output layer\n",
"])\n",
"\n",
"error = CrossEntropySoftmaxError()\n",
"# Use a Adam learning rule\n",
"learning_rule = AdamLearningRule(learning_rate=learning_rate)\n",
"\n",
"# Remember to use notebook=False when you write a script to be run in a terminal\n",
"_ = train_model_and_plot_stats(\n",
" model, error, learning_rule, train_data, valid_data, num_epochs, stats_interval, notebook=True)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

665
notebooks/Coursework_2_Pytorch_Introduction.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

122
notebooks/DropoutandPenalty_tests.ipynb
View File

@ -1,122 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"import sys\n",
"# sys.path.append('/path/to/mlpractical')\n",
"from mlp.test_methods import test_dropout_layer\n",
"import numpy as np\n",
"\n",
"fprop_test, fprop_output, fprop_correct, \\\n",
"bprop_test, bprop_output, bprop_correct = test_dropout_layer()\n",
"\n",
"assert fprop_test == 1.0, (\n",
"'The dropout layer fprop functionality test failed'\n",
"'Correct output is \\n\\n{0}\\n\\n but returned output is \\n\\n{1}\\n\\n difference is \\n\\n{2}'\n",
".format(fprop_correct, fprop_output, fprop_output-fprop_correct)\n",
")\n",
"\n",
"print(\"Dropout Layer Fprop Functionality Test Passed\")\n",
"\n",
"assert bprop_test == 1.0, (\n",
"'The dropout layer bprop functionality test failed'\n",
"'Correct output is \\n\\n{0}\\n\\n but returned output is \\n\\n{1}\\n\\n difference is \\n\\n{2}'\n",
".format(bprop_correct, bprop_output, bprop_output-bprop_correct)\n",
")\n",
"\n",
"print(\"Dropout Layer Bprop Test Passed\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from mlp.test_methods import test_L1_Penalty\n",
"\n",
"\n",
"call_test, call_output, call_correct, \\\n",
"grad_test, grad_output, grad_correct = test_L1_Penalty()\n",
"\n",
"\n",
"assert call_test == 1.0, (\n",
"'The call function for L1 Penalty test failed'\n",
"'Correct output is \\n\\n{0}\\n\\n but returned output is \\n\\n{1}\\n\\n difference is \\n\\n{2}'\n",
".format(call_correct, call_output, call_output-call_correct)\n",
")\n",
"\n",
"print(\"L1 Penalty Call Functionality Test Passed\")\n",
"\n",
"assert grad_test == 1.0, (\n",
"'The grad function for L1 Penalty test failed'\n",
"'Correct output is \\n\\n{0}\\n\\n but returned output is \\n\\n{1}\\n\\n difference is \\n\\n{2}'\n",
".format(grad_correct, grad_output, grad_output-grad_correct, grad_output/grad_correct)\n",
")\n",
"\n",
"\n",
"\n",
"print(\"L1 Penalty Grad Function Test Passed\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from mlp.test_methods import test_L2_Penalty\n",
"\n",
"\n",
"call_test, call_output, call_correct, \\\n",
"grad_test, grad_output, grad_correct = test_L2_Penalty()\n",
"\n",
"\n",
"assert call_test == 1.0, (\n",
"'The call function for L2 Penalty test failed'\n",
"'Correct output is \\n\\n{0}\\n\\n but returned output is \\n\\n{1}\\n\\n difference is \\n\\n{2}'\n",
".format(call_correct, call_output, call_output-call_correct)\n",
")\n",
"\n",
"print(\"L2 Penalty Call Functionality Test Passed\")\n",
"\n",
"assert grad_test == 1.0, (\n",
"'The grad function for L2 Penalty test failed'\n",
"'Correct output is \\n\\n{0}\\n\\n but returned output is \\n\\n{1}\\n\\n difference is \\n\\n{2}'\n",
".format(grad_correct, grad_output, grad_output-grad_correct, grad_output/grad_correct)\n",
")\n",
"\n",
"\n",
"\n",
"print(\"L2 Penalty Grad Function Test Passed\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 1
}

200
notebooks/Plot_Results.ipynb Normal file
View File

File diff suppressed because one or more lines are too long

BIN
notebooks/res/._fprop-bprop-block-diagram.png
View File

Binary file not shown.

BIN
notebooks/res/._jupyter-dashboard.png
View File

Binary file not shown.

BIN
notebooks/res/._jupyter-notebook-interface.png
View File

Binary file not shown.

BIN
notebooks/res/._singleLayerNetBP-1.png
View File

Binary file not shown.

BIN
notebooks/res/._singleLayerNetPredict.png
View File

Binary file not shown.

BIN
notebooks/res/._singleLayerNetWts-1.png
View File

Binary file not shown.

BIN
notebooks/res/._singleLayerNetWtsEqns-1.png
View File

Binary file not shown.

105
scripts/generate_regularization_layer_test_outputs.py
View File

@ -1,105 +0,0 @@
import argparse
import os
import numpy as np
import sys
# sys.path.append('/path/to/mlpractical')
from mlp.layers import DropoutLayer
from mlp.penalties import L1Penalty, L2Penalty
parser = argparse.ArgumentParser(description='Welcome to regularization test script')
parser.add_argument('--student_id', nargs="?", type=str, help='Your student id in the format "Sxxxxxxx"')
args = parser.parse_args()
student_id = args.student_id
def fprop_bprop_layer(inputs, dropout_layer, grads_wrt_outputs, weights, params=False):
if params:
dropout_layer.params = [weights]
fprop = dropout_layer.fprop(inputs)
bprop = dropout_layer.bprop(inputs, fprop, grads_wrt_outputs)
outputs = [fprop, bprop]
if params:
grads_wrt_weights = dropout_layer.grads_wrt_params(
inputs, grads_wrt_outputs)
outputs.append(grads_wrt_weights)
return outputs
def call_grad_layer(inputs, penalty_layer, grads_wrt_outputs, weights, params=False):
if params:
penalty_layer.params = [weights]
call = penalty_layer(inputs)
grad = penalty_layer.grad(inputs)
outputs = [call, grad]
if params:
grads_wrt_weights = penalty_layer.grads_wrt_params(
inputs, grads_wrt_outputs)
outputs.append(grads_wrt_weights)
return outputs
def get_student_seed(student_id):
student_seed_number = int(student_id[1:])
return student_seed_number
seed = get_student_seed(student_id)
rng = np.random.RandomState(seed)
reg_output_dict = dict()
inputs = rng.normal(loc=0.0, scale=1.0, size=(32, 3, 8, 8))
grads_wrt_outputs = rng.normal(loc=0.0, scale=1.0, size=(32, 3, 8, 8))
weights = rng.normal(loc=0.0, scale=1.0, size=(1))
reg_output_dict['inputs'] = inputs
reg_output_dict['weights'] = weights
reg_output_dict['grads_wrt_outputs'] = grads_wrt_outputs
for dropout_layer, params_flag in zip(
[DropoutLayer],
[False]):
if isinstance(dropout_layer(), DropoutLayer):
rng = np.random.RandomState(92019)
print(True)
outputs = fprop_bprop_layer(inputs, dropout_layer(
rng=rng), grads_wrt_outputs, weights, params_flag)
else:
outputs = fprop_bprop_layer(
inputs, dropout_layer(), grads_wrt_outputs, weights, params_flag)
reg_output_dict['{}_{}'.format(
dropout_layer.__name__, 'fprop')] = outputs[0]
reg_output_dict['{}_{}'.format(
dropout_layer.__name__, 'bprop')] = outputs[1]
if params_flag:
reg_output_dict['{}_{}'.format(
dropout_layer.__name__, 'grads_wrt_outputs')] = outputs[2]
for penalty_layer, params_flag in zip(
[L1Penalty, L2Penalty], [False, False]):
outputs = call_grad_layer(inputs, penalty_layer(
1e-4), grads_wrt_outputs, weights, params_flag)
reg_output_dict['{}_{}'.format(
penalty_layer.__name__, '__call__correct')] = outputs[0]
reg_output_dict['{}_{}'.format(
penalty_layer.__name__, 'grad_correct')] = outputs[1]
if params_flag:
reg_output_dict['{}_{}'.format(
penalty_layer.__name__, 'grads_wrt_outputs')] = outputs[2]
np.save(os.path.join(os.environ['MLP_DATA_DIR'],
'{}_regularization_test_pack.npy'.format(seed)), reg_output_dict)
test_data = np.load(os.path.join(
os.environ['MLP_DATA_DIR'], '{}_regularization_test_pack.npy'.format(seed)), allow_pickle=True)
for key, value in test_data.item().items():
print(key, value)