From 5820d0a35ca32e29fd6546a5bdc7986105267457 Mon Sep 17 00:00:00 2001 From: AntreasAntoniou Date: Fri, 6 Oct 2017 14:46:19 +0100 Subject: [PATCH] Lab3 initial commit --- mlp/data_providers.py | 83 +- mlp/errors.py | 144 +- mlp/layers.py | 130 +- mlp/models.py | 74 +- mlp/optimisers.py | 5 +- notebooks/02_Single_layer_models.ipynb | 2571 ++++++++++++++++++- notebooks/03_Multiple_layer_models.ipynb | 750 ++++++ notebooks/res/fprop-bprop-block-diagram.pdf | Bin 0 -> 13789 bytes notebooks/res/fprop-bprop-block-diagram.png | Bin 0 -> 7112 bytes notebooks/res/fprop-bprop-block-diagram.tex | 65 + 10 files changed, 3751 insertions(+), 71 deletions(-) create mode 100644 notebooks/03_Multiple_layer_models.ipynb create mode 100644 notebooks/res/fprop-bprop-block-diagram.pdf create mode 100644 notebooks/res/fprop-bprop-block-diagram.png create mode 100644 notebooks/res/fprop-bprop-block-diagram.tex diff --git a/mlp/data_providers.py b/mlp/data_providers.py index ebea079..3d26d5e 100644 --- a/mlp/data_providers.py +++ b/mlp/data_providers.py @@ -35,23 +35,54 @@ class DataProvider(object): """ self.inputs = inputs self.targets = targets - self.batch_size = batch_size - assert max_num_batches != 0 and not max_num_batches < -1, ( - 'max_num_batches should be -1 or > 0') - self.max_num_batches = max_num_batches + if batch_size < 1: + raise ValueError('batch_size must be >= 1') + self._batch_size = batch_size + if max_num_batches == 0 or max_num_batches < -1: + raise ValueError('max_num_batches must be -1 or > 0') + self._max_num_batches = max_num_batches + self._update_num_batches() + self.shuffle_order = shuffle_order + self._current_order = np.arange(inputs.shape[0]) + if rng is None: + rng = np.random.RandomState(DEFAULT_SEED) + self.rng = rng + self.new_epoch() + + @property + def batch_size(self): + """Number of data points to include in each batch.""" + return self._batch_size + + @batch_size.setter + def batch_size(self, value): + if value < 1: + raise ValueError('batch_size must be >= 1') + self._batch_size = value + self._update_num_batches() + + @property + def max_num_batches(self): + """Maximum number of batches to iterate over in an epoch.""" + return self._max_num_batches + + @max_num_batches.setter + def max_num_batches(self, value): + if value == 0 or value < -1: + raise ValueError('max_num_batches must be -1 or > 0') + self._max_num_batches = value + self._update_num_batches() + + def _update_num_batches(self): + """Updates number of batches to iterate over.""" # maximum possible number of batches is equal to number of whole times # batch_size divides in to the number of data points which can be # found using integer division - possible_num_batches = self.inputs.shape[0] // batch_size + possible_num_batches = self.inputs.shape[0] // self.batch_size if self.max_num_batches == -1: self.num_batches = possible_num_batches else: self.num_batches = min(self.max_num_batches, possible_num_batches) - self.shuffle_order = shuffle_order - if rng is None: - rng = np.random.RandomState(DEFAULT_SEED) - self.rng = rng - self.reset() def __iter__(self): """Implements Python iterator interface. @@ -63,27 +94,36 @@ class DataProvider(object): """ return self - def reset(self): - """Resets the provider to the initial state to use in a new epoch.""" + def new_epoch(self): + """Starts a new epoch (pass through data), possibly shuffling first.""" self._curr_batch = 0 if self.shuffle_order: self.shuffle() + def __next__(self): + self.next() + + def reset(self): + """Resets the provider to the initial state.""" + inv_perm = np.argsort(self._current_order) + self._current_order = self._current_order[inv_perm] + self.inputs = self.inputs[inv_perm] + self.targets = self.targets[inv_perm] + self.new_epoch() + def shuffle(self): """Randomly shuffles order of data.""" - new_order = self.rng.permutation(self.inputs.shape[0]) - self.inputs = self.inputs[new_order] - self.targets = self.targets[new_order] - - def __next__(self): - return self.next() + perm = self.rng.permutation(self.inputs.shape[0]) + self._current_order = self._current_order[perm] + self.inputs = self.inputs[perm] + self.targets = self.targets[perm] def next(self): """Returns next data batch or raises `StopIteration` if at end.""" if self._curr_batch + 1 > self.num_batches: - # no more batches in current iteration through data set so reset - # the dataset for another pass and indicate iteration is at end - self.reset() + # no more batches in current iteration through data set so start + # new epoch ready for another pass and indicate iteration is at end + self.new_epoch() raise StopIteration() # create an index slice corresponding to current batch number batch_slice = slice(self._curr_batch * self.batch_size, @@ -93,7 +133,6 @@ class DataProvider(object): self._curr_batch += 1 return inputs_batch, targets_batch - class MNISTDataProvider(DataProvider): """Data provider for MNIST handwritten digit images.""" diff --git a/mlp/errors.py b/mlp/errors.py index 712fe59..5ef95f7 100644 --- a/mlp/errors.py +++ b/mlp/errors.py @@ -23,9 +23,9 @@ class SumOfSquaredDiffsError(object): targets: Array of target outputs of shape (batch_size, output_dim). Returns: - Scalar error function value. + Scalar cost function value. """ - raise NotImplementedError() + return 0.5 * np.mean(np.sum((outputs - targets)**2, axis=1)) def grad(self, outputs, targets): """Calculates gradient of error function with respect to outputs. @@ -35,10 +35,142 @@ class SumOfSquaredDiffsError(object): targets: Array of target outputs of shape (batch_size, output_dim). Returns: - Gradient of error function with respect to outputs. This should be - an array of shape (batch_size, output_dim). + Gradient of error function with respect to outputs. """ - raise NotImplementedError() + return (outputs - targets) / outputs.shape[0] def __repr__(self): - return 'SumOfSquaredDiffsError' + return 'MeanSquaredErrorCost' + + +class BinaryCrossEntropyError(object): + """Binary cross entropy error.""" + + def __call__(self, outputs, targets): + """Calculates error function given a batch of outputs and targets. + + Args: + outputs: Array of model outputs of shape (batch_size, output_dim). + targets: Array of target outputs of shape (batch_size, output_dim). + + Returns: + Scalar error function value. + """ + return -np.mean( + targets * np.log(outputs) + (1. - targets) * np.log(1. - ouputs)) + + def grad(self, outputs, targets): + """Calculates gradient of error function with respect to outputs. + + Args: + outputs: Array of model outputs of shape (batch_size, output_dim). + targets: Array of target outputs of shape (batch_size, output_dim). + + Returns: + Gradient of error function with respect to outputs. + """ + return ((1. - targets) / (1. - outputs) - + (targets / outputs)) / outputs.shape[0] + + def __repr__(self): + return 'BinaryCrossEntropyError' + + +class BinaryCrossEntropySigmoidError(object): + """Binary cross entropy error with logistic sigmoid applied to outputs.""" + + def __call__(self, outputs, targets): + """Calculates error function given a batch of outputs and targets. + + Args: + outputs: Array of model outputs of shape (batch_size, output_dim). + targets: Array of target outputs of shape (batch_size, output_dim). + + Returns: + Scalar error function value. + """ + probs = 1. / (1. + np.exp(-outputs)) + return -np.mean( + targets * np.log(probs) + (1. - targets) * np.log(1. - probs)) + + def grad(self, outputs, targets): + """Calculates gradient of error function with respect to outputs. + + Args: + outputs: Array of model outputs of shape (batch_size, output_dim). + targets: Array of target outputs of shape (batch_size, output_dim). + + Returns: + Gradient of error function with respect to outputs. + """ + probs = 1. / (1. + np.exp(-outputs)) + return (probs - targets) / outputs.shape[0] + + def __repr__(self): + return 'BinaryCrossEntropySigmoidError' + + +class CrossEntropyError(object): + """Multi-class cross entropy error.""" + + def __call__(self, outputs, targets): + """Calculates error function given a batch of outputs and targets. + + Args: + outputs: Array of model outputs of shape (batch_size, output_dim). + targets: Array of target outputs of shape (batch_size, output_dim). + + Returns: + Scalar error function value. + """ + return -np.mean(np.sum(targets * np.log(outputs), axis=1)) + + def grad(self, outputs, targets): + """Calculates gradient of error function with respect to outputs. + + Args: + outputs: Array of model outputs of shape (batch_size, output_dim). + targets: Array of target outputs of shape (batch_size, output_dim). + + Returns: + Gradient of error function with respect to outputs. + """ + return -(targets / outputs) / outputs.shape[0] + + def __repr__(self): + return 'CrossEntropyError' + + +class CrossEntropySoftmaxError(object): + """Multi-class cross entropy error with Softmax applied to outputs.""" + + def __call__(self, outputs, targets): + """Calculates error function given a batch of outputs and targets. + + Args: + outputs: Array of model outputs of shape (batch_size, output_dim). + targets: Array of target outputs of shape (batch_size, output_dim). + + Returns: + Scalar error function value. + """ + probs = np.exp(outputs) + probs /= probs.sum(-1)[:, None] + return -np.mean(np.sum(targets * np.log(probs), axis=1)) + + def grad(self, outputs, targets): + """Calculates gradient of error function with respect to outputs. + + Args: + outputs: Array of model outputs of shape (batch_size, output_dim). + targets: Array of target outputs of shape (batch_size, output_dim). + + Returns: + Gradient of error function with respect to outputs. + """ + probs = np.exp(outputs) + probs /= probs.sum(-1)[:, None] + return (probs - targets) / outputs.shape[0] + + def __repr__(self): + return 'CrossEntropySoftmaxError' diff --git a/mlp/layers.py b/mlp/layers.py index e2e871b..cc4cdda 100644 --- a/mlp/layers.py +++ b/mlp/layers.py @@ -73,7 +73,18 @@ class LayerWithParameters(Layer): """Returns a list of parameters of layer. Returns: - List of current parameter values. + List of current parameter values. This list should be in the + corresponding order to the `values` argument to `set_params`. + """ + raise NotImplementedError() + + @params.setter + def params(self, values): + """Sets layer parameters from a list of values. + + Args: + values: List of values to set parameters to. This list should be + in the corresponding order to what is returned by `get_params`. """ raise NotImplementedError() @@ -86,8 +97,7 @@ class AffineLayer(LayerWithParameters): def __init__(self, input_dim, output_dim, weights_initialiser=init.UniformInit(-0.1, 0.1), - biases_initialiser=init.ConstantInit(0.), - weights_cost=None, biases_cost=None): + biases_initialiser=init.ConstantInit(0.)): """Initialises a parameterised affine layer. Args: @@ -113,7 +123,26 @@ class AffineLayer(LayerWithParameters): Returns: outputs: Array of layer outputs of shape (batch_size, output_dim). """ - raise NotImplementedError() + return inputs.dot(self.weights.T) + self.biases + + def bprop(self, inputs, outputs, grads_wrt_outputs): + """Back propagates gradients through a layer. + + Given gradients with respect to the outputs of the layer calculates the + gradients with respect to the layer inputs. + + Args: + inputs: Array of layer inputs of shape (batch_size, input_dim). + outputs: Array of layer outputs calculated in forward pass of + shape (batch_size, output_dim). + grads_wrt_outputs: Array of gradients with respect to the layer + outputs of shape (batch_size, output_dim). + + Returns: + Array of gradients with respect to the layer inputs of shape + (batch_size, input_dim). + """ + return grads_wrt_outputs.dot(self.weights) def grads_wrt_params(self, inputs, grads_wrt_outputs): """Calculates gradients with respect to layer parameters. @@ -127,13 +156,104 @@ class AffineLayer(LayerWithParameters): list of arrays of gradients with respect to the layer parameters `[grads_wrt_weights, grads_wrt_biases]`. """ - raise NotImplementedError() + + grads_wrt_weights = np.dot(grads_wrt_outputs.T, inputs) + grads_wrt_biases = np.sum(grads_wrt_outputs, axis=0) + return [grads_wrt_weights, grads_wrt_biases] @property def params(self): """A list of layer parameter values: `[weights, biases]`.""" return [self.weights, self.biases] + @params.setter + def params(self, values): + self.weights = values[0] + self.biases = values[1] + def __repr__(self): return 'AffineLayer(input_dim={0}, output_dim={1})'.format( self.input_dim, self.output_dim) + + +class SigmoidLayer(Layer): + """Layer implementing an element-wise logistic sigmoid transformation.""" + + def fprop(self, inputs): + """Forward propagates activations through the layer transformation. + + For inputs `x` and outputs `y` this corresponds to + `y = 1 / (1 + exp(-x))`. + + Args: + inputs: Array of layer inputs of shape (batch_size, input_dim). + + Returns: + outputs: Array of layer outputs of shape (batch_size, output_dim). + """ + return 1. / (1. + np.exp(-inputs)) + + def bprop(self, inputs, outputs, grads_wrt_outputs): + """Back propagates gradients through a layer. + + Given gradients with respect to the outputs of the layer calculates the + gradients with respect to the layer inputs. + + Args: + inputs: Array of layer inputs of shape (batch_size, input_dim). + outputs: Array of layer outputs calculated in forward pass of + shape (batch_size, output_dim). + grads_wrt_outputs: Array of gradients with respect to the layer + outputs of shape (batch_size, output_dim). + + Returns: + Array of gradients with respect to the layer inputs of shape + (batch_size, input_dim). + """ + return grads_wrt_outputs * outputs * (1. - outputs) + + def __repr__(self): + return 'SigmoidLayer' + + +class SoftmaxLayer(Layer): + """Layer implementing a softmax transformation.""" + + def fprop(self, inputs): + """Forward propagates activations through the layer transformation. + + For inputs `x` and outputs `y` this corresponds to + + `y = exp(x) / sum(exp(x))`. + + Args: + inputs: Array of layer inputs of shape (batch_size, input_dim). + + Returns: + outputs: Array of layer outputs of shape (batch_size, output_dim). + """ + exp_inputs = np.exp(inputs) + return exp_inputs / exp_inputs.sum(-1)[:, None] + + def bprop(self, inputs, outputs, grads_wrt_outputs): + """Back propagates gradients through a layer. + + Given gradients with respect to the outputs of the layer calculates the + gradients with respect to the layer inputs. + + Args: + inputs: Array of layer inputs of shape (batch_size, input_dim). + outputs: Array of layer outputs calculated in forward pass of + shape (batch_size, output_dim). + grads_wrt_outputs: Array of gradients with respect to the layer + outputs of shape (batch_size, output_dim). + + Returns: + Array of gradients with respect to the layer inputs of shape + (batch_size, input_dim). + """ + return (outputs * (grads_wrt_outputs - + (grads_wrt_outputs * outputs).sum(-1)[:, None])) + + def __repr__(self): + return 'SoftmaxLayer' diff --git a/mlp/models.py b/mlp/models.py index 86a0472..5e35dbc 100644 --- a/mlp/models.py +++ b/mlp/models.py @@ -59,9 +59,75 @@ class SingleLayerModel(object): """ return self.layer.grads_wrt_params(activations[0], grads_wrt_outputs) - def params_cost(self): - """Calculates the parameter dependent cost term of the model.""" - return self.layer.params_cost() - def __repr__(self): return 'SingleLayerModel(' + str(layer) + ')' + + +class MultipleLayerModel(object): + """A model consisting of multiple layers applied sequentially.""" + + def __init__(self, layers): + """Create a new multiple layer model instance. + + Args: + layers: List of the the layer objecst defining the model in the + order they should be applied from inputs to outputs. + """ + self.layers = layers + + @property + def params(self): + """A list of all of the parameters of the model.""" + params = [] + for layer in self.layers: + if isinstance(layer, LayerWithParameters): + params += layer.params + return params + + def fprop(self, inputs): + """Forward propagates a batch of inputs through the model. + + Args: + inputs: Batch of inputs to the model. + + Returns: + List of the activations at the output of all layers of the model + plus the inputs (to the first layer) as the first element. The + last element of the list corresponds to the model outputs. + """ + activations = [inputs] + for i, layer in enumerate(self.layers): + activations.append(self.layers[i].fprop(activations[i])) + return activations + + def grads_wrt_params(self, activations, grads_wrt_outputs): + """Calculates gradients with respect to the model parameters. + + Args: + activations: List of all activations from forward pass through + model using `fprop`. + grads_wrt_outputs: Gradient with respect to the model outputs of + the scalar function parameter gradients are being calculated + for. + + Returns: + List of gradients of the scalar function with respect to all model + parameters. + """ + grads_wrt_params = [] + for i, layer in enumerate(self.layers[::-1]): + inputs = activations[-i - 2] + outputs = activations[-i - 1] + grads_wrt_inputs = layer.bprop(inputs, outputs, grads_wrt_outputs) + if isinstance(layer, LayerWithParameters): + grads_wrt_params += layer.grads_wrt_params( + inputs, grads_wrt_outputs)[::-1] + grads_wrt_outputs = grads_wrt_inputs + return grads_wrt_params[::-1] + + def __repr__(self): + return ( + 'MultiLayerModel(\n ' + + '\n '.join([str(layer) for layer in self.layers]) + + '\n)' + ) diff --git a/mlp/optimisers.py b/mlp/optimisers.py index 01dd8b6..8222807 100644 --- a/mlp/optimisers.py +++ b/mlp/optimisers.py @@ -121,6 +121,7 @@ class Optimiser(object): and the second being a dict mapping the labels for the statistics recorded to their column index in the array. """ + start_train_time = time.clock() run_stats = [list(self.get_epoch_stats().values())] for epoch in range(1, num_epochs + 1): start_time = time.clock() @@ -130,5 +131,7 @@ class Optimiser(object): stats = self.get_epoch_stats() self.log_stats(epoch, epoch_time, stats) run_stats.append(list(stats.values())) - return np.array(run_stats), {k: i for i, k in enumerate(stats.keys())} + finish_train_time = time.clock() + total_train_time = finish_train_time - start_train_time + return np.array(run_stats), {k: i for i, k in enumerate(stats.keys())}, total_train_time diff --git a/notebooks/02_Single_layer_models.ipynb b/notebooks/02_Single_layer_models.ipynb index 3b5a09f..53d6f88 100644 --- a/notebooks/02_Single_layer_models.ipynb +++ b/notebooks/02_Single_layer_models.ipynb @@ -17,7 +17,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -74,7 +74,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -92,9 +92,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.05 ms ± 148 µs per loop (mean ± std. dev. of 3 runs, 100 loops each)\n" + ] + } + ], "source": [ "%%timeit -n 100 -r 3\n", "c = np.empty(size)\n", @@ -111,9 +119,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "3.01 µs ± 1.53 µs per loop (mean ± std. dev. of 3 runs, 100 loops each)\n" + ] + } + ], "source": [ "%%timeit -n 100 -r 3\n", "c = a + b" @@ -152,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -171,7 +187,7 @@ " Returns:\n", " outputs: Array of layer outputs of shape (batch_size, output_dim).\n", " \"\"\"\n", - " raise NotImplementedError('Delete this raise statement and write your code here instead.')" + " return inputs.dot(weights.T) + biases" ] }, { @@ -183,9 +199,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "All outputs correct!\n" + ] + } + ], "source": [ "inputs = np.array([[0., -1., 2.], [-6., 3., 1.]])\n", "weights = np.array([[2., -3., -1.], [-5., 7., 2.]])\n", @@ -213,9 +237,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 6. 6. 6.]\n", + " [ 24. 24. 24.]\n", + " [ 42. 42. 42.]]\n", + "[[ 18. 24. 30.]\n", + " [ 18. 24. 30.]\n", + " [ 18. 24. 30.]]\n", + "[ 0.8 2.6 4.4]\n", + "[ 2.4 3. 3.6]\n", + "0.2\n" + ] + } + ], "source": [ "# Initiliase arrays with arbitrary values\n", "A = np.arange(9).reshape((3, 3))\n", @@ -240,9 +280,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[ 0.1 1.2]\n", + " [ 2.1 3.2]\n", + " [ 4.1 5.2]]\n", + "[[-1. 0.]\n", + " [ 2. 3.]\n", + " [ 5. 6.]]\n", + "[[ 0. 0.2]\n", + " [ 0.2 0.6]\n", + " [ 0.4 1. ]]\n" + ] + } + ], "source": [ "# Initiliase arrays with arbitrary values\n", "A = np.arange(6).reshape((3, 2))\n", @@ -275,7 +331,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -313,9 +369,799 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/javascript": [ + "/* Put everything inside the global mpl namespace */\n", + "window.mpl = {};\n", + "\n", + "\n", + "mpl.get_websocket_type = function() {\n", + " if (typeof(WebSocket) !== 'undefined') {\n", + " return WebSocket;\n", + " } else if (typeof(MozWebSocket) !== 'undefined') {\n", + " return MozWebSocket;\n", + " } else {\n", + " alert('Your browser does not have WebSocket support.' +\n", + " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", + " 'Firefox 4 and 5 are also supported but you ' +\n", + " 'have to enable WebSockets in about:config.');\n", + " };\n", + "}\n", + "\n", + "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", + " this.id = figure_id;\n", + "\n", + " this.ws = websocket;\n", + "\n", + " this.supports_binary = (this.ws.binaryType != undefined);\n", + "\n", + " if (!this.supports_binary) {\n", + " var warnings = document.getElementById(\"mpl-warnings\");\n", + " if (warnings) {\n", + " warnings.style.display = 'block';\n", + " warnings.textContent = (\n", + " \"This browser does not support binary websocket messages. \" +\n", + " \"Performance may be slow.\");\n", + " }\n", + " }\n", + "\n", + " this.imageObj = new Image();\n", + "\n", + " this.context = undefined;\n", + " this.message = undefined;\n", + " this.canvas = undefined;\n", + " this.rubberband_canvas = undefined;\n", + " this.rubberband_context = undefined;\n", + " this.format_dropdown = undefined;\n", + "\n", + " this.image_mode = 'full';\n", + "\n", + " this.root = $('
');\n", + " this._root_extra_style(this.root)\n", + " this.root.attr('style', 'display: inline-block');\n", + "\n", + " $(parent_element).append(this.root);\n", + "\n", + " this._init_header(this);\n", + " this._init_canvas(this);\n", + " this._init_toolbar(this);\n", + "\n", + " var fig = this;\n", + "\n", + " this.waiting = false;\n", + "\n", + " this.ws.onopen = function () {\n", + " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", + " fig.send_message(\"send_image_mode\", {});\n", + " if (mpl.ratio != 1) {\n", + " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n", + " }\n", + " fig.send_message(\"refresh\", {});\n", + " }\n", + "\n", + " this.imageObj.onload = function() {\n", + " if (fig.image_mode == 'full') {\n", + " // Full images could contain transparency (where diff images\n", + " // almost always do), so we need to clear the canvas so that\n", + " // there is no ghosting.\n", + " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", + " }\n", + " fig.context.drawImage(fig.imageObj, 0, 0);\n", + " };\n", + "\n", + " this.imageObj.onunload = function() {\n", + " this.ws.close();\n", + " }\n", + "\n", + " this.ws.onmessage = this._make_on_message_function(this);\n", + "\n", + " this.ondownload = ondownload;\n", + "}\n", + "\n", + "mpl.figure.prototype._init_header = function() {\n", + " var titlebar = $(\n", + " '
');\n", + " var titletext = $(\n", + " '
');\n", + " titlebar.append(titletext)\n", + " this.root.append(titlebar);\n", + " this.header = titletext[0];\n", + "}\n", + "\n", + "\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", + "\n", + "}\n", + "\n", + "\n", + "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", + "\n", + "}\n", + "\n", + "mpl.figure.prototype._init_canvas = function() {\n", + " var fig = this;\n", + "\n", + " var canvas_div = $('
');\n", + "\n", + " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", + "\n", + " function canvas_keyboard_event(event) {\n", + " return fig.key_event(event, event['data']);\n", + " }\n", + "\n", + " canvas_div.keydown('key_press', canvas_keyboard_event);\n", + " canvas_div.keyup('key_release', canvas_keyboard_event);\n", + " this.canvas_div = canvas_div\n", + " this._canvas_extra_style(canvas_div)\n", + " this.root.append(canvas_div);\n", + "\n", + " var canvas = $('');\n", + " canvas.addClass('mpl-canvas');\n", + " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", + "\n", + " this.canvas = canvas[0];\n", + " this.context = canvas[0].getContext(\"2d\");\n", + "\n", + " var backingStore = this.context.backingStorePixelRatio ||\n", + "\tthis.context.webkitBackingStorePixelRatio ||\n", + "\tthis.context.mozBackingStorePixelRatio ||\n", + "\tthis.context.msBackingStorePixelRatio ||\n", + "\tthis.context.oBackingStorePixelRatio ||\n", + "\tthis.context.backingStorePixelRatio || 1;\n", + "\n", + " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n", + "\n", + " var rubberband = $('');\n", + " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", + "\n", + " var pass_mouse_events = true;\n", + "\n", + " canvas_div.resizable({\n", + " start: function(event, ui) {\n", + " pass_mouse_events = false;\n", + " },\n", + " resize: function(event, ui) {\n", + " fig.request_resize(ui.size.width, ui.size.height);\n", + " },\n", + " stop: function(event, ui) {\n", + " pass_mouse_events = true;\n", + " fig.request_resize(ui.size.width, ui.size.height);\n", + " },\n", + " });\n", + "\n", + " function mouse_event_fn(event) {\n", + " if (pass_mouse_events)\n", + " return fig.mouse_event(event, event['data']);\n", + " }\n", + "\n", + " rubberband.mousedown('button_press', mouse_event_fn);\n", + " rubberband.mouseup('button_release', mouse_event_fn);\n", + " // Throttle sequential mouse events to 1 every 20ms.\n", + " rubberband.mousemove('motion_notify', mouse_event_fn);\n", + "\n", + " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", + " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", + "\n", + " canvas_div.on(\"wheel\", function (event) {\n", + " event = event.originalEvent;\n", + " event['data'] = 'scroll'\n", + " if (event.deltaY < 0) {\n", + " event.step = 1;\n", + " } else {\n", + " event.step = -1;\n", + " }\n", + " mouse_event_fn(event);\n", + " });\n", + "\n", + " canvas_div.append(canvas);\n", + " canvas_div.append(rubberband);\n", + "\n", + " this.rubberband = rubberband;\n", + " this.rubberband_canvas = rubberband[0];\n", + " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", + " this.rubberband_context.strokeStyle = \"#000000\";\n", + "\n", + " this._resize_canvas = function(width, height) {\n", + " // Keep the size of the canvas, canvas container, and rubber band\n", + " // canvas in synch.\n", + " canvas_div.css('width', width)\n", + " canvas_div.css('height', height)\n", + "\n", + " canvas.attr('width', width * mpl.ratio);\n", + " canvas.attr('height', height * mpl.ratio);\n", + " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n", + "\n", + " rubberband.attr('width', width);\n", + " rubberband.attr('height', height);\n", + " }\n", + "\n", + " // Set the figure to an initial 600x600px, this will subsequently be updated\n", + " // upon first draw.\n", + " this._resize_canvas(600, 600);\n", + "\n", + " // Disable right mouse context menu.\n", + " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", + " return false;\n", + " });\n", + "\n", + " function set_focus () {\n", + " canvas.focus();\n", + " canvas_div.focus();\n", + " }\n", + "\n", + " window.setTimeout(set_focus, 100);\n", + "}\n", + "\n", + "mpl.figure.prototype._init_toolbar = function() {\n", + " var fig = this;\n", + "\n", + " var nav_element = $('
')\n", + " nav_element.attr('style', 'width: 100%');\n", + " this.root.append(nav_element);\n", + "\n", + " // Define a callback function for later on.\n", + " function toolbar_event(event) {\n", + " return fig.toolbar_button_onclick(event['data']);\n", + " }\n", + " function toolbar_mouse_event(event) {\n", + " return fig.toolbar_button_onmouseover(event['data']);\n", + " }\n", + "\n", + " for(var toolbar_ind in mpl.toolbar_items) {\n", + " var name = mpl.toolbar_items[toolbar_ind][0];\n", + " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", + " var image = mpl.toolbar_items[toolbar_ind][2];\n", + " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", + "\n", + " if (!name) {\n", + " // put a spacer in here.\n", + " continue;\n", + " }\n", + " var button = $('');\n", + " button.click(method_name, toolbar_event);\n", + " button.mouseover(tooltip, toolbar_mouse_event);\n", + " nav_element.append(button);\n", + " }\n", + "\n", + " // Add the status bar.\n", + " var status_bar = $('');\n", + " nav_element.append(status_bar);\n", + " this.message = status_bar[0];\n", + "\n", + " // Add the close button to the window.\n", + " var buttongrp = $('
');\n", + " var button = $('');\n", + " button.click(function (evt) { fig.handle_close(fig, {}); } );\n", + " button.mouseover('Stop Interaction', toolbar_mouse_event);\n", + " buttongrp.append(button);\n", + " var titlebar = this.root.find($('.ui-dialog-titlebar'));\n", + " titlebar.prepend(buttongrp);\n", + "}\n", + "\n", + "mpl.figure.prototype._root_extra_style = function(el){\n", + " var fig = this\n", + " el.on(\"remove\", function(){\n", + "\tfig.close_ws(fig, {});\n", + " });\n", + "}\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function(el){\n", + " // this is important to make the div 'focusable\n", + " el.attr('tabindex', 0)\n", + " // reach out to IPython and tell the keyboard manager to turn it's self\n", + " // off when our div gets focus\n", + "\n", + " // location in version 3\n", + " if (IPython.notebook.keyboard_manager) {\n", + " IPython.notebook.keyboard_manager.register_events(el);\n", + " }\n", + " else {\n", + " // location in version 2\n", + " IPython.keyboard_manager.register_events(el);\n", + " }\n", + "\n", + "}\n", + "\n", + "mpl.figure.prototype._key_event_extra = function(event, name) {\n", + " var manager = IPython.notebook.keyboard_manager;\n", + " if (!manager)\n", + " manager = IPython.keyboard_manager;\n", + "\n", + " // Check for shift+enter\n", + " if (event.shiftKey && event.which == 13) {\n", + " this.canvas_div.blur();\n", + " // select the cell after this one\n", + " var index = IPython.notebook.find_cell_index(this.cell_info[0]);\n", + " IPython.notebook.select(index + 1);\n", + " }\n", + "}\n", + "\n", + "mpl.figure.prototype.handle_save = function(fig, msg) {\n", + " fig.ondownload(fig, null);\n", + "}\n", + "\n", + "\n", + "mpl.find_output_cell = function(html_output) {\n", + " // Return the cell and output element which can be found *uniquely* in the notebook.\n", + " // Note - this is a bit hacky, but it is done because the \"notebook_saving.Notebook\"\n", + " // IPython event is triggered only after the cells have been serialised, which for\n", + " // our purposes (turning an active figure into a static one), is too late.\n", + " var cells = IPython.notebook.get_cells();\n", + " var ncells = cells.length;\n", + " for (var i=0; i= 3 moved mimebundle to data attribute of output\n", + " data = data.data;\n", + " }\n", + " if (data['text/html'] == html_output) {\n", + " return [cell, data, j];\n", + " }\n", + " }\n", + " }\n", + " }\n", + "}\n", + "\n", + "// Register the function which deals with the matplotlib target/channel.\n", + "// The kernel may be null if the page has been refreshed.\n", + "if (IPython.notebook.kernel != null) {\n", + " IPython.notebook.kernel.comm_manager.register_target('matplotlib', mpl.mpl_figure_comm);\n", + "}\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from mlp.layers import AffineLayer\n", "from mlp.errors import SumOfSquaredDiffsError\n", @@ -600,7 +2290,7 @@ "\n", "# Run the optimiser for 5 epochs (full passes through the training set)\n", "# printing statistics every epoch.\n", - "stats, keys = optimiser.train(num_epochs=10, stats_interval=1)\n", + "stats, keys, _ = optimiser.train(num_epochs=10, stats_interval=1)\n", "\n", "# Plot the change in the error over training.\n", "fig = plt.figure(figsize=(8, 4))\n", @@ -619,9 +2309,799 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "application/javascript": [ + "/* Put everything inside the global mpl namespace */\n", + "window.mpl = {};\n", + "\n", + "\n", + "mpl.get_websocket_type = function() {\n", + " if (typeof(WebSocket) !== 'undefined') {\n", + " return WebSocket;\n", + " } else if (typeof(MozWebSocket) !== 'undefined') {\n", + " return MozWebSocket;\n", + " } else {\n", + " alert('Your browser does not have WebSocket support.' +\n", + " 'Please try Chrome, Safari or Firefox ≥ 6. ' +\n", + " 'Firefox 4 and 5 are also supported but you ' +\n", + " 'have to enable WebSockets in about:config.');\n", + " };\n", + "}\n", + "\n", + "mpl.figure = function(figure_id, websocket, ondownload, parent_element) {\n", + " this.id = figure_id;\n", + "\n", + " this.ws = websocket;\n", + "\n", + " this.supports_binary = (this.ws.binaryType != undefined);\n", + "\n", + " if (!this.supports_binary) {\n", + " var warnings = document.getElementById(\"mpl-warnings\");\n", + " if (warnings) {\n", + " warnings.style.display = 'block';\n", + " warnings.textContent = (\n", + " \"This browser does not support binary websocket messages. \" +\n", + " \"Performance may be slow.\");\n", + " }\n", + " }\n", + "\n", + " this.imageObj = new Image();\n", + "\n", + " this.context = undefined;\n", + " this.message = undefined;\n", + " this.canvas = undefined;\n", + " this.rubberband_canvas = undefined;\n", + " this.rubberband_context = undefined;\n", + " this.format_dropdown = undefined;\n", + "\n", + " this.image_mode = 'full';\n", + "\n", + " this.root = $('
');\n", + " this._root_extra_style(this.root)\n", + " this.root.attr('style', 'display: inline-block');\n", + "\n", + " $(parent_element).append(this.root);\n", + "\n", + " this._init_header(this);\n", + " this._init_canvas(this);\n", + " this._init_toolbar(this);\n", + "\n", + " var fig = this;\n", + "\n", + " this.waiting = false;\n", + "\n", + " this.ws.onopen = function () {\n", + " fig.send_message(\"supports_binary\", {value: fig.supports_binary});\n", + " fig.send_message(\"send_image_mode\", {});\n", + " if (mpl.ratio != 1) {\n", + " fig.send_message(\"set_dpi_ratio\", {'dpi_ratio': mpl.ratio});\n", + " }\n", + " fig.send_message(\"refresh\", {});\n", + " }\n", + "\n", + " this.imageObj.onload = function() {\n", + " if (fig.image_mode == 'full') {\n", + " // Full images could contain transparency (where diff images\n", + " // almost always do), so we need to clear the canvas so that\n", + " // there is no ghosting.\n", + " fig.context.clearRect(0, 0, fig.canvas.width, fig.canvas.height);\n", + " }\n", + " fig.context.drawImage(fig.imageObj, 0, 0);\n", + " };\n", + "\n", + " this.imageObj.onunload = function() {\n", + " this.ws.close();\n", + " }\n", + "\n", + " this.ws.onmessage = this._make_on_message_function(this);\n", + "\n", + " this.ondownload = ondownload;\n", + "}\n", + "\n", + "mpl.figure.prototype._init_header = function() {\n", + " var titlebar = $(\n", + " '
');\n", + " var titletext = $(\n", + " '
');\n", + " titlebar.append(titletext)\n", + " this.root.append(titlebar);\n", + " this.header = titletext[0];\n", + "}\n", + "\n", + "\n", + "\n", + "mpl.figure.prototype._canvas_extra_style = function(canvas_div) {\n", + "\n", + "}\n", + "\n", + "\n", + "mpl.figure.prototype._root_extra_style = function(canvas_div) {\n", + "\n", + "}\n", + "\n", + "mpl.figure.prototype._init_canvas = function() {\n", + " var fig = this;\n", + "\n", + " var canvas_div = $('
');\n", + "\n", + " canvas_div.attr('style', 'position: relative; clear: both; outline: 0');\n", + "\n", + " function canvas_keyboard_event(event) {\n", + " return fig.key_event(event, event['data']);\n", + " }\n", + "\n", + " canvas_div.keydown('key_press', canvas_keyboard_event);\n", + " canvas_div.keyup('key_release', canvas_keyboard_event);\n", + " this.canvas_div = canvas_div\n", + " this._canvas_extra_style(canvas_div)\n", + " this.root.append(canvas_div);\n", + "\n", + " var canvas = $('');\n", + " canvas.addClass('mpl-canvas');\n", + " canvas.attr('style', \"left: 0; top: 0; z-index: 0; outline: 0\")\n", + "\n", + " this.canvas = canvas[0];\n", + " this.context = canvas[0].getContext(\"2d\");\n", + "\n", + " var backingStore = this.context.backingStorePixelRatio ||\n", + "\tthis.context.webkitBackingStorePixelRatio ||\n", + "\tthis.context.mozBackingStorePixelRatio ||\n", + "\tthis.context.msBackingStorePixelRatio ||\n", + "\tthis.context.oBackingStorePixelRatio ||\n", + "\tthis.context.backingStorePixelRatio || 1;\n", + "\n", + " mpl.ratio = (window.devicePixelRatio || 1) / backingStore;\n", + "\n", + " var rubberband = $('');\n", + " rubberband.attr('style', \"position: absolute; left: 0; top: 0; z-index: 1;\")\n", + "\n", + " var pass_mouse_events = true;\n", + "\n", + " canvas_div.resizable({\n", + " start: function(event, ui) {\n", + " pass_mouse_events = false;\n", + " },\n", + " resize: function(event, ui) {\n", + " fig.request_resize(ui.size.width, ui.size.height);\n", + " },\n", + " stop: function(event, ui) {\n", + " pass_mouse_events = true;\n", + " fig.request_resize(ui.size.width, ui.size.height);\n", + " },\n", + " });\n", + "\n", + " function mouse_event_fn(event) {\n", + " if (pass_mouse_events)\n", + " return fig.mouse_event(event, event['data']);\n", + " }\n", + "\n", + " rubberband.mousedown('button_press', mouse_event_fn);\n", + " rubberband.mouseup('button_release', mouse_event_fn);\n", + " // Throttle sequential mouse events to 1 every 20ms.\n", + " rubberband.mousemove('motion_notify', mouse_event_fn);\n", + "\n", + " rubberband.mouseenter('figure_enter', mouse_event_fn);\n", + " rubberband.mouseleave('figure_leave', mouse_event_fn);\n", + "\n", + " canvas_div.on(\"wheel\", function (event) {\n", + " event = event.originalEvent;\n", + " event['data'] = 'scroll'\n", + " if (event.deltaY < 0) {\n", + " event.step = 1;\n", + " } else {\n", + " event.step = -1;\n", + " }\n", + " mouse_event_fn(event);\n", + " });\n", + "\n", + " canvas_div.append(canvas);\n", + " canvas_div.append(rubberband);\n", + "\n", + " this.rubberband = rubberband;\n", + " this.rubberband_canvas = rubberband[0];\n", + " this.rubberband_context = rubberband[0].getContext(\"2d\");\n", + " this.rubberband_context.strokeStyle = \"#000000\";\n", + "\n", + " this._resize_canvas = function(width, height) {\n", + " // Keep the size of the canvas, canvas container, and rubber band\n", + " // canvas in synch.\n", + " canvas_div.css('width', width)\n", + " canvas_div.css('height', height)\n", + "\n", + " canvas.attr('width', width * mpl.ratio);\n", + " canvas.attr('height', height * mpl.ratio);\n", + " canvas.attr('style', 'width: ' + width + 'px; height: ' + height + 'px;');\n", + "\n", + " rubberband.attr('width', width);\n", + " rubberband.attr('height', height);\n", + " }\n", + "\n", + " // Set the figure to an initial 600x600px, this will subsequently be updated\n", + " // upon first draw.\n", + " this._resize_canvas(600, 600);\n", + "\n", + " // Disable right mouse context menu.\n", + " $(this.rubberband_canvas).bind(\"contextmenu\",function(e){\n", + " return false;\n", + " });\n", + "\n", + " function set_focus () {\n", + " canvas.focus();\n", + " canvas_div.focus();\n", + " }\n", + "\n", + " window.setTimeout(set_focus, 100);\n", + "}\n", + "\n", + "mpl.figure.prototype._init_toolbar = function() {\n", + " var fig = this;\n", + "\n", + " var nav_element = $('
')\n", + " nav_element.attr('style', 'width: 100%');\n", + " this.root.append(nav_element);\n", + "\n", + " // Define a callback function for later on.\n", + " function toolbar_event(event) {\n", + " return fig.toolbar_button_onclick(event['data']);\n", + " }\n", + " function toolbar_mouse_event(event) {\n", + " return fig.toolbar_button_onmouseover(event['data']);\n", + " }\n", + "\n", + " for(var toolbar_ind in mpl.toolbar_items) {\n", + " var name = mpl.toolbar_items[toolbar_ind][0];\n", + " var tooltip = mpl.toolbar_items[toolbar_ind][1];\n", + " var image = mpl.toolbar_items[toolbar_ind][2];\n", + " var method_name = mpl.toolbar_items[toolbar_ind][3];\n", + "\n", + " if (!name) {\n", + " // put a spacer in here.\n", + " continue;\n", + " }\n", + " var button = $('