From a021ce585bbd344f0be5f5a379f292be1b708220 Mon Sep 17 00:00:00 2001
From: Matt Graham <m.m.graham@ed.ac.uk>
Date: Thu, 20 Oct 2016 13:23:19 +0100
Subject: [PATCH] Applying softmax stability fix to cost gradient.

---
 mlp/errors.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/mlp/errors.py b/mlp/errors.py
index 8412d4c..a57decc 100644
--- a/mlp/errors.py
+++ b/mlp/errors.py
@@ -170,7 +170,9 @@ class CrossEntropySoftmaxError(object):
         Returns:
             Gradient of error function with respect to outputs.
         """
-        probs = np.exp(outputs)
+        # subtract max inside exponential to improve numerical stability -
+        # when we divide through by sum this term cancels
+        probs = np.exp(outputs - outputs.max(-1)[:, None])
         probs /= probs.sum(-1)[:, None]
         return (probs - targets) / outputs.shape[0]