Applying softmax stability fix to cost gradient.

2016-10-20 13:23:19 +01:00 · 2016-10-20 13:23:19 +01:00 · a021ce585b
commit a021ce585b
parent b4b00e0c65
1 changed files with 3 additions and 1 deletions
--- a/mlp/errors.py
+++ b/mlp/errors.py
@ -170,7 +170,9 @@ class CrossEntropySoftmaxError(object):
        Returns:
            Gradient of error function with respect to outputs.
        """
-        probs = np.exp(outputs)
+        # subtract max inside exponential to improve numerical stability -
+        # when we divide through by sum this term cancels
+        probs = np.exp(outputs - outputs.max(-1)[:, None])
        probs /= probs.sum(-1)[:, None]
        return (probs - targets) / outputs.shape[0]