Numerically stable softmax

2017-11-13 23:52:39 +00:00 · 2017-11-13 23:52:39 +00:00 · b9147c269c
commit b9147c269c
parent 6a0cdbea3a
1 changed files with 3 additions and 3 deletions
--- a/mlp/errors.py
+++ b/mlp/errors.py
@ -154,9 +154,9 @@ class CrossEntropySoftmaxError(object):
        Returns:
            Scalar error function value.
        """
-        probs = np.exp(outputs - outputs.max(-1)[:, None])
-        probs /= probs.sum(-1)[:, None]
-        return -np.mean(np.sum(targets * np.log(probs), axis=1))
+        normOutputs = outputs - outputs.max(-1)[:, None]
+        logProb = normOutputs - np.log(np.sum(np.exp(normOutputs))(-1)[:, None])
+        return -np.mean(np.sum(targets * logProb, axis=1))

    def grad(self, outputs, targets):
        """Calculates gradient of error function with respect to outputs.