Adding regularisation penalties module.

mlp/penalties.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+"""Parameter penalties.
+
+This module defines parameter penalty functions which can be used to regularise
+training by adding an additional term to the objective function being
+minimised which aims to restrict 'model complexity' by some measure.
+"""
+
+import numpy as np
+
+
+class L1Penalty(object):
+    """L1 parameter penalty.
+
+    Term to add to the objective function penalising parameters
+    based on their L1 norm.
+    """
+
+    def __init__(self, coefficient):
+        """Create a new L1 penalty object.
+
+        Args:
+            coefficient: Positive constant to scale penalty term by.
+        """
+        assert coefficient > 0., 'Penalty coefficient must be positive.'
+        self.coefficient = coefficient
+
+    def __call__(self, parameter):
+        """Calculate L1 penalty value for a parameter.
+
+        Args:
+            parameter: Array corresponding to a model parameter.
+
+        Returns:
+            Value of penalty term.
+        """
+        return self.coefficient * abs(parameter).sum()
+
+    def grad(self, parameter):
+        """Calculate the penalty gradient with respect to the parameter.
+
+        Args:
+            parameter: Array corresponding to a model parameter.
+
+        Returns:
+            Value of penalty gradient with respect to parameter. This
+            should be an array of the same shape as the parameter.
+        """
+        return self.coefficient * np.sign(parameter)
+
+    def __repr__(self):
+        return 'L1Penalty({0})'.format(self.coefficient)
+
+
+class L2Penalty(object):
+    """L1 parameter penalty.
+
+    Term to add to the objective function penalising parameters
+    based on their L2 norm.
+    """
+
+    def __init__(self, coefficient):
+        """Create a new L2 penalty object.
+
+        Args:
+            coefficient: Positive constant to scale penalty term by.
+        """
+        assert coefficient > 0., 'Penalty coefficient must be positive.'
+        self.coefficient = coefficient
+
+    def __call__(self, parameter):
+        """Calculate L2 penalty value for a parameter.
+
+        Args:
+            parameter: Array corresponding to a model parameter.
+
+        Returns:
+            Value of penalty term.
+        """
+        return 0.5 * self.coefficient * (parameter**2).sum()
+
+    def grad(self, parameter):
+        """Calculate the penalty gradient with respect to the parameter.
+
+        Args:
+            parameter: Array corresponding to a model parameter.
+
+        Returns:
+            Value of penalty gradient with respect to parameter. This
+            should be an array of the same shape as the parameter.
+        """
+        return self.coefficient * parameter
+
+    def __repr__(self):
+        return 'L2Penalty({0})'.format(self.coefficient)