From 02e29ddcc6670bd64fb83dd437c1df23334770a7 Mon Sep 17 00:00:00 2001
From: Matt Graham <m.m.graham@ed.ac.uk>
Date: Thu, 3 Nov 2016 18:31:59 +0000
Subject: [PATCH] Adding time-dependent learning rule schedulers.

---
 mlp/schedulers.py | 99 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 99 insertions(+)

diff --git a/mlp/schedulers.py b/mlp/schedulers.py
index 1488f44..b83aab2 100644
--- a/mlp/schedulers.py
+++ b/mlp/schedulers.py
@@ -32,3 +32,102 @@ class ConstantLearningRateScheduler(object):
             epoch_number: Integer index of training epoch about to be run.
         """
         learning_rule.learning_rate = self.learning_rate
+
+
+class ExponentialLearningRateScheduler(object):
+    """Exponential decay learning rate scheduler."""
+
+    def __init__(self, init_learning_rate, decay_param):
+        """Construct a new learning rate scheduler object.
+
+        Args:
+            init_learning_rate: Initial learning rate at epoch 0. Should be a
+                positive value.
+            decay_param: Parameter governing rate of learning rate decay.
+                Should be a positive value.
+        """
+        self.init_learning_rate = init_learning_rate
+        self.decay_param = decay_param
+
+    def update_learning_rule(self, learning_rule, epoch_number):
+        """Update the hyperparameters of the learning rule.
+
+        Runs at the beginning of each epoch.
+
+        Args:
+            learning_rule: Learning rule object being used in training run,
+                any scheduled hyperparameters to be altered should be
+                attributes of this object.
+            epoch_number: Integer index of training epoch about to be run.
+        """
+        learning_rule.learning_rate = (
+            self.init_learning_rate * np.exp(-epoch_number / self.decay_param))
+
+
+class ReciprocalLearningRateScheduler(object):
+    """Reciprocal decay learning rate scheduler."""
+
+    def __init__(self, init_learning_rate, decay_param):
+        """Construct a new learning rate scheduler object.
+
+        Args:
+            init_learning_rate: Initial learning rate at epoch 0. Should be a
+                positive value.
+            decay_param: Parameter governing rate of learning rate decay.
+                Should be a positive value.
+        """
+        self.init_learning_rate = init_learning_rate
+        self.decay_param = decay_param
+
+    def update_learning_rule(self, learning_rule, epoch_number):
+        """Update the hyperparameters of the learning rule.
+
+        Runs at the beginning of each epoch.
+
+        Args:
+            learning_rule: Learning rule object being used in training run,
+                any scheduled hyperparameters to be altered should be
+                attributes of this object.
+            epoch_number: Integer index of training epoch about to be run.
+        """
+        learning_rule.learning_rate = (
+            self.init_learning_rate / (1. + epoch_number / self.decay_param)
+        )
+
+
+class ReciprocalMomentumCoefficientScheduler(object):
+    """Reciprocal growth momentum coefficient scheduler."""
+
+    def __init__(self, max_mom_coeff=0.99, growth_param=3., epoch_offset=5.):
+        """Construct a new reciprocal momentum coefficient scheduler object.
+
+        Args:
+            max_mom_coeff: Maximum momentum coefficient to tend to. Should be
+                in [0, 1].
+            growth_param: Parameter governing rate of increase of momentum
+                coefficient over training. Should be >= 0 and <= epoch_offset.
+            epoch_offset: Offset to epoch counter to in scheduler updates to
+                govern how quickly momentum initially increases. Should be
+                >= 1.
+        """
+        assert max_mom_coeff >= 0. and max_mom_coeff <= 1.
+        assert growth_param >= 0. and growth_param <= epoch_offset
+        assert epoch_offset >= 1.
+        self.max_mom_coeff = max_mom_coeff
+        self.growth_param = growth_param
+        self.epoch_offset = epoch_offset
+
+    def update_learning_rule(self, learning_rule, epoch_number):
+        """Update the hyperparameters of the learning rule.
+
+        Runs at the beginning of each epoch.
+
+        Args:
+            learning_rule: Learning rule object being used in training run,
+                any scheduled hyperparameters to be altered should be
+                attributes of this object.
+            epoch_number: Integer index of training epoch about to be run.
+        """
+        learning_rule.mom_coeff = self.max_mom_coeff * (
+            1. - self.growth_param / (epoch_number + self.epoch_offset)
+        )