Added testing version of optimised RMSProp from Atari paper

Author: yobibyte

DeepFried2/optimizers/DQNProp.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+import DeepFried2 as df
+
+
+class DQNProp(df.Optimizer):
+    """
+    RMSProp as described here on page 23:
+    http://arxiv.org/pdf/1308.0850v5.pdf
+    
+    Also used by DeepMind here:
+    https://sites.google.com/a/deepmind.com/dqn/
+    In NeuralQLearner.lua
+
+    The updates are:
+
+        g_{e+1} = ρ * g_e + (1-ρ) * ∇p_e
+        g²_{e+1} = ρ * g²_e + (1-ρ) * ∇p_e²
+        p_{e+1} = p_e - lr * ∇p_e / √(g²_{e+1} - g_{e+1}²)
+
+    This roughly corresponds to dividing the gradients by their standard deviation
+    over the past batches, in a rolling-momentum fashion.
+    The more "unstable" a gradient, the lower its effective learning-rate.
+    
+    """
+
+    def __init__(self, lr, rho, eps=1e-7):
+        df.Optimizer.__init__(self, lr=lr, rho=rho, eps=eps)
+
+    def get_updates(self, params, grads, lr, rho, eps):
+        updates = []
+
+        for param, grad in zip(params, grads):
+            g_state = df.utils.create_param_state_as(param)
+            new_g = rho*g_state + (1-rho)*grad
+            g2_state = df.utils.create_param_state_as(param)
+            new_g2 = rho*g2_state+(1-rho)*grad*grad
+            updates.append((g_state, new_g))
+            updates.append((g2_state, new_g2))
+            updates.append((param, param - lr*(grad/df.T.sqrt(new_g2-new_g*new_g+eps))))
+
+        return updates

DeepFried2/optimizers/__init__.py

@@ -3,5 +3,6 @@
 from .Nesterov import Nesterov
 from .AdaGrad import AdaGrad
 from .RMSProp import RMSProp
+from .DQNProp import DQNProp
 from .AdaDelta import AdaDelta
 from .Adam import Adam

examples/Optimizers/run.py

@@ -43,5 +43,6 @@ def run(optim):
     run(df.Nesterov(lr=1e-2, momentum=0.90))
     run(df.AdaGrad(lr=1e-2, eps=1e-4))
     run(df.RMSProp(lr=1e-3, rho=0.90, eps=1e-5))
+    run(df.DQNProp(lr=1e-3, rho=0.90, eps=1e-5))
     run(df.AdaDelta(rho=0.99, lr=5e-1, eps=1e-4))
     run(df.Adam(alpha=1e-3, beta1=0.95, beta2=0.9, eps=1e-8))