[WIP] Implement step_size_decay (#2434)

* Revert "Remove unnecessary parameter"

This reverts commit 3537b14.

* add docstring for step_size_decay

Author: Junpeng Lao

pymc3/step_methods/sgmcmc.py

@@ -73,6 +73,9 @@ class BaseStochasticGradient(ArrayStepShared):
         Total size of the training data
     step_size : float
         Step size for the parameter update
+    step_size_decay : int
+        Step size decay rate. Every `step_size_decay` iteration the step size reduce 
+        to the half of the previous step size
     model : PyMC Model
         Optional model for sampling step. Defaults to None (taken from context)
     random_seed : int
@@ -98,6 +101,7 @@ def __init__(self,
                  batch_size=None,
                  total_size=None,
                  step_size=1.0,
+                 step_size_decay=100,
                  model=None,
                  random_seed=None,
                  minibatches=None,
@@ -129,6 +133,7 @@ def __init__(self,
             self.random = tt_rng(random_seed)
 
         self.step_size = step_size
+        self.step_size_decay = step_size_decay
         shared = make_shared_replacements(vars, model)
         self.q_size = int(sum(v.dsize for v in self.vars))
 
@@ -237,7 +242,7 @@ def mk_training_fn(self):
         avg_I = self.avg_I
         t = self.t
         updates = self.updates
-        step_size = self.step_size
+        epsilon = self.step_size / pow(2.0, t // self.step_size_decay)
         random = self.random
         inarray = self.inarray
         gt, dlog_prior = self.dlogp_elemwise, self.dlog_prior
@@ -268,11 +273,11 @@ def mk_training_fn(self):
         # where B_ch is cholesky decomposition of B
         # i.e. B = dot(B_ch, B_ch^T)
         B_ch = tt.slinalg.cholesky(B)
-        noise_term = tt.dot((2.*B_ch)/tt.sqrt(step_size), \
+        noise_term = tt.dot((2.*B_ch)/tt.sqrt(epsilon), \
                 random.normal((q_size,), dtype=theano.config.floatX))
         # 9.
         # Inv. Fisher Cov. Matrix
-        cov_mat = (gamma * I_t * N) + ((4. / step_size) * B)
+        cov_mat = (gamma * I_t * N) + ((4. / epsilon) * B)
         inv_cov_mat = tt.nlinalg.matrix_inverse(cov_mat)
         # Noise Coefficient
         noise_coeff = (dlog_prior + (N * avg_gt) + noise_term)