AMSGrad optimizer added - Issue #583 (#636)

* AMSGrad and related unittest added. Missing Documentation.

* YAPF error correct

* PR number added

* Codacy errors fix

* TL documentation updated

* Documentation Error Fix

* Changelog Updated

Author: DEKHTIARJonathan

CHANGELOG.md

@@ -77,13 +77,18 @@ To release a new version, please update the changelog as followed:
   - `test_utils_predict.py` added to reproduce and fix issue #288 (by @2wins in #566)
   - `Layer_DeformableConvolution_Test` added to reproduce issue #572 with deformable convolution (by @DEKHTIARJonathan in #573)
   - `Array_Op_Alphas_Test` and `Array_Op_Alphas_Like_Test` added to test `tensorlayer/array_ops.py` file (by @DEKHTIARJonathan in #580)
+  - `test_optimizer_amsgrad.py` added to test `AMSGrad` optimizer (by @DEKHTIARJonathan in #636)
 - CI Tool:
   - Danger CI has been added to enforce the update of the changelog (by @lgarithm and @DEKHTIARJonathan in #563)
   - https://github.com/apps/stale/ added to clean stale issues (by @DEKHTIARJonathan in #573)
 - Layer:
   - ElementwiseLambdaLayer added to use custom function to connect multiple layer inputs (by @One-sixth in #579)
 - Documentation:
   - Release semantic version added on index page (by @DEKHTIARJonathan in #633)
+  - Optimizers page added (by @DEKHTIARJonathan in #636)
+  - `AMSGrad` added on Optimizers page added (by @DEKHTIARJonathan in #636)
+- Optimizer:
+  - AMSGrad Optimizer added based on `On the Convergence of Adam and Beyond (ICLR 2018)` (by @DEKHTIARJonathan in #636)
 
 ### Changed
 - Tensorflow CPU & GPU dependencies moved to separated requirement files in order to allow PyUP.io to parse them (by @DEKHTIARJonathan in #573)

docs/index.rst

@@ -53,6 +53,7 @@ method, this part of the documentation is for you.
   modules/models
   modules/nlp
   modules/layers
+  modules/optimizers
   modules/prepro
   modules/rein
   modules/utils

docs/modules/optimizers.rst

@@ -0,0 +1,19 @@
+API - Optimizers
+================
+
+.. automodule:: tensorlayer.optimizers
+
+TensorLayer provides simple API and tools to ease research, development and reduce the time to production.
+Therefore, we provide the latest state of the art optimizers that work with Tensorflow.
+
+Optimizers List
+---------------
+
+.. autosummary::
+
+   AMSGrad
+
+AMSGrad Optimizer
+-----------------
+.. autoclass:: AMSGrad
+  :members:

tensorlayer/__init__.py

@@ -7,23 +7,28 @@
     import tensorflow
 
     from . import activation
-    from .array_ops import alphas, alphas_like
+    from . import array_ops
     from . import cost
+    from . import distributed
     from . import files
     from . import iterate
     from . import layers
     from . import models
-    from . import utils
-    from . import visualize
-    from . import prepro
     from . import nlp
+    from . import optimizers
+    from . import prepro
     from . import rein
-    from . import distributed
+    from . import utils
+    from . import visualize
 
     # alias
     act = activation
     vis = visualize
 
+    alphas = array_ops.alphas
+    alphas_like = array_ops.alphas_like
+
+    # global vars
     global_flag = {}
     global_dict = {}
 

tensorlayer/optimizers/__init__.py

@@ -0,0 +1,10 @@
+"""
+TensorLayer provides rich layer implementations trailed for
+various benchmarks and domain-specific problems. In addition, we also
+support transparent access to native TensorFlow parameters.
+For example, we provide not only layers for local response normalization, but also
+layers that allow user to apply ``tf.nn.lrn`` on ``network.outputs``.
+More functions can be found in `TensorFlow API <https://www.tensorflow.org/versions/master/api_docs/index.html>`__.
+"""
+
+from .amsgrad import AMSGrad

tensorlayer/optimizers/amsgrad.py

@@ -0,0 +1,197 @@
+"""AMSGrad Implementation based on the paper: "On the Convergence of Adam and Beyond" (ICLR 2018)
+Article Link: https://openreview.net/pdf?id=ryQu7f-RZ
+Original Implementation by: https://github.com/taki0112/AMSGrad-Tensorflow
+"""
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import optimizer
+
+
+class AMSGrad(optimizer.Optimizer):
+    """Implementation of the AMSGrad optimization algorithm.\n
+    See: `On the Convergence of Adam and Beyond - [Reddi et al., 2018] <https://openreview.net/pdf?id=ryQu7f-RZ>`__.
+
+    Parameters
+    ----------
+    learning_rate: float
+        A Tensor or a floating point value.  The learning rate.
+    beta1: float
+        A float value or a constant float tensor.
+        The exponential decay rate for the 1st moment estimates.
+    beta2: float
+        A float value or a constant float tensor.
+        The exponential decay rate for the 2nd moment estimates.
+    epsilon: float
+        A small constant for numerical stability.
+        This epsilon is "epsilon hat" in the Kingma and Ba paper
+        (in the formula just before Section 2.1), not the epsilon in Algorithm 1 of the paper.
+    use_locking: bool
+        If True use locks for update operations.
+    name: str
+        Optional name for the operations created when applying gradients.
+        Defaults to "AMSGrad".
+    """
+
+    def __init__(self, learning_rate=0.01, beta1=0.9, beta2=0.99, epsilon=1e-8, use_locking=False, name="AMSGrad"):
+        """Construct a new Adam optimizer.
+        """
+        super(AMSGrad, self).__init__(use_locking, name)
+        self._lr = learning_rate
+        self._beta1 = beta1
+        self._beta2 = beta2
+        self._epsilon = epsilon
+
+        self._lr_t = None
+        self._beta1_t = None
+        self._beta2_t = None
+        self._epsilon_t = None
+
+        self._beta1_power = None
+        self._beta2_power = None
+
+    def _create_slots(self, var_list):
+        first_var = min(var_list, key=lambda x: x.name)
+
+        create_new = self._beta1_power is None
+        if not create_new and context.in_graph_mode():
+            create_new = (self._beta1_power.graph is not first_var.graph)
+
+        if create_new:
+            with ops.colocate_with(first_var):
+                self._beta1_power = variable_scope.variable(self._beta1, name="beta1_power", trainable=False)
+                self._beta2_power = variable_scope.variable(self._beta2, name="beta2_power", trainable=False)
+        # Create slots for the first and second moments.
+        for v in var_list:
+            self._zeros_slot(v, "m", self._name)
+            self._zeros_slot(v, "v", self._name)
+            self._zeros_slot(v, "vhat", self._name)
+
+    def _prepare(self):
+        self._lr_t = ops.convert_to_tensor(self._lr)
+        self._beta1_t = ops.convert_to_tensor(self._beta1)
+        self._beta2_t = ops.convert_to_tensor(self._beta2)
+        self._epsilon_t = ops.convert_to_tensor(self._epsilon)
+
+    def _apply_dense(self, grad, var):
+        beta1_power = math_ops.cast(self._beta1_power, var.dtype.base_dtype)
+        beta2_power = math_ops.cast(self._beta2_power, var.dtype.base_dtype)
+        lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype)
+        beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype)
+        beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype)
+        epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype)
+
+        lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
+
+        # m_t = beta1 * m + (1 - beta1) * g_t
+        m = self.get_slot(var, "m")
+        m_scaled_g_values = grad * (1 - beta1_t)
+        m_t = state_ops.assign(m, beta1_t * m + m_scaled_g_values, use_locking=self._use_locking)
+
+        # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
+        v = self.get_slot(var, "v")
+        v_scaled_g_values = (grad * grad) * (1 - beta2_t)
+        v_t = state_ops.assign(v, beta2_t * v + v_scaled_g_values, use_locking=self._use_locking)
+
+        # amsgrad
+        vhat = self.get_slot(var, "vhat")
+        vhat_t = state_ops.assign(vhat, math_ops.maximum(v_t, vhat))
+        v_sqrt = math_ops.sqrt(vhat_t)
+
+        var_update = state_ops.assign_sub(var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking)
+        return control_flow_ops.group(*[var_update, m_t, v_t, vhat_t])
+
+    def _resource_apply_dense(self, grad, var):
+        var = var.handle
+        beta1_power = math_ops.cast(self._beta1_power, grad.dtype.base_dtype)
+        beta2_power = math_ops.cast(self._beta2_power, grad.dtype.base_dtype)
+        lr_t = math_ops.cast(self._lr_t, grad.dtype.base_dtype)
+        beta1_t = math_ops.cast(self._beta1_t, grad.dtype.base_dtype)
+        beta2_t = math_ops.cast(self._beta2_t, grad.dtype.base_dtype)
+        epsilon_t = math_ops.cast(self._epsilon_t, grad.dtype.base_dtype)
+
+        lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
+
+        # m_t = beta1 * m + (1 - beta1) * g_t
+        m = self.get_slot(var, "m").handle
+        m_scaled_g_values = grad * (1 - beta1_t)
+        m_t = state_ops.assign(m, beta1_t * m + m_scaled_g_values, use_locking=self._use_locking)
+
+        # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
+        v = self.get_slot(var, "v").handle
+        v_scaled_g_values = (grad * grad) * (1 - beta2_t)
+        v_t = state_ops.assign(v, beta2_t * v + v_scaled_g_values, use_locking=self._use_locking)
+
+        # amsgrad
+        vhat = self.get_slot(var, "vhat").handle
+        vhat_t = state_ops.assign(vhat, math_ops.maximum(v_t, vhat))
+        v_sqrt = math_ops.sqrt(vhat_t)
+
+        var_update = state_ops.assign_sub(var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking)
+        return control_flow_ops.group(*[var_update, m_t, v_t, vhat_t])
+
+    def _apply_sparse_shared(self, grad, var, indices, scatter_add):
+        beta1_power = math_ops.cast(self._beta1_power, var.dtype.base_dtype)
+        beta2_power = math_ops.cast(self._beta2_power, var.dtype.base_dtype)
+        lr_t = math_ops.cast(self._lr_t, var.dtype.base_dtype)
+        beta1_t = math_ops.cast(self._beta1_t, var.dtype.base_dtype)
+        beta2_t = math_ops.cast(self._beta2_t, var.dtype.base_dtype)
+        epsilon_t = math_ops.cast(self._epsilon_t, var.dtype.base_dtype)
+
+        lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
+
+        # m_t = beta1 * m + (1 - beta1) * g_t
+        m = self.get_slot(var, "m")
+        m_scaled_g_values = grad * (1 - beta1_t)
+        m_t = state_ops.assign(m, m * beta1_t, use_locking=self._use_locking)
+        with ops.control_dependencies([m_t]):
+            m_t = scatter_add(m, indices, m_scaled_g_values)
+
+        # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
+        v = self.get_slot(var, "v")
+        v_scaled_g_values = (grad * grad) * (1 - beta2_t)
+        v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking)
+        with ops.control_dependencies([v_t]):
+            v_t = scatter_add(v, indices, v_scaled_g_values)
+
+        # amsgrad
+        vhat = self.get_slot(var, "vhat")
+        vhat_t = state_ops.assign(vhat, math_ops.maximum(v_t, vhat))
+        v_sqrt = math_ops.sqrt(vhat_t)
+        var_update = state_ops.assign_sub(var, lr * m_t / (v_sqrt + epsilon_t), use_locking=self._use_locking)
+        return control_flow_ops.group(*[var_update, m_t, v_t, vhat_t])
+
+    def _apply_sparse(self, grad, var):
+        return self._apply_sparse_shared(
+            grad.values,
+            var,
+            grad.indices,
+            lambda x, i, v: state_ops.
+            scatter_add(  # pylint: disable=g-long-lambda
+                x, i, v, use_locking=self._use_locking
+            )
+        )
+
+    def _resource_scatter_add(self, x, i, v):
+        with ops.control_dependencies([resource_variable_ops.resource_scatter_add(x.handle, i, v)]):
+            return x.value()
+
+    def _resource_apply_sparse(self, grad, var, indices):
+        return self._apply_sparse_shared(grad, var, indices, self._resource_scatter_add)
+
+    def _finish(self, update_ops, name_scope):
+        # Update the power accumulators.
+        with ops.control_dependencies(update_ops):
+            with ops.colocate_with(self._beta1_power):
+                update_beta1 = self._beta1_power.assign(
+                    self._beta1_power * self._beta1_t, use_locking=self._use_locking
+                )
+                update_beta2 = self._beta2_power.assign(
+                    self._beta2_power * self._beta2_t, use_locking=self._use_locking
+                )
+        return control_flow_ops.group(*update_ops + [update_beta1, update_beta2], name=name_scope)

tests/test_optimizer_amsgrad.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import unittest
+
+import tensorflow as tf
+import tensorlayer as tl
+
+try:
+    from tests.unittests_helper import CustomTestCase
+except ImportError:
+    from unittests_helper import CustomTestCase
+
+
+class Layer_Pooling_Test(CustomTestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.x = tf.placeholder(tf.float32, shape=[None, 784], name='x')
+        cls.y_ = tf.placeholder(tf.int64, shape=[None], name='y_')
+
+        # define the network
+        cls.network = tl.layers.InputLayer(cls.x, name='input')
+        cls.network = tl.layers.DropoutLayer(cls.network, keep=0.8, name='drop1')
+        cls.network = tl.layers.DenseLayer(cls.network, 800, tf.nn.relu, name='relu1')
+        cls.network = tl.layers.DropoutLayer(cls.network, keep=0.5, name='drop2')
+        cls.network = tl.layers.DenseLayer(cls.network, 800, tf.nn.relu, name='relu2')
+        cls.network = tl.layers.DropoutLayer(cls.network, keep=0.5, name='drop3')
+
+        cls.network = tl.layers.DenseLayer(cls.network, n_units=10, act=tf.identity, name='output')
+
+        # define cost function and metric.
+        cls.y = cls.network.outputs
+        cls.cost = tl.cost.cross_entropy(cls.y, cls.y_, name='cost')
+
+        correct_prediction = tf.equal(tf.argmax(cls.y, 1), cls.y_)
+
+        cls.acc = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+
+        # define the optimizer
+        train_params = cls.network.all_params
+        optimizer = tl.optimizers.AMSGrad(learning_rate=1e-4, beta1=0.9, beta2=0.999, epsilon=1e-8)
+        cls.train_op = optimizer.minimize(cls.cost, var_list=train_params)
+
+    @classmethod
+    def tearDownClass(cls):
+        tf.reset_default_graph()
+
+    def test_training(self):
+
+        with self.assertNotRaises(Exception):
+
+            X_train, y_train, X_val, y_val, _, _ = tl.files.load_mnist_dataset(shape=(-1, 784))
+
+            with tf.Session() as sess:
+                # initialize all variables in the session
+                tl.layers.initialize_global_variables(sess)
+
+                # print network information
+                self.network.print_params()
+                self.network.print_layers()
+
+                # train the network
+                tl.utils.fit(
+                    sess, self.network, self.train_op, self.cost, X_train, y_train, self.x, self.y_, acc=self.acc,
+                    batch_size=500, n_epoch=2, print_freq=1, X_val=X_val, y_val=y_val, eval_train=False
+                )
+
+
+if __name__ == '__main__':
+
+    # tf.logging.set_verbosity(tf.logging.INFO)
+    tf.logging.set_verbosity(tf.logging.DEBUG)
+
+    unittest.main()