update init and docs

Author: hanjr92

docs/conf.py

@@ -51,8 +51,8 @@
     'sphinx.ext.napoleon',
     'sphinx.ext.todo',
     'sphinx.ext.viewcode',
+    'sphinx-mathjax-offline',
 ]
-
 autodoc_mock_imports = [
     'cv2',
     'gridfs',

docs/modules/initializers.rst

@@ -16,6 +16,7 @@ For more complex activation, TensorFlow(MindSpore, PaddlePaddle, PyTorch) API wi
    RandomNormal
    TruncatedNormal
    HeNormal
+   HeUniform
    deconv2d_bilinear_upsampling_initializer
    XavierNormal
    XavierUniform
@@ -52,6 +53,10 @@ HeNormal
 ------------
 .. autoclass:: HeNormal
 
+HeUniform
+------------
+.. autoclass:: HeUniform
+
 deconv2d_bilinear_upsampling_initializer
 ------------------------------------------
 .. autofunction:: deconv2d_bilinear_upsampling_initializer

docs/modules/nn.rst

@@ -35,6 +35,7 @@ Layer list
    Conv1d
    Conv2d
    Conv3d
+   ConvTranspose1d
    ConvTranspose2d
    ConvTranspose3d
    DepthwiseConv2d
@@ -196,6 +197,10 @@ Conv3d
 Deconvolutions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+ConvTranspose1d
+"""""""""""""""""""""
+.. autoclass:: ConvTranspose1d
+
 ConvTranspose2d
 """""""""""""""""""""
 .. autoclass:: ConvTranspose2d

requirements/requirements_doc.txt

@@ -6,4 +6,5 @@ sphinx_rtd_theme>=0.4,<0.5
 wrapt>=1.11.1
 h5py>=2.9
 cloudpickle>=0.8.1
-tensorflow>=2.0.0
+tensorflow>=2.0.0
+sphinx-mathjax-offline

tensorlayerx/backend/__init__.py

@@ -152,7 +152,7 @@
 )
 # initlizers
 from .ops import (
-    zeros, ones, constant, random_uniform, random_normal, truncated_normal, he_normal, xavier_normal, xavier_uniform
+    zeros, ones, constant, random_uniform, random_normal, truncated_normal, he_normal, he_uniform, xavier_normal, xavier_uniform
 )
 # backend
 from .ops import Reshape

tensorlayerx/backend/ops/__init__.py

@@ -204,7 +204,7 @@
 )
 # initlizers
 from .load_backend import (
-    zeros, ones, constant, random_uniform, random_normal, truncated_normal, he_normal, xavier_normal, xavier_uniform
+    zeros, ones, constant, random_uniform, random_normal, truncated_normal, he_normal, he_uniform, xavier_normal, xavier_uniform
 )
 # backend
 from .load_backend import BACKEND

tensorlayerx/backend/ops/load_backend.py

@@ -58,8 +58,8 @@
     import mindspore.context as context
     import os
     os.environ['DEVICE_ID'] = '0'
-    context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU'),
-    # context.set_context(mode=context.PYNATIVE_MODE, device_target='CPU'),
+    # context.set_context(mode=context.PYNATIVE_MODE, device_target='GPU'),
+    context.set_context(mode=context.PYNATIVE_MODE, device_target='CPU'),
     # enable_task_sink=True, enable_loop_sink=True)
     # context.set_context(mode=context.PYNATIVE_MODE, device_target='Ascend')
     sys.stderr.write('Using MindSpore backend.\n')

tensorlayerx/backend/ops/mindspore_backend.py

@@ -9,8 +9,8 @@
 
 from mindspore.common.parameter import Parameter
 from mindspore.common.initializer import (
-    initializer, Constant, Normal, TruncatedNormal, Initializer, _assignment, _calculate_in_and_out, One, Zero,
-    _calculate_fan_in_and_fan_out
+    initializer, Constant, Normal, TruncatedNormal, Initializer, _assignment, _calculate_gain, One, Zero,
+    _calculate_fan_in_and_fan_out, _calculate_correct_fan
 )
 from mindspore.common.tensor import Tensor
 from mindspore.ops import operations as P
@@ -321,31 +321,7 @@ def truncated_normal(shape, mean=0.0, stddev=1.0, dtype=mstype.float32, seed=Non
     return Tensor(arr, dtype=dtype)
 
 
-class HeNormal(Initializer):
-    r"""
-    he_normal: It draws samples from a truncated normal distribution centered on 0 with
-    stddev = sqrt(2 / fan_in) where fan_in is the number of input units in the weight tensor.
-
-    Args:
-        arr (Array): The array to be assigned.
-
-    Returns:
-        Array, assigned array.
-    """
-
-    def __init__(self, seed=None):
-        super(HeNormal, self).__init__(seed=seed)
-        self.seed = seed
-
-    def _initialize(self, arr):
-        n_in, _ = _calculate_in_and_out(arr)
-        boundary = np.sqrt(2.0 / n_in)
-        random.seed(self.seed)
-        data = np.random.normal(-boundary, boundary, arr.shape)
-        _assignment(arr, data)
-
-
-def he_normal(shape, dtype, seed=None):
+def he_normal(shape, a = 0, mode = 'fan_in', nonlinearity='leaky_relu', dtype=mstype.float32, seed=None):
     """
     He normal initializer.
 
@@ -362,54 +338,43 @@ def he_normal(shape, dtype, seed=None):
     -------
         A tensor of the specified shape filled with he normal values.
     """
-    # shape = shape[::-1]
     arr = np.ndarray(shape)
-    init_obj = HeNormal(seed)
-    init_obj(arr)
+    fan = _calculate_correct_fan(shape, mode)
+    gain = _calculate_gain(nonlinearity, a)
+    std = gain / math.sqrt(fan)
+    data = np.random.normal(0, std, shape)
+    _assignment(arr, data)
     return Tensor(arr, dtype=dtype)
 
-
-class XavierUniform(Initializer):
-
-    def __init__(self, seed=None):
-        super(XavierUniform, self).__init__(seed=seed)
-        self.seed = seed
-
-    def _initialize(self, arr):
-        n_in, n_out = _calculate_fan_in_and_fan_out(arr.shape)
-        boundary = math.sqrt(6.0 / (n_in + n_out))
-        random.seed(self.seed)
-        data = np.random.uniform(-boundary, boundary, arr.shape)
-        _assignment(arr, data)
-
-
-def xavier_uniform(shape, dtype, seed=None):
+def he_uniform(shape, a = 0, mode = 'fan_in', nonlinearity='leaky_relu',dtype=mstype.float32, seed=None):
 
     arr = np.ndarray(shape)
-    init_obj = XavierUniform(seed)
-    init_obj(arr)
+    fan = _calculate_correct_fan(shape, mode)
+    gain = _calculate_gain(nonlinearity, a)
+    std = gain / math.sqrt(fan)
+    boundary = math.sqrt(3.0) * std
+    data = np.random.uniform(-boundary, boundary, shape)
+    _assignment(arr, data)
     return Tensor(arr, dtype=dtype)
 
 
-class XavierNormal(Initializer):
+def xavier_uniform(shape, gain = 1.0, dtype=mstype.float32, seed=None):
 
-    def __init__(self, seed=None):
-        super(XavierNormal, self).__init__(seed=seed)
-        self.seed = seed
-
-    def _initialize(self, arr):
-        n_in, n_out = _calculate_fan_in_and_fan_out(arr.shape)
-        boundary = math.sqrt(2.0 / (n_in + n_out))
-        random.seed(self.seed)
-        data = np.random.normal(0, boundary, arr.shape)
-        _assignment(arr, data)
+    arr = np.ndarray(shape)
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(shape)
+    bound = gain * math.sqrt(6.0 / (fan_in + fan_out))
+    data = np.random.uniform(-bound, bound, shape)
+    _assignment(arr, data)
+    return Tensor(arr, dtype=dtype)
 
 
-def xavier_normal(shape, dtype, seed=None):
+def xavier_normal(shape, gain = 1.0, dtype=mstype.float32, seed=None):
 
     arr = np.ndarray(shape)
-    init_obj = XavierNormal(seed)
-    init_obj(arr)
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(shape)
+    std = gain * math.sqrt(2.0 / float(fan_in + fan_out))
+    data = np.random.normal(0, std, shape)
+    _assignment(arr, data)
     return Tensor(arr, dtype=dtype)