fix torch opti, update ops

Author: hanjr92

docs/index.rst

@@ -56,6 +56,7 @@ method, this part of the documentation is for you.
 
   modules/activation
   modules/losses
+  modules/metrics
   modules/dataflow
   modules/files
   modules/nn

docs/modules/losses.rst

@@ -2,7 +2,7 @@ API - Losses
 ==================
 
 To make TensorLayerX simple, we minimize the number of cost functions as much as we can.
-For more complex activation, TensorFlow(MindSpore, PaddlePaddle, PyTorch) API will be required.
+For more complex loss function, TensorFlow(MindSpore, PaddlePaddle, PyTorch) API will be required.
 
 .. note::
     Please refer to `Getting Started <https://github.com/tensorlayer/TensorLayerX/tree/master/docs/user>`_ for getting specific weights for weight regularization.

docs/modules/metrics.rst

@@ -0,0 +1,55 @@
+API - Metrics
+==================
+
+The tensorlayerx.metrics directory contians Accuracy, Auc, Precision and Recall.
+For more complex metrics, you can encapsulates metric logic and APIs by base class.
+
+
+.. automodule:: tensorlayerx.metrics
+
+Metric list
+-------------
+
+.. autosummary::
+
+   Metric
+   Accuracy
+   Auc
+   Precision
+   Recall
+   acc
+
+
+
+Metric
+^^^^^^^^^^^^^^^^
+.. autoclass:: Metric
+
+
+Accuracy
+""""""""""""""""""""""""""
+.. autoclass:: Accuracy
+    :members:
+
+
+Auc
+""""""""""""""""""""""""""
+.. autoclass:: Auc
+    :members:
+
+
+Precision
+""""""""""""""""""""""""""
+.. autoclass:: Precision
+    :members:
+
+
+Recall
+""""""""""""""""""""""""""
+.. autoclass:: Recall
+    :members:
+
+
+acc
+""""""""""""""""""""""""""
+.. autofunction:: acc

docs/modules/ops.rst

@@ -114,6 +114,8 @@ API - Operations
    is_tensor
    tensor_scatter_nd_update
    diag
+   mask_select
+   eye
 
 TensorLayerX Tensor Operations
 --------------------------------
@@ -548,4 +550,12 @@ tensor_scatter_nd_update
 
 diag
 ^^^^^^^^^^^^^^^^^^^^^^^
-.. autofunction:: diag
+.. autofunction:: diag
+
+mask_select
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: mask_select
+
+eye
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: eye

docs/user/get_start_advance.rst

@@ -95,8 +95,8 @@ The full implementation is as follow, which supports both automatic inference in
     With TensorLayerx
 
     >>> net = tlx.nn.Input([100, 50], name='input')
-    >>> linear = tlx.nn.Linear(out_features=800, act=tlx.ReLU, in_features=50, name='linear_1')
-    >>> tensor = tlx.nn.Linear(out_features=800, act=tlx.ReLU, name='linear_2')(net)
+    >>> linear = tlx.nn.Linear(out_features=800, act=tlx.nn.ReLU, in_features=50, name='linear_1')
+    >>> tensor = tlx.nn.Linear(out_features=800, act=tlx.nn.ReLU, name='linear_2')(net)
 
     Notes
     -----

docs/user/get_start_model.rst

@@ -18,9 +18,9 @@ Sequential model
 
   def get_model():
       layer_list = []
-      layer_list.append(Linear(out_features=800, act=tlx.ReLU, in_features=784, name='Linear1'))
-      layer_list.append(Linear(out_features=800, act=tlx.ReLU, in_features=800, name='Linear2'))
-      layer_list.append(Linear(out_features=10, act=tlx.ReLU, in_features=800, name='Linear3'))
+      layer_list.append(Linear(out_features=800, act=tlx.nn.ReLU, in_features=784, name='Linear1'))
+      layer_list.append(Linear(out_features=800, act=tlx.nn.ReLU, in_features=800, name='Linear2'))
+      layer_list.append(Linear(out_features=10, act=tlx.nn.ReLU, in_features=800, name='Linear3'))
       MLP = Sequential(layer_list)
       return MLP
 
@@ -43,9 +43,9 @@ In this case, you need to manually input the output shape of the previous layer
           super(CustomModel, self).__init__()
 
           self.dropout1 = Dropout(p=0.2)
-          self.linear1 = Linear(out_features=800, act=tlx.ReLU, in_features=784)
+          self.linear1 = Linear(out_features=800, act=tlx.nn.ReLU, in_features=784)
           self.dropout2 = Dropout(p=0.2)
-          self.linear2 = Linear(out_features=800, act=tlx.ReLU, in_features=800)
+          self.linear2 = Linear(out_features=800, act=tlx.nn.ReLU, in_features=800)
           self.dropout3 = Dropout(p=0.2)
           self.linear3 = Linear(out_features=10, act=None, in_features=800)
 
@@ -83,9 +83,9 @@ In this case, you do not manually input the output shape of the previous layer t
           super(CustomModel, self).__init__()
 
           self.dropout1 = Dropout(p=0.2)
-          self.linear1 = Linear(out_features=800, act=tlx.ReLU)
+          self.linear1 = Linear(out_features=800, act=tlx.nn.ReLU)
           self.dropout2 = Dropout(p=0.2)
-          self.linear2 = Linear(out_features=800, act=tlx.ReLU)
+          self.linear2 = Linear(out_features=800, act=tlx.nn.ReLU)
           self.dropout3 = Dropout(p=0.2)
           self.linear3 = Linear(out_features=10, act=None)
 
@@ -135,9 +135,9 @@ For dynamic model, call the layer multiple time in forward function
   class MyModel(Module):
       def __init__(self):
           super(MyModel, self).__init__()
-          self.linear_shared = Linear(out_features=800, act=tlx.ReLU, in_features=784)
-          self.linear1 = Linear(out_features=10, act=tlx.ReLU, in_features=800)
-          self.linear2 = Linear(out_features=10, act=tlx.ReLU, in_features=800)
+          self.linear_shared = Linear(out_features=800, act=tlx.nn.ReLU, in_features=784)
+          self.linear1 = Linear(out_features=10, act=tlx.nn.ReLU, in_features=800)
+          self.linear2 = Linear(out_features=10, act=tlx.nn.ReLU, in_features=800)
           self.cat = Concat()
 
       def forward(self, x):

examples/basic_tutorials/automatic_inference_input_shape.py

@@ -20,11 +20,11 @@ def __init__(self):
         super(CustomModel, self).__init__()
         self.dropout1 = Dropout(p=0.2)
         self.linear1 = Linear(out_features=800)
-        self.batchnorm = BatchNorm1d(act=tlx.ReLU)
+        self.batchnorm = BatchNorm1d(act=tlx.nn.ReLU)
         self.dropout2 = Dropout(p=0.2)
-        self.linear2 = Linear(out_features=800, act=tlx.ReLU)
+        self.linear2 = Linear(out_features=800, act=tlx.nn.ReLU)
         self.dropout3 = Dropout(p=0.2)
-        self.linear3 = Linear(out_features=10, act=tlx.ReLU)
+        self.linear3 = Linear(out_features=10, act=tlx.nn.ReLU)
 
     def forward(self, x, foo=None):
         z = self.dropout1(x)

examples/basic_tutorials/cifar10_cnn.py

@@ -33,17 +33,17 @@ def __init__(self):
         b_init2 = tlx.nn.initializers.constant(value=0.1)
 
         self.conv1 = Conv2d(64, (5, 5), (1, 1), padding='SAME', W_init=W_init, b_init=None, name='conv1', in_channels=3)
-        self.bn = BatchNorm2d(num_features=64, act=tlx.ReLU)
+        self.bn = BatchNorm2d(num_features=64, act=tlx.nn.ReLU)
         self.maxpool1 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool1')
 
         self.conv2 = Conv2d(
-            64, (5, 5), (1, 1), padding='SAME', act=tlx.ReLU, W_init=W_init, b_init=None, name='conv2', in_channels=64
+            64, (5, 5), (1, 1), padding='SAME', act=tlx.nn.ReLU, W_init=W_init, b_init=None, name='conv2', in_channels=64
         )
         self.maxpool2 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool2')
 
         self.flatten = Flatten(name='flatten')
-        self.linear1 = Linear(384, act=tlx.ReLU, W_init=W_init2, b_init=b_init2, name='linear1relu', in_features=2304)
-        self.linear2 = Linear(192, act=tlx.ReLU, W_init=W_init2, b_init=b_init2, name='linear2relu', in_features=384)
+        self.linear1 = Linear(384, act=tlx.nn.ReLU, W_init=W_init2, b_init=b_init2, name='linear1relu', in_features=2304)
+        self.linear2 = Linear(192, act=tlx.nn.ReLU, W_init=W_init2, b_init=b_init2, name='linear2relu', in_features=384)
         self.linear3 = Linear(10, act=None, W_init=W_init2, name='output', in_features=192)
 
     def forward(self, x):
@@ -179,17 +179,17 @@ def forward(self, data, label):
 #         b_init2 = tlx.nn.initializers.constant(value=0.1)
 #
 #         self.conv1 = Conv2d(64, (5, 5), (1, 1), padding='SAME', W_init=W_init, b_init=None, name='conv1', in_channels=3)
-#         self.bn = BatchNorm2d(num_features=64, act=tlx.ReLU)
+#         self.bn = BatchNorm2d(num_features=64, act=tlx.nn.ReLU)
 #         self.maxpool1 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool1')
 #
 #         self.conv2 = Conv2d(
-#             64, (5, 5), (1, 1), padding='SAME', act=tlx.ReLU, W_init=W_init, b_init=None, name='conv2', in_channels=64
+#             64, (5, 5), (1, 1), padding='SAME', act=tlx.nn.ReLU, W_init=W_init, b_init=None, name='conv2', in_channels=64
 #         )
 #         self.maxpool2 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool2')
 #
 #         self.flatten = Flatten(name='flatten')
-#         self.linear1 = Linear(384, act=tlx.ReLU, W_init=W_init2, b_init=b_init2, name='linear1relu', in_channels=2304)
-#         self.linear2 = Linear(192, act=tlx.ReLU, W_init=W_init2, b_init=b_init2, name='linear2relu', in_channels=384)
+#         self.linear1 = Linear(384, act=tlx.nn.ReLU, W_init=W_init2, b_init=b_init2, name='linear1relu', in_channels=2304)
+#         self.linear2 = Linear(192, act=tlx.nn.ReLU, W_init=W_init2, b_init=b_init2, name='linear2relu', in_channels=384)
 #         self.linear3 = Linear(10, act=None, W_init=W_init2, name='output', in_channels=192)
 #
 #     def forward(self, x):
@@ -364,18 +364,18 @@ def forward(self, data, label):
 #     def __init__(self):
 #         super(CNN, self).__init__()
 #         self.conv1 = Conv2d(
-#             64, (5, 5), (2, 2), b_init=None, name='conv1', in_channels=3, act=tlx.ReLU, data_format='channels_first'
+#             64, (5, 5), (2, 2), b_init=None, name='conv1', in_channels=3, act=tlx.nn.ReLU, data_format='channels_first'
 #         )
-#         self.bn = BatchNorm2d(num_features=64, act=tlx.ReLU, data_format='channels_first')
+#         self.bn = BatchNorm2d(num_features=64, act=tlx.nn.ReLU, data_format='channels_first')
 #         self.maxpool1 = MaxPool2d((3, 3), (2, 2), name='pool1', data_format='channels_first')
 #         self.conv2 = Conv2d(
-#             128, (5, 5), (2, 2), act=tlx.ReLU, b_init=None, name='conv2', in_channels=64, data_format='channels_first'
+#             128, (5, 5), (2, 2), act=tlx.nn.ReLU, b_init=None, name='conv2', in_channels=64, data_format='channels_first'
 #         )
 #         self.maxpool2 = MaxPool2d((3, 3), (2, 2), name='pool2', data_format='channels_first')
 #
 #         self.flatten = Flatten(name='flatten')
-#         self.linear1 = Linear(120, act=tlx.ReLU, name='linear1relu', in_channels=512)
-#         self.linear2 = Linear(84, act=tlx.ReLU, name='linear2relu', in_channels=120)
+#         self.linear1 = Linear(120, act=tlx.nn.ReLU, name='linear1relu', in_channels=512)
+#         self.linear2 = Linear(84, act=tlx.nn.ReLU, name='linear2relu', in_channels=120)
 #         self.linear3 = Linear(10, act=None, name='output', in_channels=84)
 #
 #     def forward(self, x):
@@ -509,18 +509,18 @@ def forward(self, data, label):
 #         b_init2 = tlx.nn.initializers.constant(value=0.1)
 #
 #         self.conv1 = Conv2d(64, (5, 5), (1, 1), padding='SAME', W_init=W_init, b_init=None, name='conv1', in_channels=3)
-#         self.bn1 = BatchNorm2d(num_features=64, act=tlx.ReLU)
+#         self.bn1 = BatchNorm2d(num_features=64, act=tlx.nn.ReLU)
 #         self.maxpool1 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool1')
 #
 #         self.conv2 = Conv2d(
 #             64, (5, 5), (1, 1), padding='SAME', W_init=W_init, b_init=None, name='conv2', in_channels=64
 #         )
-#         self.bn2 = BatchNorm2d(num_features=64, act=tlx.ReLU)
+#         self.bn2 = BatchNorm2d(num_features=64, act=tlx.nn.ReLU)
 #         self.maxpool2 = MaxPool2d((3, 3), (2, 2), padding='SAME', name='pool2')
 #
 #         self.flatten = Flatten(name='flatten')
-#         self.linear1 = Linear(384, act=tlx.ReLU, W_init=W_init2, b_init=b_init2, name='linear1relu', in_channels=2304)
-#         self.linear2 = Linear(192, act=tlx.ReLU, W_init=W_init2, b_init=b_init2, name='linear2relu', in_channels=384)
+#         self.linear1 = Linear(384, act=tlx.nn.ReLU, W_init=W_init2, b_init=b_init2, name='linear1relu', in_channels=2304)
+#         self.linear2 = Linear(192, act=tlx.nn.ReLU, W_init=W_init2, b_init=b_init2, name='linear2relu', in_channels=384)
 #         self.linear3 = Linear(10, act=None, W_init=W_init2, name='output', in_channels=192)
 #
 #     def forward(self, x):

examples/basic_tutorials/gradient_clip_mixed_tensorflow.py

@@ -22,8 +22,8 @@ class CustomModel(Module):
     def __init__(self):
         super(CustomModel, self).__init__()
         self.linear1 = Linear(out_features=800, in_features=784)
-        self.linear2 = Linear(out_features=800, act=tlx.ReLU, in_features=800)
-        self.linear3 = Linear(out_features=10, act=tlx.ReLU, in_features=800)
+        self.linear2 = Linear(out_features=800, act=tlx.nn.ReLU, in_features=800)
+        self.linear3 = Linear(out_features=10, act=tlx.nn.ReLU, in_features=800)
 
     def forward(self, x, foo=None):
         z = self.linear1(x)

examples/basic_tutorials/imdb_LSTM_simple.py

@@ -43,7 +43,7 @@ def __init__(self):
         super(ImdbNet, self).__init__()
         self.embedding = Embedding(num_embeddings=vocab_size, embedding_dim=64)
         self.lstm = LSTM(input_size=64, hidden_size=64)
-        self.linear1 = Linear(in_features=64, out_features=64, act=tlx.ReLU)
+        self.linear1 = Linear(in_features=64, out_features=64, act=tlx.nn.ReLU)
         self.linear2 = Linear(in_features=64, out_features=2)
 
     def forward(self, x):