update docs

Author: hanjr92

examples/basic_tutorials/automatic_inference_input_shape.py

@@ -62,7 +62,7 @@ def __len__(self):
 batch_size = 500
 print_freq = 5
 train_weights = MLP.trainable_weights
-optimizer = tlx.optimizers.Adam(learning_rate=0.0001)
+optimizer = tlx.optimizers.Adam(0.0001)
 train_dataset = mnistdataset(data=X_train, label=y_train)
 train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 val_dataset = mnistdataset(data=X_val, label=y_val)

examples/basic_tutorials/cifar10_cnn.py

@@ -3,9 +3,9 @@
 
 import os
 # os.environ['TL_BACKEND'] = 'paddle'
-os.environ['TL_BACKEND'] = 'tensorflow'
+# os.environ['TL_BACKEND'] = 'tensorflow'
 # os.environ['TL_BACKEND'] = 'mindspore'
-# os.environ['TL_BACKEND'] = 'torch'
+os.environ['TL_BACKEND'] = 'torch'
 
 import time
 from tensorlayerx.dataflow import Dataset, DataLoader

examples/basic_tutorials/gradient_clip_mixed_tensorflow.py

@@ -54,7 +54,7 @@ def __len__(self):
 train_weights = MLP.trainable_weights
 train_dataset = mnistdataset(data=X_train, label=y_train)
 train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
-optimizer = tlx.optimizers.Adam(learning_rate=0.0001, weight_decay= 0.001, grad_clip=tlx.ops.ClipGradByValue())
+optimizer = tlx.optimizers.Adam(lr=0.0001, weight_decay= 0.001, grad_clip=tlx.ops.ClipGradByValue())
 
 net_with_loss = tlx.model.WithLoss(backbone=MLP, loss_fn=tlx.losses.softmax_cross_entropy_with_logits)
 net_with_grad_train = tlx.model.TrainOneStep(net_with_loss, optimizer, train_weights)

examples/basic_tutorials/imdb_LSTM_simple.py

@@ -5,8 +5,8 @@
 import os
 # os.environ['TL_BACKEND'] = 'tensorflow'
 # os.environ['TL_BACKEND'] = 'mindspore'
-# os.environ['TL_BACKEND'] = 'paddle'
-os.environ['TL_BACKEND'] = 'torch'
+os.environ['TL_BACKEND'] = 'paddle'
+# os.environ['TL_BACKEND'] = 'torch'
 import tensorlayerx as tlx
 from tensorlayerx.nn import Module
 from tensorlayerx.nn import Linear, LSTM, Embedding
@@ -41,7 +41,7 @@ class ImdbNet(Module):
 
     def __init__(self):
         super(ImdbNet, self).__init__()
-        self.embedding = Embedding(vocabulary_size=vocab_size, embedding_size=64)
+        self.embedding = Embedding(num_embeddings=vocab_size, embedding_dim=64)
         self.lstm = LSTM(input_size=64, hidden_size=64)
         self.linear1 = Linear(in_features=64, out_features=64, act=tlx.ReLU)
         self.linear2 = Linear(in_features=64, out_features=2)

examples/basic_tutorials/mnist_gan.py

@@ -109,8 +109,8 @@ def forward(self, real_data, g_data):
 # loss_fn = tlx.losses.sigmoid_cross_entropy
 # optimizer = tlx.optimizers.Momentum(learning_rate=5e-4, momentum=0.5)
 loss_fn = tlx.losses.mean_squared_error
-optimizer_g = tlx.optimizers.Adam(learning_rate=3e-4, beta_1=0.5, beta_2=0.999)
-optimizer_d = tlx.optimizers.Adam(learning_rate=3e-4)
+optimizer_g = tlx.optimizers.Adam(lr=3e-4, beta_1=0.5, beta_2=0.999)
+optimizer_d = tlx.optimizers.Adam(lr=3e-4)
 
 g_weights = G.trainable_weights
 d_weights = D.trainable_weights

examples/basic_tutorials/mnist_mlp.py

@@ -3,10 +3,10 @@
 
 # The same set of code can switch the backend with one line
 import os
-os.environ['TL_BACKEND'] = 'tensorflow'
+# os.environ['TL_BACKEND'] = 'tensorflow'
 # os.environ['TL_BACKEND'] = 'mindspore'
 # os.environ['TL_BACKEND'] = 'paddle'
-# os.environ['TL_BACKEND'] = 'torch'
+os.environ['TL_BACKEND'] = 'torch'
 
 import tensorlayerx as tlx
 from tensorlayerx.nn import Module

examples/basic_tutorials/tutorial_using_tensorboradX.py

@@ -91,7 +91,7 @@ def forward(self, x, foo=None):
 batch_size = 500
 print_freq = 1
 train_weights = MLP.trainable_weights
-optimizer = tlx.optimizers.Adam(learning_rate=0.0001)
+optimizer = tlx.optimizers.Adam(lr=0.0001)
 train_batch = 0
 test_batch = 0
 

tensorlayerx/backend/ops/__init__.py

@@ -11,6 +11,7 @@
 from .load_backend import relu
 from .load_backend import elu
 from .load_backend import relu6
+from .load_backend import prelu
 from .load_backend import leaky_relu
 from .load_backend import sigmoid
 from .load_backend import softmax
@@ -43,6 +44,7 @@
 from .load_backend import ReLU
 from .load_backend import ELU
 from .load_backend import ReLU6
+from .load_backend import PReLU
 from .load_backend import LeakyReLU
 from .load_backend import Softplus
 from .load_backend import Tanh

tensorlayerx/backend/ops/mindspore_nn.py

@@ -271,15 +271,15 @@ def relu6(x):
 
 class LeakyReLU(Cell):
 
-    def __init__(self, alpha=0.2):
+    def __init__(self, negative_slope=0.01):
         super(LeakyReLU, self).__init__()
-        self.leakyrelu = ms.nn.LeakyReLU(alpha=alpha)
+        self.leakyrelu = ms.nn.LeakyReLU(alpha=negative_slope)
 
     def construct(self, x):
         return self.leakyrelu(x)
 
 
-def leaky_relu(x, alpha=0.2):
+def leaky_relu(x, negative_slope=0.2):
     """
     Compute the Leaky ReLU activation function.
 
@@ -294,9 +294,9 @@ def leaky_relu(x, alpha=0.2):
         The activation value.
     """
 
-    leaky_relu = LeakyReLU(alpha=alpha)
+    leaky_relu = ms.nn.LeakyReLU(alpha=negative_slope)
     output = leaky_relu(x)
-    return leaky_relu
+    return output
 
 
 class Softplus(Cell):
@@ -348,15 +348,15 @@ def sigmoid(x):
 
 class Softmax(Cell):
 
-    def __init__(self):
+    def __init__(self, axis = -1):
         super(Softmax, self).__init__()
-        self.softmax = P.Softmax()
+        self.softmax = P.Softmax(axis)
 
     def construct(self, x):
         return self.softmax(x)
 
 
-def softmax(logits, axis=None):
+def softmax(logits, axis=-1):
     """
     Computes softmax activations.
 
@@ -2392,3 +2392,22 @@ def __init__(
 
     def construct(self, inputs):
         raise NotImplementedError
+
+class PReLU(Cell):
+
+    def __init__(self, data_format):
+        super(PReLU, self).__init__()
+        self.data_format = data_format
+
+    def __call__(self, input, weight):
+
+        prelu = P.PReLU()
+        v = prelu(input, F.cast(weight, input.dtype))
+        return v
+
+
+def prelu(input, weight, data_format):
+
+    prelu = P.PReLU()
+    v = prelu(input, F.cast(weight, input.dtype))
+    return v

tensorlayerx/backend/ops/paddle_nn.py

@@ -257,14 +257,14 @@ def relu6(x):
 
 class LeakyReLU(object):
 
-    def __init__(self, alpha=0.2):
-        self.alpha = alpha
+    def __init__(self, negative_slope=0.2):
+        self.negative_slope = negative_slope
 
     def __call__(self, x):
-        return F.leaky_relu(x, negative_slope=self.alpha)
+        return F.leaky_relu(x, negative_slope=self.negative_slope)
 
 
-def leaky_relu(x):
+def leaky_relu(x, negative_slope=0.01):
     """
     Compute the Leaky ReLU activation function.
 
@@ -279,7 +279,7 @@ def leaky_relu(x):
         The activation value.
     """
 
-    return F.leaky_relu(x)
+    return F.leaky_relu(x, negative_slope)
 
 
 class Softplus(object):
@@ -327,11 +327,11 @@ def sigmoid(x):
 
 class Softmax(object):
 
-    def __init__(self):
-        pass
+    def __init__(self, axis = -1):
+        self.axis = axis
 
     def __call__(self, x):
-        return F.softmax(x)
+        return F.softmax(x, axis=self.axis)
 
 
 def softmax(logits, axis=-1):
@@ -2011,3 +2011,19 @@ def __init__(
 
     def __call__(self, inputs):
         raise NotImplementedError
+
+
+class PReLU(object):
+
+    def __init__(self, data_format):
+        self.data_format, _ = preprocess_2d_format(data_format, None)
+
+    def __call__(self, input, weight):
+
+        return F.prelu(input, weight, data_format=self.data_format)
+
+
+def prelu(input, weight, data_format):
+
+    data_format, _ = preprocess_2d_format(data_format, None)
+    return F.prelu(input, weight, data_format=data_format)