release 0.5.0

Author: Laicheng0830

README.md

@@ -1,6 +1,6 @@
 <a href="https://tensorlayerx.readthedocs.io/">
     <div align="center">
-        <img src="https://git.openi.org.cn/hanjr/tensorlayerx-image/raw/branch/master/tlx-LOGO-04.png" width="50%" height="30%"/>
+        <img src="https://git.openi.org.cn/hanjr/tensorlayerx-image/raw/branch/master/tlx-LOGO--02.jpg" width="50%" height="30%"/>
     </div>
 </a>
 
@@ -18,18 +18,13 @@
 
 🇨🇳 TensorLayerX 是一个跨平台开发框架，可以运行在各类操作系统和AI硬件上，并支持混合框架的开发。目前支持TensorFlow、MindSpore、PaddlePaddle框架常用神经网络层以及算子，PyTorch支持特性正在开发中，[支持列表](https://shimo.im/sheets/kJGCCTxXvqj99RGV/F5m5Z)。
 
+# News
+🔥 **TensorLayerX has been released, it supports TensorFlow、MindSpore and PaddlePaddle backends, and supports some PyTorch operator backends, allowing users to run the code on different hardware like Nvidia-GPU and Huawei-Ascend. Feel free to use it and make suggestions.**
 
-<details>
-    <summary>🇷🇺 TensorLayerX</summary>
-input text here.
-</details>
+🔥 **We need more people to join the dev team, if you are interested, please email [email protected]**
 
-<details>
-    <summary>🇸🇦 TensorLayerX</summary>
-input text here.
-</details>
 
-# TensorLayerX
+# Design Features
 
 Compare with [TensorLayer](https://github.com/tensorlayer/TensorLayer), TensorLayerX （TLX) is a brand new seperated project for platform-agnostic purpose. 
 
@@ -47,13 +42,6 @@ Comparison of TensorLayer version
 
 🔥**Feel free to use TensorLayerX and make suggestions. We need more people to join the dev team, if you are interested, please email [email protected]**
 
-# Examples
-
-- [Basic Examples](https://github.com/tensorlayer/TensorLayerX/tree/main/examples)
-- [TLCV]**Coming soon!**
-
-
-
 # Quick Start
 
 - Installation
@@ -62,34 +50,49 @@ Comparison of TensorLayer version
 pip3 install tensorlayerx 
 # install from Github
 pip3 install git+https://github.com/tensorlayer/tensorlayerx.git 
-# install from OpenI
-pip3 install git+https://git.openi.org.cn/OpenI/tensorlayerX.git
-```
-If you want to use TensorFlow backend, you should install TensorFlow：
-```bash
-pip3 install tensorflow # if you want to use GPUs, CUDA and CuDNN are required.
 ```
+For more installation instructions, please refer to [Installtion](https://tensorlayerx.readthedocs.io/en/latest/user/installation.html)
 
+- Define a model
 
-If you want to use MindSpore backend, you should install mindspore>=1.2.1
-```bash
-pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/1.2.1/MindSpore/gpu/ubuntu_x86/cuda-10.1/mindspore_gpu-1.2.1-cp37-cp37m-linux_x86_64.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com -i https://pypi.tuna.tsinghua.edu.cn/simple
-```
+You can immediately use tensorlayerx to define a model, using your favourite framework in the background, like so:
+```python
+import os
+os.environ['TL_BACKEND'] = 'tensorflow' # change to any framework!
 
-If you want to use paddlepaddle backend, you should install paddlepaddle>=2.1.1
-```bash
-python -m pip install paddlepaddle -i https://mirror.baidu.com/pypi/simple
-```
+import tensorlayerx as tlx
+from tensorlayerx.nn import Module
+from tensorlayerx.nn import Dense
+class CustomModel(Module):
 
-If you want to use PyTorch backend, you should install PyTorch>=1.8.0
-```bash
-pip3 install torch==1.8.2+cu102 torchvision==0.9.2+cu102 torchaudio===0.8.2 -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
+  def __init__(self):
+      super(CustomModel, self).__init__()
+
+      self.dense1 = Dense(n_units=800, act=tlx.ReLU, in_channels=784)
+      self.dense2 = Dense(n_units=800, act=tlx.ReLU, in_channels=800)
+      self.dense3 = Dense(n_units=10, act=None, in_channels=800)
+
+  def forward(self, x, foo=False):
+      z = self.dense1(x)
+      z = self.dense2(z)
+      out = self.dense3(z)
+      if foo:
+          out = tlx.softmax(out)
+      return out
+
+MLP = CustomModel()
+MLP.set_eval()
 ```
 
-- [Tutorial](https://github.com/tensorlayer/TensorLayerX/tree/main/examples/basic_tutorials)
+# Document
+TensorLayer has extensive documentation for both beginners and professionals. 
+
+[![English Documentation](https://img.shields.io/badge/documentation-english-blue.svg)](https://tensorlayerx.readthedocs.io/en/latest/)
 
-- Discussion: [Slack](https://join.slack.com/t/tensorlayer/shared_invite/enQtODk1NTQ5NTY1OTM5LTQyMGZhN2UzZDBhM2I3YjYzZDBkNGExYzcyZDNmOGQzNmYzNjc3ZjE3MzhiMjlkMmNiMmM3Nzc4ZDY2YmNkMTY) , [QQ-Group] , [WeChat-Group]
+# Examples
 
+- [Basic Examples](https://github.com/tensorlayer/TensorLayerX/tree/main/examples)
+- [TLCV]**Coming soon!**
 
 
 # Contact

tensorlayerx/backend/ops/torch_backend.py

@@ -1005,7 +1005,7 @@ def __init__(self, axis=None, epsilon=1e-12):
         self.epsilon = epsilon
 
     def __call__(self, input, *args, **kwargs):
-        raise NotImplementedError
+        return torch.linalg.norm(input, ord=2, dim=self.axis)
 
 
 class EmbeddingLookup(object):

tensorlayerx/model/core.py

@@ -2,7 +2,10 @@
 # -*- coding: utf-8 -*-
 
 from collections.abc import Iterable
-from tensorlayerx.nn.core.common import _save_weights, _load_weights, _save_standard_weights_dict, _load_standard_weights_dict
+from tensorlayerx.nn.core.common import _save_weights, _load_weights, \
+    _save_standard_weights_dict, _load_standard_weights_dict
+from .utils import WithLoss, WithGradPD, WithGradMS, WithGradTF, TrainOneStepWithPD, \
+    TrainOneStepWithMS, TrainOneStepWithTH, TrainOneStepWithTF, GradWrap
 import tensorlayerx as tlx
 from tensorlayerx.nn import Module
 import numpy as np
@@ -11,9 +14,7 @@
 if tlx.BACKEND == 'tensorflow':
     import tensorflow as tf
 if tlx.BACKEND == 'mindspore':
-    from mindspore.ops import composite
     from mindspore.ops import operations as P
-    from mindspore.common import ParameterTuple
 if tlx.BACKEND == 'paddle':
     import paddle as pd
 if tlx.BACKEND == 'torch':
@@ -108,6 +109,12 @@ def train(self, n_epoch, train_dataset=None, test_dataset=False, print_train_bat
                 train_weights=self.train_weights, optimizer=self.optimizer, metrics=self.metrics,
                 print_train_batch=print_train_batch, print_freq=print_freq, test_dataset=test_dataset
             )
+        elif tlx.BACKEND == 'torch':
+            self.th_train(
+                n_epoch=n_epoch, train_dataset=train_dataset, network=self.network, loss_fn=self.loss_fn,
+                train_weights=self.train_weights, optimizer=self.optimizer, metrics=self.metrics,
+                print_train_batch=print_train_batch, print_freq=print_freq, test_dataset=test_dataset
+            )
 
     def eval(self, test_dataset):
         self.network.set_eval()
@@ -436,10 +443,9 @@ def th_train(
 
                 train_loss += loss
                 if metrics:
-                    pass
-                    # metrics.update(output, y_batch)
-                    # train_acc += metrics.result()
-                    # metrics.reset()
+                    metrics.update(output, y_batch)
+                    train_acc += metrics.result()
+                    metrics.reset()
                 else:
                     train_acc += (output.argmax(1) == y_batch).type(torch.float).sum().item()
                 n_iter += 1
@@ -454,180 +460,23 @@ def th_train(
                 print("   train loss: {}".format(train_loss / n_iter))
                 print("   train acc:  {}".format(train_acc / n_iter))
 
-
-class WithLoss(Module):
-    """
-    High-Level API for Training or Testing.
-
-    Wraps the network with loss function. This Module accepts data and label as inputs and
-    the computed loss will be returned.
-
-    Parameters
-    ----------
-    backbone : tensorlayer model
-        The tensorlayer network.
-    loss_fn : function
-        Objective function
-
-    Methods
-    ---------
-    forward()
-        Model inference.
-
-    Examples
-    --------
-    >>> import tensorlayerx as tlx
-    >>> net = vgg16()
-    >>> loss_fn = tlx.losses.softmax_cross_entropy_with_logits
-    >>> net_with_loss = tlx.model.WithLoss(net, loss_fn)
-
-    """
-
-    def __init__(self, backbone, loss_fn):
-        super(WithLoss, self).__init__()
-        self._backbone = backbone
-        self._loss_fn = loss_fn
-
-    def forward(self, data, label):
-        out = self._backbone(data)
-        return self._loss_fn(out, label)
-
-    @property
-    def backbone_network(self):
-        return self._backbone
-
-
-class GradWrap(Module):
-    """ GradWrap definition """
-
-    def __init__(self, network, trainable_weights):
-        super(GradWrap, self).__init__(auto_prefix=False)
-        self.network = network
-        self.weights = ParameterTuple(trainable_weights)
-
-    def forward(self, x, label):
-        return composite.GradOperation(get_by_list=True)(self.network, self.weights)(x, label)
-
-
-class WithGradMS(Module):
-    "Module that returns the gradients."
-
-    def __init__(self, network, loss_fn=None, sens=None, optimizer=None):
-        super(WithGradMS, self).__init__()
-        self.network = network
-        self.loss_fn = loss_fn
-        self.weights = ParameterTuple(network.trainable_weights)
-        self.grad = composite.GradOperation(get_by_list=True, sens_param=(sens is not None))
-        self.sens = sens
-        self.optimizer = optimizer
-        if self.loss_fn is None:
-            self.network_with_loss = network
-        else:
-            self.network_with_loss = WithLoss(self.network, self.loss_fn)
-        self.network.set_train()
-
-    def forward(self, inputs, label):
-        grads = self.grad(self.network_with_loss, self.weights)(inputs, label)
-        return grads
-
-
-class WithGradTF(object):
-
-    def __init__(self, network, loss_fn=None, optimizer=None):
-        self.network = network
-        self.loss_fn = loss_fn
-        self.train_weights = self.network.trainable_weights
-        self.optimizer = optimizer
-        if loss_fn is None:
-            self.network_with_loss = network
-        else:
-            self.network_with_loss = WithLoss(self.network, self.loss_fn)
-        self.network.set_train()
-
-    def __call__(self, inputs, label):
-        with tf.GradientTape() as tape:
-            loss = self.network_with_loss(inputs, label)
-        grads = tape.gradient(loss, self.train_weights)
-        return grads
-
-
-class WithGradPD(object):
-
-    def __init__(self, network, loss_fn=None, optimizer=None):
-        self.network = network
-        self.loss_fn = loss_fn
-        self.train_weights = self.network.trainable_weights
-        self.optimizer = optimizer
-        if loss_fn is None:
-            self.network_with_loss = network
-        else:
-            self.network_with_loss = WithLoss(self.network, self.loss_fn)
-        self.network.set_train()
-
-    def __call__(self, inputs, label):
-        loss = self.network_with_loss(inputs, label)
-        grads = self.optimizer.gradient(loss, self.train_weights)
-        return grads
-
-
-class TrainOneStepWithTF(object):
-
-    def __init__(self, net_with_loss, optimizer, train_weights):
-        self.net_with_loss = net_with_loss
-        self.optimzer = optimizer
-        self.train_weights = train_weights
-
-    def __call__(self, data, label):
-        with tf.GradientTape() as tape:
-            loss = self.net_with_loss(data, label)
-        grad = tape.gradient(loss, self.train_weights)
-        self.optimzer.apply_gradients(zip(grad, self.train_weights))
-        return loss
-
-
-class TrainOneStepWithMS(object):
-
-    def __init__(self, net_with_loss, optimizer, train_weights):
-        self.net_with_loss = net_with_loss
-        self.optimizer = optimizer
-        self.train_weights = train_weights
-        self.net_with_loss = net_with_loss
-        self.train_network = GradWrap(net_with_loss, train_weights)
-
-    def __call__(self, data, label):
-        loss = self.net_with_loss(data, label)
-        grads = self.train_network(data, label)
-        self.optimizer.apply_gradients(zip(grads, self.train_weights))
-        loss = loss.asnumpy()
-        return loss
-
-
-class TrainOneStepWithPD(object):
-
-    def __init__(self, net_with_loss, optimizer, train_weights):
-        self.net_with_loss = net_with_loss
-        self.optimizer = optimizer
-        self.train_weights = train_weights
-
-    def __call__(self, data, label):
-        loss = self.net_with_loss(data, label)
-        grads = self.optimizer.gradient(loss, self.train_weights)
-        self.optimizer.apply_gradients(zip(grads, self.train_weights))
-        return loss.numpy()
-
-
-class TrainOneStepWithTH(object):
-
-    def __init__(self, net_with_loss, optimizer, train_weights):
-        self.net_with_loss = net_with_loss
-        self.optimizer = optimizer
-        self.train_weights = train_weights
-
-    def __call__(self, data, label):
-        loss = self.net_with_loss(data, label)
-        grads = self.optimizer.gradient(loss, self.train_weights)
-        self.optimizer.apply_gradients(zip(grads, self.train_weights))
-        return loss
+            if test_dataset:
+                # use training and evaluation sets to evaluate the model every print_freq epoch
+                if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
+                    network.set_eval()
+                    val_loss, val_acc, n_iter = 0, 0, 0
+                    for X_batch, y_batch in test_dataset:
+                        _logits = network(X_batch)  # is_train=False, disable dropout
+                        val_loss += loss_fn(_logits, y_batch, name='eval_loss')
+                        if metrics:
+                            metrics.update(_logits, y_batch)
+                            val_acc += metrics.result()
+                            metrics.reset()
+                        else:
+                            val_acc += (_logits.argmax(1) == y_batch).type(torch.float).sum().item()
+                        n_iter += 1
+                    print("   val loss: {}".format(val_loss / n_iter))
+                    print("   val acc:  {}".format(val_acc / n_iter))
 
 
 class WithGrad(object):
@@ -713,3 +562,11 @@ def __init__(self, net_with_loss, optimizer, train_weights):
     def __call__(self, data, label):
         loss = self.net_with_train(data, label)
         return loss
+
+
+class TrainOneStepWithGradientClipping(object):
+    def __init__(self):
+        pass
+
+    def __call__(self, data, label):
+        pass