Merge branch 'main' of github.com:tensorlayer/TensorLayerX into main

Author: Laicheng0830

README.md

@@ -14,9 +14,12 @@
 [![Downloads](https://pepy.tech/badge/tensorlayerx/week)](https://pepy.tech/project/tensorlayerx/week)
 [![Docker Pulls](https://img.shields.io/docker/pulls/tensorlayer/tensorlayer.svg)](https://hub.docker.com/r/tensorlayer/tensorlayer/)
 
-🇬🇧 TensorLayerX is a multi-backend AI framework, which supports TensorFlow, Pytorch, MindSpore, PaddlePaddle, OneFlow and Jittor as the backends, allowing users to run the code on different hardware like Nvidia-GPU and Huawei-Ascend. [supported layers](https://shimo.im/sheets/kJGCCTxXvqj99RGV/F5m5Z). 
+🇬🇧 TensorLayerX is a multi-backend AI framework, which supports TensorFlow, Pytorch, MindSpore, PaddlePaddle, OneFlow and Jittor as the backends, allowing users to run the code on different hardware like Nvidia-GPU and Huawei-Ascend. 
+This project is maintained by researchers from Peking University, Imperial College London, Princeton, Stanford, Tsinghua, Edinburgh and Peng Cheng Lab.
+[supported layers](https://shimo.im/sheets/kJGCCTxXvqj99RGV/F5m5Z). 
 
-🇨🇳 TensorLayerX 是一个跨平台开发框架，支持TensorFlow, Pytorch, MindSpore, PaddlePaddle, OneFlow和Jittor，用户不需要修改任何代码即可以运行在各类操作系统和AI硬件上（如Nvidia-GPU 和 Huawei-Ascend），并支持混合框架的开发。[支持列表](https://shimo.im/sheets/kJGCCTxXvqj99RGV/F5m5Z)。
+🇨🇳 TensorLayerX 是一个跨平台开发框架，支持TensorFlow, Pytorch, MindSpore, PaddlePaddle, OneFlow和Jittor，用户不需要修改任何代码即可以运行在各类操作系统和AI硬件上（如Nvidia-GPU 和 Huawei-Ascend），并支持混合框架的开发。这个项目由北京大学、鹏城实验室、爱丁堡大学、帝国理工、清华、普林斯顿、斯坦福等机构的研究人员维护。
+[支持列表](https://shimo.im/sheets/kJGCCTxXvqj99RGV/F5m5Z)。
 
 
 

docs/modules/nn.rst

@@ -13,8 +13,9 @@ Layer list
 .. autosummary::
 
    Module
-   ModuleList
    Sequential
+   ModuleList
+   ModuleDict
 
    Input
 
@@ -125,13 +126,17 @@ Module
 ^^^^^^^^^^^^^^^^
 .. autoclass:: Module
 
+Sequential
+^^^^^^^^^^^^^^^^
+.. autoclass:: Sequential
+
 ModuleList
 ^^^^^^^^^^^^^^^^
 .. autoclass:: ModuleList
 
-Sequential
+ModuleDict
 ^^^^^^^^^^^^^^^^
-.. autoclass:: Sequential
+.. autoclass:: ModuleDict
 
 .. -----------------------------------------------------------
 ..                        Input Layer

examples/basic_tutorials/tutorial_ModuleList.py renamed to examples/basic_tutorials/tutorial_ModuleContainer.py

@@ -2,13 +2,17 @@
 # -*- coding: utf-8 -*-
 
 import os
-# os.environ['TL_BACKEND'] = 'tensorflow'
+os.environ['TL_BACKEND'] = 'tensorflow'
 # os.environ['TL_BACKEND'] = 'mindspore'
-os.environ['TL_BACKEND'] = 'paddle'
-
-from tensorlayerx.nn import Module, ModuleList, Linear
+# os.environ['TL_BACKEND'] = 'paddle'
+# os.environ['TL_BACKEND'] = 'torch'
+import numpy as np
+from tensorlayerx.nn import Module, ModuleList, Linear, ModuleDict
 import tensorlayerx as tlx
 
+
+####################### Holds submodules in a list ########################################
+
 d1 = Linear(out_features=800, act=tlx.ReLU, in_features=784, name='linear1')
 d2 = Linear(out_features=800, act=tlx.ReLU, in_features=800, name='linear2')
 d3 = Linear(out_features=10, act=tlx.ReLU, in_features=800, name='linear3')
@@ -42,3 +46,21 @@ def forward(self, inputs):
 print(net.trainable_weights)
 print(net)
 print(net(tlx.nn.Input((10, 784))))
+
+####################### Holds submodules in a Dict ########################################
+class MyModule(Module):
+
+    def __init__(self):
+        super(MyModule, self).__init__()
+        self.dict = ModuleDict({
+                'linear1': Linear(out_features=800, act=tlx.ReLU, in_features=784, name='linear1'),
+                'linear2': Linear(out_features=800, act=tlx.ReLU, in_features=800, name='linear2')
+               })
+    def forward(self, x, linear):
+        x = self.dict[linear](x)
+        return x
+
+x = tlx.convert_to_tensor(np.ones(shape=(1,784)), dtype=tlx.float32)
+net = MyModule()
+x = net(x, 'linear1')
+print(x)

tensorlayerx/__init__.py

@@ -21,8 +21,6 @@
     from tensorlayerx import losses
     from tensorlayerx import decorators
     from tensorlayerx import files
-    from .utils import lazy_imports
-    from . import utils
     from tensorlayerx import logging
     from tensorlayerx import model
     from tensorlayerx import optimizers

tensorlayerx/backend/ops/mindspore_backend.py

@@ -1054,8 +1054,7 @@ def transpose(a, perm=None, conjugate=False):
         A transposed Tensor.
     """
     # TODO conjugate
-    trans_obj = P.Transpose()
-    outputs = trans_obj(a, perm)
+    outputs = msnp.transpose(a, perm)
     print(outputs)
 
 
@@ -1816,4 +1815,9 @@ def set_seed(seed):
 
 def is_tensor(x):
 
-    return isinstance(x, ms.Tensor)
+    return isinstance(x, ms.Tensor)
+
+def tensor_scatter_nd_update(tensor, indices, updates):
+
+    op = ms.ops.TensorScatterUpdate()
+    return op(tensor, indices, updates)

tensorlayerx/backend/ops/paddle_backend.py

@@ -824,7 +824,15 @@ def transpose(a, perm=None, conjugate=False):
     -------
         A transposed Tensor.
     """
-
+    if perm == None:
+        if len(a.shape) <= 2:
+            return pd.t(a)
+        if len(a.shape) == 3:
+            perm = [2, 1, 0]
+        if len(a.shape) == 4:
+            perm = [3, 2, 1, 0]
+        if len(a.shape) == 5:
+            perm = [4, 3, 2, 1, 0]
     return pd.transpose(a, perm)
 
 
@@ -1503,4 +1511,12 @@ def set_seed(seed):
 
 def is_tensor(x):
 
-    return pd.is_tensor(x)
+    return pd.is_tensor(x)
+
+
+def tensor_scatter_nd_update(tensor, indices, updates):
+    a = pd.scatter_nd(indices, pd.ones_like(updates), tensor.shape)
+    a = pd.multiply(tensor, -a)
+    tensor = tensor + a
+    x = pd.scatter_nd_add(tensor, indices, updates)
+    return x

tensorlayerx/backend/ops/tensorflow_backend.py

@@ -3597,4 +3597,21 @@ def is_tensor(x):
     >>> tlx.ops.is_tensor(a)
     """
 
-    return tf.is_tensor(x)
+    return tf.is_tensor(x)
+
+
+def tensor_scatter_nd_update(tensor, indices, updates):
+    """
+
+    Parameters
+    ----------
+    tensor
+    indices
+    updates
+
+    Returns
+    -------
+
+    """
+
+    return tf.tensor_scatter_nd_update(tensor, indices, updates)

tensorlayerx/backend/ops/torch_backend.py

@@ -1630,4 +1630,12 @@ def set_seed(seed):
 
 def is_tensor(x):
 
-    return isinstance(x, torch.Tensor)
+    return isinstance(x, torch.Tensor)
+
+def tensor_scatter_nd_update(tensor, indices, updates):
+
+    indices = torch.flatten(indices)
+    tensor[indices] = updates
+    return tensor
+
+

tensorlayerx/dataflow/utils.py

@@ -8,14 +8,12 @@
 import numbers
 import itertools
 import multiprocessing
-import threading
 import queue
 from collections import namedtuple
 from dataclasses import dataclass
 import sys
 import traceback
 
-
 def default_convert(data):
     data_type = type(data)
     if isinstance(data, np.ndarray):
@@ -264,7 +262,6 @@ def __init__(self, loader):
         self._persistent_workers = loader.persistent_workers
         self._time_out = loader.time_out
         self._sampler_iter = iter(self._index_sampler)
-        # self._pin_memory = loader.pin_memory
         self._num_yielded = 0
 
     def __iter__(self):
@@ -321,7 +318,6 @@ def __init__(self, loader):
         self._worker_result_queue = multiprocessing.Queue()
         self._worker_done_event = multiprocessing.Event()
         self._worker_pids_set = False
-        self._shutdown = False
 
         self._index_queues = []
         self._workers = []
@@ -357,6 +353,7 @@ def _reset(self, loader, first_iter=False):
             while resume_iteration_cnt > 0:
                 return_idx, return_data = self._get_data()
                 if isinstance(return_idx, _ResumeIteration):
+                    assert return_data is None
                     resume_iteration_cnt -= 1
         for _ in range(self._prefetch_factor * self._num_workers):
 
@@ -469,26 +466,19 @@ def _shutdown_workers(self):
         if not self._shutdown:
             self._shutdown = True
             try:
-                if hasattr(self, '_pin_memory_thread'):
-                    self._pin_memory_thread_done_event.set()
-                    self._worker_result_queue.put((None, None))
-                    self._pin_memory_thread.join()
-                    self._worker_result_queue.cancel_join_thread()
-                    self._worker_result_queue.close()
-
                 self._worker_done_event.set()
                 for worker_id in range(len(self._workers)):
                     if self._persistent_workers or self._workers_status[worker_id]:
                         self._mark_worker_as_unavailable(worker_id, shutdown=True)
                 for w in self._workers:
                     w.join(timeout=5.0)
-                    if w.is_alive():
-                        w.terminate()
                 for q in self._index_queues:
                     q.cancel_join_thread()
                     q.close()
             finally:
-                pass
+                for w in self._workers:
+                    if w.is_alive():
+                        w.terminate()
 
     def __del__(self):
         self._shutdown_workers()
@@ -565,7 +555,15 @@ def _worker_loop(
                 try:
                     data = fetcher.fetch(index)
                 except Exception as e:
-                    data = ExceptionWrapper(where="in DataLoader worker process {}".format(worker_id))
+                    if isinstance(e, StopIteration) and dataset_kind == _DatasetKind.Iter:
+                        data = _IterableDatasetStopIteration(worker_id)
+                        iteration_end = True
+                    else:
+                        # It is important that we don't store exc_info in a variable.
+                        # `ExceptionWrapper` does the correct thing.
+                        # See NOTE [ Python Traceback Reference Cycle Problem ]
+                        data = ExceptionWrapper(
+                            where="in DataLoader worker process {}".format(worker_id))
             data_queue.put((idx, data))
             del data, idx, index, r
     except KeyboardInterrupt:

tensorlayerx/files/dataset_loaders/cyclegan_dataset.py

@@ -6,7 +6,7 @@
 import numpy as np
 
 from tensorlayerx import logging
-from tensorlayerx.utils import visualize
+from tensorlayerx.vision import load_images
 from tensorlayerx.files.utils import (del_file, folder_exists, load_file_list, maybe_download_and_extract)
 
 __all__ = ['load_cyclegan_dataset']
@@ -36,8 +36,7 @@ def load_cyclegan_dataset(filename='summer2winter_yosemite', path='data'):
         del_file(os.path.join(path, filename + '.zip'))
 
     def load_image_from_folder(path):
-        path_imgs = load_file_list(path=path, regx='\\.jpg', printable=False)
-        return visualize.read_images(path_imgs, path=path, n_threads=10, printable=False)
+        return load_images(path=path, n_threads=10)
 
     im_train_A = load_image_from_folder(os.path.join(path, filename, "trainA"))
     im_train_B = load_image_from_folder(os.path.join(path, filename, "trainB"))