Added TF collection of tuple parameters

Author: Laicheng0830

tensorlayerx/nn/core/core_mindspore.py

@@ -12,7 +12,9 @@
 import numpy
 from mindspore.common.api import _pynative_executor
 from collections import OrderedDict, abc as container_abcs
-__all__ = ['Module', 'Sequential', 'ModuleList', 'ModuleDict', 'Parameter', 'ParameterList', 'ParameterDict']
+from mindspore.nn.cell import ParameterTuple
+
+__all__ = ['Module', 'Sequential', 'ModuleList', 'ModuleDict', 'Parameter', 'ParameterList', 'ParameterDict', 'ParameterTuple']
 
 _global_layer_name_dict = {}
 _global_layer_node = []

tensorlayerx/nn/core/core_paddle.py

@@ -14,13 +14,12 @@
 from paddle.fluid.dygraph import parallel_helper
 import paddle as pd
 from collections import OrderedDict, abc as container_abcs
-import tensorlayerx as tlx
-from queue import Queue
+from paddle.nn import ParameterList as ParameterTuple
 
 _global_layer_name_dict = {}
 _global_layer_node = []
-
-__all__ = ['Module', 'Sequential', 'ModuleList', 'ModuleDict', 'Parameter', 'ParameterList', 'ParameterDict']
+# TODO Need to implement ParameterTuple
+__all__ = ['Module', 'Sequential', 'ModuleList', 'ModuleDict', 'Parameter', 'ParameterList', 'ParameterDict', 'ParameterTuple']
 
 
 class Module(Layer):
@@ -744,7 +743,6 @@ def update(self, parameters):
     def __call__(self, input):
         raise RuntimeError('ParameterDict should not be called.')
 
-
 def _valid_index(layer_num, index):
     if not isinstance(index, int):
         raise TypeError("Index {} is not int type")

tensorlayerx/nn/core/core_tensorflow.py

@@ -10,7 +10,7 @@
 import tensorflow as tf
 from tensorlayerx.nn.layers.utils import (get_variable_with_initializer, random_normal)
 
-__all__ = ['Module', 'Sequential', 'ModuleList', 'ModuleDict', 'Parameter', 'ParameterList', 'ParameterDict']
+__all__ = ['Module', 'Sequential', 'ModuleList', 'ModuleDict', 'Parameter', 'ParameterList', 'ParameterDict', 'ParameterTuple']
 
 _global_layer_name_dict = {}
 _global_layer_node = []
@@ -55,6 +55,7 @@ class Module(object):
     def __init__(self, name=None, act=None, *args, **kwargs):
         self._params = OrderedDict()
         self._layers = OrderedDict()
+        self._params_tuple = OrderedDict()
         self._params_status = OrderedDict()
         self._parameter_layout_dict = {}
         self._create_time = int(time.time() * 1e9)
@@ -145,6 +146,9 @@ def __setattr__(self, name, value):
                 raise TypeError("Expected type is Module, but got Parameter.")
             self.insert_param_to_layer(name, value)
 
+        elif isinstance(value, ParameterTuple):
+            self.set_attr_for_parameter_tuple(name, value)
+
         elif isinstance(value, Module):
             if layers is None:
                 raise AttributeError("Can not assign layers before Module.__init__() call.")
@@ -293,6 +297,27 @@ def _compute_shape(tensors):
             shape_mem = tlx.get_tensor_shape(tensors)
         return shape_mem
 
+    def set_attr_for_parameter_tuple(self, name, value):
+        """Set attr for parameter in ParameterTuple."""
+        params = self.__dict__.get('_params')
+        params_tuple = self.__dict__.get('_params_tuple')
+        if params is None:
+            raise AttributeError("For 'Module', can not assign params before Module.__init__() is called.")
+        exist_names = set("")
+
+        for item in value:
+            self.insert_param_to_layer(item.name, item, check_name=False)
+            if item.name in exist_names:
+                raise ValueError("The value {} , its name '{}' already exists.".
+                                 format(value, item.name))
+            exist_names.add(item.name)
+
+        if name in self.__dict__:
+            del self.__dict__[name]
+        if name in params:
+            del params[name]
+        params_tuple[name] = value
+
     def insert_param_to_layer(self, param_name, param, check_name=True):
         """
         Adds a parameter to the current layer.
@@ -344,6 +369,11 @@ def __getattr__(self, name):
             params_status = self.__dict__['_params_status']
             if name in params_status:
                 return params_status[name]
+        if '_params_tuple' in self.__dict__:
+            params_tuple = self.__dict__['_params_tuple']
+            if name in params_tuple:
+                para_list = params_tuple[name]
+                return para_list
         raise AttributeError("'{}' object has no attribute '{}'.".format(type(self).__name__, name))
 
     def __delattr__(self, name):
@@ -1258,6 +1288,33 @@ def __call__(self, input):
         raise RuntimeError('ParameterDict should not be called.')
 
 
+class ParameterTuple(tuple):
+    """
+    ParameterTuple for storing tuple of parameters.
+    """
+    def __new__(cls, iterable):
+        data = tuple(iterable)
+        ids = set()
+        orders = {}
+        for x in data:
+            if not isinstance(x, tf.Variable):
+                raise TypeError(f"ParameterTuple input should be `Parameter` collection."
+                                f"But got a {type(iterable)}, {iterable}")
+            if id(x) not in ids:
+                ids.add(id(x))
+                if x.name not in orders.keys():
+                    orders[x.name] = [0, x]
+                else:
+                    if isinstance(orders[x.name], list):
+                        name = x.name
+                        orders[name][1].name = name + "_" + str(0)
+                        x.name = x.name + "_" + str(1)
+                        orders[name] = 1
+                    else:
+                        orders[x.name] += 1
+                        x.name = x.name + "_" + str(orders[x.name])
+        return tuple.__new__(ParameterTuple, tuple(data))
+
 def _valid_index(layer_num, index):
     if not isinstance(index, int):
         raise TypeError("Index {} is not int type")

tensorlayerx/nn/core/core_torch.py

@@ -12,11 +12,12 @@
 from collections import OrderedDict, abc as container_abcs
 import warnings
 import tensorlayerx as tlx
+from torch.nn import ParameterList as ParameterTuple
 
 _global_layer_name_dict = {}
 _global_layer_node = []
 
-__all__ = ['Module', 'Sequential', 'ModuleList', 'ModuleDict', 'Parameter', 'ParameterList', 'ParameterDict']
+__all__ = ['Module', 'Sequential', 'ModuleList', 'ModuleDict', 'Parameter', 'ParameterList', 'ParameterDict', 'ParameterTuple']
 
 
 class Module(T_Module):
@@ -122,21 +123,21 @@ def _call_impl_tlx(self, *input, **kwargs):
 
         result = self._call_impl(*input, **kwargs)
         return result
-
-    __call__: Callable[..., Any] = _call_impl_tlx
-
-    def _named_members(self, get_members_fn, prefix='', recurse=True):
-        r"""Helper method for yielding various names + members of modules."""
-        memo = set()
-        modules = self.named_modules(prefix=prefix) if recurse else [(prefix, self)]
-        for module_prefix, module in modules:
-            members = get_members_fn(module)
-            for k, v in members:
-                if v is None or v in memo:
-                    continue
-                memo.add(v)
-                name = module.name + '/' + k
-                yield name, v
+    # # TODO RNN enabled after repair
+    # __call__: Callable[..., Any] = _call_impl_tlx
+    #
+    # def _named_members(self, get_members_fn, prefix='', recurse=True):
+    #     r"""Helper method for yielding various names + members of modules."""
+    #     memo = set()
+    #     modules = self.named_modules(prefix=prefix) if recurse else [(prefix, self)]
+    #     for module_prefix, module in modules:
+    #         members = get_members_fn(module)
+    #         for k, v in members:
+    #             if v is None or v in memo:
+    #                 continue
+    #             memo.add(v)
+    #             name = module.name + '/' + k
+    #             yield name, v
 
     @property
     def all_weights(self):

tensorlayerx/nn/layers/recurrent.py

@@ -4,7 +4,7 @@
 import numpy as np
 import tensorlayerx as tlx
 from tensorlayerx import logging
-from tensorlayerx.nn.core import Module
+from tensorlayerx.nn.core import Module, ParameterTuple
 
 __all__ = [
     'RNN',
@@ -488,7 +488,10 @@ def build(self, inputs_shape):
                             var_name='bias_hh_l{}{}'.format(layer, suffix), shape=(gate_size, ), init=_init
                         )
                     )
-
+        # self.weight_ih = ParameterTuple(self.w_ih)
+        # self.weight_hh = ParameterTuple(self.w_hh)
+        # self.bias_ih = ParameterTuple(self.b_ih)
+        # self.bias_hh = ParameterTuple(self.b_hh)
         self.rnn = tlx.ops.rnnbase(
             mode=self.mode, input_size=self.input_size, hidden_size=self.hidden_size, num_layers=self.num_layers,
             bias=self.bias, batch_first=self.batch_first, dropout=self.dropout, bidirectional=self.bidirectional,