new prepro for sequences/ update Seq2Seq for Chatbot example

Author: zsdonghao

docs/modules/prepro.rst

@@ -59,8 +59,11 @@ Some of the code in this package are borrowed from Keras.
    erosion
 
    pad_sequences
+   remove_pad_sequences
    process_sequences
    sequences_add_start_id
+   sequences_add_end_id
+   sequences_add_end_id_after_pad
    sequences_get_mask
 
    distorted_images
@@ -185,6 +188,11 @@ Padding
 ^^^^^^^^^
 .. autofunction:: pad_sequences
 
+Remove Padding
+^^^^^^^^^^^^^^^^^
+.. autofunction:: remove_pad_sequences
+
+
 Process
 ^^^^^^^^^
 .. autofunction:: process_sequences
@@ -193,6 +201,15 @@ Add Start ID
 ^^^^^^^^^^^^^^^
 .. autofunction:: sequences_add_start_id
 
+
+Add End ID
+^^^^^^^^^^^^^^^
+.. autofunction:: sequences_add_end_id
+
+Add End ID after pad
+^^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: sequences_add_end_id_after_pad
+
 Get Mask
 ^^^^^^^^^
 .. autofunction:: sequences_get_mask

tensorlayer/layers.py

@@ -4864,10 +4864,12 @@ def __init__(
 # Seq2seq
 class Seq2Seq(Layer):
     """
-    The :class:`Seq2Seq` class is a simple :class:`DynamicRNNLayer` based Seq2seq layer,
-    both encoder and decoder are :class:`DynamicRNNLayer`, network details
-    see `Model <https://camo.githubusercontent.com/242210d7d0151cae91107ee63bff364a860db5dd/687474703a2f2f6936342e74696e797069632e636f6d2f333031333674652e706e67>`_
-    and `Sequence to Sequence Learning with Neural Networks <https://arxiv.org/abs/1409.3215>`_ .
+    The :class:`Seq2Seq` class is a Simple :class:`DynamicRNNLayer` based Seq2seq layer without using `tl.contrib.seq2seq <https://www.tensorflow.org/api_guides/python/contrib.seq2seq>`_.
+    See `Model <https://camo.githubusercontent.com/242210d7d0151cae91107ee63bff364a860db5dd/687474703a2f2f6936342e74696e797069632e636f6d2f333031333674652e706e67>`_
+    and `Sequence to Sequence Learning with Neural Networks <https://arxiv.org/abs/1409.3215>`_.
+
+    - Please check the example `Twitter Chatbot <>`_.
+    - The Author recommends users to read the source code of :class:`DynamicRNNLayer` and :class:`Seq2Seq`.
 
     Parameters
     ----------
@@ -4904,9 +4906,23 @@ class Seq2Seq(Layer):
     ------------
     outputs : a tensor
         The output of RNN decoder.
+    initial_state_encode : a tensor or StateTuple
+        Initial state of RNN encoder.
+    initial_state_decode : a tensor or StateTuple
+        Initial state of RNN decoder.
+    final_state_encode : a tensor or StateTuple
+        Final state of RNN encoder.
+    final_state_decode : a tensor or StateTuple
+        Final state of RNN decoder.
 
-    final_state : a tensor or StateTuple
-        Final state of decoder, see :class:`DynamicRNNLayer` .
+    Notes
+    --------
+    - How to feed data: `Sequence to Sequence Learning with Neural Networks <https://arxiv.org/pdf/1409.3215v3.pdf>`_
+    - input_seqs : ``['how', 'are', 'you', '<PAD_ID'>]``
+    - decode_seqs : ``['<START_ID>', 'I', 'am', 'fine', '<PAD_ID'>]``
+    - target_seqs : ``['I', 'am', 'fine', '<END_ID', '<PAD_ID'>]``
+    - target_mask : ``[1, 1, 1, 1, 0]``
+    - related functions : tl.prepro <pad_sequences, precess_sequences, sequences_add_start_id, sequences_get_mask>
 
     Examples
     ----------
@@ -4948,14 +4964,7 @@ class Seq2Seq(Layer):
     >>> y = tf.nn.softmax(net_out.outputs)
     >>> net_out.print_params(False)
 
-    Notes
-    --------
-    - How to feed data: `Sequence to Sequence Learning with Neural Networks <https://arxiv.org/pdf/1409.3215v3.pdf>`_
-    - input_seqs : ``['how', 'are', 'you', '<PAD_ID'>]``
-    - decode_seqs : ``['<START_ID>', 'I', 'am', 'fine', '<PAD_ID'>]``
-    - target_seqs : ``['I', 'am', 'fine', '<END_ID']``
-    - target_mask : ``[1, 1, 1, 1, 0]``
-    - related functions : tl.prepro <pad_sequences, precess_sequences, sequences_add_start_id, sequences_get_mask>
+
     """
     def __init__(
         self,
@@ -5018,6 +5027,10 @@ def __init__(
 
             rnn_variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name)
 
+        # Initial state
+        self.initial_state_encode = network_encode.initial_state
+        self.initial_state_decode = network_decode.initial_state
+
         # Final state
         self.final_state_encode = network_encode.final_state
         self.final_state_decode = network_decode.final_state

tensorlayer/prepro.py

@@ -1386,6 +1386,33 @@ def pad_sequences(sequences, maxlen=None, dtype='int32', padding='post', truncat
             raise ValueError('Padding type "%s" not understood' % padding)
     return x
 
+def remove_pad_sequences(sequences, pad_id=0):
+    """Remove padding.
+
+    Parameters
+    -----------
+    sequences : list of list.
+    pad_id : int.
+
+    Examples
+    ----------
+    >>> sequences = [[2,3,4,0,0], [5,1,2,3,4,0,0,0], [4,5,0,2,4,0,0,0]]
+    >>> print(remove_pad_sequences(sequences, pad_id=0))
+    ... [[2, 3, 4], [5, 1, 2, 3, 4], [4, 5, 0, 2, 4]]
+    """
+    import copy
+    sequences_out = copy.deepcopy(sequences)
+    for i in range(len(sequences)):
+        # for j in range(len(sequences[i])):
+        #     if sequences[i][j] == pad_id:
+        #         sequences_out[i] = sequences_out[i][:j]
+        #         break
+        for j in range(1, len(sequences[i])):
+            if sequences[i][-j] != pad_id:
+                sequences_out[i] = sequences_out[i][0:-j+1]
+                break
+    return sequences_out
+
 def process_sequences(sequences, end_id=0, pad_val=0, is_shorten=True, remain_end_id=False):
     """Set all tokens(ids) after END token to the padding value, and then shorten (option) it to the maximum sequence length in this batch.
 
@@ -1451,6 +1478,62 @@ def sequences_add_start_id(sequences, start_id=0, remove_last=False):
             sequences_out[i] = [start_id] + sequences[i]
     return sequences_out
 
+def sequences_add_end_id(sequences, end_id=888):
+    """Add special end token(id) in the end of each sequence.
+
+    Parameters
+    -----------
+    sequences : list of list.
+    end_id : int.
+
+    Examples
+    ---------
+    >>> sequences = [[1,2,3],[4,5,6,7]]
+    >>> print(sequences_add_end_id(sequences, end_id=999))
+    ... [[1, 2, 3, 999], [4, 5, 6, 999]]
+    """
+    sequences_out = [[] for _ in range(len(sequences))]#[[]] * len(sequences)
+    for i in range(len(sequences)):
+        sequences_out[i] = sequences[i] + [end_id]
+    return sequences_out
+
+def sequences_add_end_id_after_pad(sequences, end_id=888, pad_id=0):
+    """Add special end token(id) in the end of each sequence.
+
+    Parameters
+    -----------
+    sequences : list of list.
+    end_id : int.
+    pad_id : int.
+
+    Examples
+    ---------
+    >>> sequences = [[1,2,0,0], [1,2,3,0], [1,2,3,4]]
+    >>> print(sequences_add_end_id_after_pad(sequences, end_id=99, pad_id=0))
+    ... [[1, 2, 99, 0], [1, 2, 3, 99], [1, 2, 3, 4]]
+    """
+    # sequences_out = [[] for _ in range(len(sequences))]#[[]] * len(sequences)
+    import copy
+    sequences_out = copy.deepcopy(sequences)
+    # # add a pad to all
+    # for i in range(len(sequences)):
+    #     for j in range(len(sequences[i])):
+    #         sequences_out[i].append(pad_id)
+    # # pad -- > end
+    # max_len = 0
+    for i in range(len(sequences)):
+        for j in range(len(sequences[i])):
+            if sequences[i][j] == pad_id:
+                sequences_out[i][j] = end_id
+                # if j > max_len:
+                #     max_len = j
+                break
+    # # remove pad if too long
+    # for i in range(len(sequences)):
+    #     for j in range(len(sequences[i])):
+    #         sequences_out[i] = sequences_out[i][:max_len+1]
+    return sequences_out
+
 def sequences_get_mask(sequences, pad_val=0):
     """Return mask for sequences.