Layer API Refactoring (#692)

* Layer API Refactoring

* Conv Layers refactored

* dense layers refactored

* Error fix in ElementwiseLayer

* Various cleaning

* docstring cleaning

* Documentation print clean

* Changelog updated

* @zsdonghao corrections added

* Import Bug Fix

Author: Jonathan DEKHTIAR

CHANGELOG.md

@@ -239,10 +239,14 @@ To release a new version, please update the changelog as followed:
 - Input Layers have been removed from `tl.layers.core` and added to `tl.layers.inputs` (by @DEKHTIARJonathan in #675)
 - Input Layers are now considered as true layers in the graph (they represent a placeholder), unittests have been updated (by @DEKHTIARJonathan in #675)
 - Layer API is simplified, with automatic feeding `prev_layer` into `self.inputs` (by @DEKHTIARJonathan in #675)
+- Convolution Layers have been refactored under the folder `tensorlayer/layers/convolution` (by @DEKHTIARJonathan in #692)
+- Dense Layers have been refactored under the folder `tensorlayer/layers/dense` (by @DEKHTIARJonathan in #692)
+- Numerous Layer utility functions have been refactored in the file `tensorlayer/layers/utils.py` (by @DEKHTIARJonathan in #692)
 
 ### Deprecated
 - `tl.layers.TimeDistributedLayer` argurment `args` is deprecated in favor of `layer_args` (by @DEKHTIARJonathan in #667)
-- `tl.act.leaky_relu` have been deprecated in favor of `tf.nn.leaky_relu` (by @DEKHTIARJonathan in #686)
+- `tl.act.leaky_relu` has been deprecated in favor of `tf.nn.leaky_relu` (by @DEKHTIARJonathan in #686)
+- `tl.layers.initialize_global_variables` has been deprecated in favor of `tf.global_variables_initializer` (by @DEKHTIARJonathan in #692)
 
 ### Removed
 - `assert()` calls remove and replaced by `raise AssertionError()` (by @DEKHTIARJonathan in #667)

example/tutorial_imagenet_inceptionV3_distributed.py

@@ -408,12 +408,12 @@ def run_worker(task_spec, checkpoints_path, batch_size=32, epochs=10):
     logging.basicConfig(level=logging.INFO, format='%(asctime)-15s %(message)s')
 
     if not tf.gfile.Exists(ILSVRC_DIR):
-        logging.error('We couldn\'t find the directory "{}"'.format(ILSVRC_DIR))
-        logging.error('You need to modify the variable BASE_DIR with the path where the dataset is.')
-        logging.error(
-            'The dataset can be downloaded from http://www.image-net.org/ or from the Kaggle competition: https://www.kaggle.com/c/imagenet-object-localization-challenge/data'
+        raise FileNotFoundError(
+            'We cannot find the directory "{}"\n'
+            'You need to modify the variable BASE_DIR with the path where the dataset is.\n'
+            'The dataset can be downloaded from http://www.image-net.org/ or from the Kaggle competition:\n'
+            'https://www.kaggle.com/c/imagenet-object-localization-challenge/data'.format(ILSVRC_DIR)
         )
-        exit(-1)
 
     # args
     parser = argparse.ArgumentParser()

tensorlayer/__init__.py

@@ -32,6 +32,7 @@
     from . import decorators
     from . import distributed
     from . import files
+    from . import initializers
     from . import iterate
     from . import layers
     from . import lazy_imports

tensorlayer/cost.py

@@ -390,7 +390,7 @@ def cross_entropy_seq(logits, target_seqs, batch_size=None):  #, batch_size=1, n
     >>> targets = tf.placeholder(tf.int32, [batch_size, n_steps])
     >>> # build the network
     >>> print(net.outputs)
-    ... (batch_size * n_steps, n_classes)
+    (batch_size * n_steps, n_classes)
     >>> cost = tl.cost.cross_entropy_seq(network.outputs, targets)
 
     """
@@ -445,10 +445,10 @@ def cross_entropy_seq_with_mask(logits, target_seqs, input_mask, return_details=
     ...         return_seq_2d = True,
     ...         name = 'dynamicrnn')
     >>> print(net.outputs)
-    ... (?, 256)
+    (?, 256)
     >>> net = tl.layers.DenseLayer(net, n_units=vocab_size, name="output")
     >>> print(net.outputs)
-    ... (?, 10000)
+    (?, 10000)
     >>> loss = tl.cost.cross_entropy_seq_with_mask(net.outputs, target_seqs, input_mask)
 
     """

tensorlayer/db.py

@@ -367,9 +367,9 @@ def save_job(self, script=None, args=None):
         >>> # Run your job
         >>> temp = db.find_one_job(args={'job_id': 1})
         >>> print(temp['learning_rate'])
-        ... 0.01
+        0.01
         >>> import _your_script
-        ... running your script
+        running your script
         """
 
         if args is None:

tensorlayer/files/dataset_loaders/imdb_dataset.py

@@ -40,9 +40,9 @@ def load_imdb_dataset(
     >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(
     ...                                 nb_words=20000, test_split=0.2)
     >>> print('X_train.shape', X_train.shape)
-    ... (20000,)  [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
+    (20000,)  [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
     >>> print('y_train.shape', y_train.shape)
-    ... (20000,)  [1 0 0 ..., 1 0 1]
+    (20000,)  [1 0 0 ..., 1 0 1]
 
     References
     -----------

tensorlayer/files/dataset_loaders/voc_dataset.py

@@ -67,24 +67,24 @@ def load_voc_dataset(path='data', dataset='2012', contain_classes_in_person=Fals
     >>>     n_objs_list, objs_info_list, objs_info_dicts = tl.files.load_voc_dataset(dataset="2012", contain_classes_in_person=False)
     >>> idx = 26
     >>> print(classes)
-    ... ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
+    ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
     >>> print(classes_dict)
-    ... {'sheep': 16, 'horse': 12, 'bicycle': 1, 'bottle': 4, 'cow': 9, 'sofa': 17, 'car': 6, 'dog': 11, 'cat': 7, 'person': 14, 'train': 18, 'diningtable': 10, 'aeroplane': 0, 'bus': 5, 'pottedplant': 15, 'tvmonitor': 19, 'chair': 8, 'bird': 2, 'boat': 3, 'motorbike': 13}
+    {'sheep': 16, 'horse': 12, 'bicycle': 1, 'bottle': 4, 'cow': 9, 'sofa': 17, 'car': 6, 'dog': 11, 'cat': 7, 'person': 14, 'train': 18, 'diningtable': 10, 'aeroplane': 0, 'bus': 5, 'pottedplant': 15, 'tvmonitor': 19, 'chair': 8, 'bird': 2, 'boat': 3, 'motorbike': 13}
     >>> print(imgs_file_list[idx])
-    ... data/VOC/VOC2012/JPEGImages/2007_000423.jpg
+    data/VOC/VOC2012/JPEGImages/2007_000423.jpg
     >>> print(n_objs_list[idx])
-    ... 2
+    2
     >>> print(imgs_ann_file_list[idx])
-    ... data/VOC/VOC2012/Annotations/2007_000423.xml
+    data/VOC/VOC2012/Annotations/2007_000423.xml
     >>> print(objs_info_list[idx])
-    ... 14 0.173 0.461333333333 0.142 0.496
-    ... 14 0.828 0.542666666667 0.188 0.594666666667
+    14 0.173 0.461333333333 0.142 0.496
+    14 0.828 0.542666666667 0.188 0.594666666667
     >>> ann = tl.prepro.parse_darknet_ann_str_to_list(objs_info_list[idx])
     >>> print(ann)
-    ... [[14, 0.173, 0.461333333333, 0.142, 0.496], [14, 0.828, 0.542666666667, 0.188, 0.594666666667]]
+    [[14, 0.173, 0.461333333333, 0.142, 0.496], [14, 0.828, 0.542666666667, 0.188, 0.594666666667]]
     >>> c, b = tl.prepro.parse_darknet_ann_list_to_cls_box(ann)
     >>> print(c, b)
-    ... [14, 14] [[0.173, 0.461333333333, 0.142, 0.496], [0.828, 0.542666666667, 0.188, 0.594666666667]]
+    [14, 14] [[0.173, 0.461333333333, 0.142, 0.496], [0.828, 0.542666666667, 0.188, 0.594666666667]]
 
     References
     -------------

tensorlayer/files/utils.py

@@ -532,9 +532,9 @@ def load_imdb_dataset(
     >>> X_train, y_train, X_test, y_test = tl.files.load_imdb_dataset(
     ...                                 nb_words=20000, test_split=0.2)
     >>> print('X_train.shape', X_train.shape)
-    ... (20000,)  [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
+    (20000,)  [[1, 62, 74, ... 1033, 507, 27],[1, 60, 33, ... 13, 1053, 7]..]
     >>> print('y_train.shape', y_train.shape)
-    ... (20000,)  [1 0 0 ..., 1 0 1]
+    (20000,)  [1 0 0 ..., 1 0 1]
 
     References
     -----------
@@ -1046,24 +1046,24 @@ def load_voc_dataset(path='data', dataset='2012', contain_classes_in_person=Fals
     >>>     n_objs_list, objs_info_list, objs_info_dicts = tl.files.load_voc_dataset(dataset="2012", contain_classes_in_person=False)
     >>> idx = 26
     >>> print(classes)
-    ... ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
+    ['aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor']
     >>> print(classes_dict)
-    ... {'sheep': 16, 'horse': 12, 'bicycle': 1, 'bottle': 4, 'cow': 9, 'sofa': 17, 'car': 6, 'dog': 11, 'cat': 7, 'person': 14, 'train': 18, 'diningtable': 10, 'aeroplane': 0, 'bus': 5, 'pottedplant': 15, 'tvmonitor': 19, 'chair': 8, 'bird': 2, 'boat': 3, 'motorbike': 13}
+    {'sheep': 16, 'horse': 12, 'bicycle': 1, 'bottle': 4, 'cow': 9, 'sofa': 17, 'car': 6, 'dog': 11, 'cat': 7, 'person': 14, 'train': 18, 'diningtable': 10, 'aeroplane': 0, 'bus': 5, 'pottedplant': 15, 'tvmonitor': 19, 'chair': 8, 'bird': 2, 'boat': 3, 'motorbike': 13}
     >>> print(imgs_file_list[idx])
-    ... data/VOC/VOC2012/JPEGImages/2007_000423.jpg
+    data/VOC/VOC2012/JPEGImages/2007_000423.jpg
     >>> print(n_objs_list[idx])
-    ... 2
+    2
     >>> print(imgs_ann_file_list[idx])
-    ... data/VOC/VOC2012/Annotations/2007_000423.xml
+    data/VOC/VOC2012/Annotations/2007_000423.xml
     >>> print(objs_info_list[idx])
-    ... 14 0.173 0.461333333333 0.142 0.496
-    ... 14 0.828 0.542666666667 0.188 0.594666666667
+    14 0.173 0.461333333333 0.142 0.496
+    14 0.828 0.542666666667 0.188 0.594666666667
     >>> ann = tl.prepro.parse_darknet_ann_str_to_list(objs_info_list[idx])
     >>> print(ann)
-    ... [[14, 0.173, 0.461333333333, 0.142, 0.496], [14, 0.828, 0.542666666667, 0.188, 0.594666666667]]
+    [[14, 0.173, 0.461333333333, 0.142, 0.496], [14, 0.828, 0.542666666667, 0.188, 0.594666666667]]
     >>> c, b = tl.prepro.parse_darknet_ann_list_to_cls_box(ann)
     >>> print(c, b)
-    ... [14, 14] [[0.173, 0.461333333333, 0.142, 0.496], [0.828, 0.542666666667, 0.188, 0.594666666667]]
+    [14, 14] [[0.173, 0.461333333333, 0.142, 0.496], [0.828, 0.542666666667, 0.188, 0.594666666667]]
 
     References
     -------------
@@ -1901,7 +1901,7 @@ def save_any_to_npy(save_dict=None, name='file.npy'):
     >>> tl.files.save_any_to_npy(save_dict={'data': ['a','b']}, name='test.npy')
     >>> data = tl.files.load_npy_to_any(name='test.npy')
     >>> print(data)
-    ... {'data': ['a','b']}
+    {'data': ['a','b']}
 
     """
     if save_dict is None:
@@ -2124,9 +2124,9 @@ def natural_keys(text):
     ----------
     >>> l = ['im1.jpg', 'im31.jpg', 'im11.jpg', 'im21.jpg', 'im03.jpg', 'im05.jpg']
     >>> l.sort(key=tl.files.natural_keys)
-    ... ['im1.jpg', 'im03.jpg', 'im05', 'im11.jpg', 'im21.jpg', 'im31.jpg']
+    ['im1.jpg', 'im03.jpg', 'im05', 'im11.jpg', 'im21.jpg', 'im31.jpg']
     >>> l.sort() # that is what we dont want
-    ... ['im03.jpg', 'im05', 'im1.jpg', 'im11.jpg', 'im21.jpg', 'im31.jpg']
+    ['im03.jpg', 'im05', 'im1.jpg', 'im11.jpg', 'im21.jpg', 'im31.jpg']
 
     References
     ----------

tensorlayer/initializers.py

@@ -0,0 +1,81 @@
+#! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import tensorflow as tf
+
+from tensorlayer.layers.core import LayersConfig
+
+__all__ = ['deconv2d_bilinear_upsampling_initializer']
+
+
+def deconv2d_bilinear_upsampling_initializer(shape):
+    """Returns the initializer that can be passed to DeConv2dLayer for initializing the
+    weights in correspondence to channel-wise bilinear up-sampling.
+    Used in segmentation approaches such as [FCN](https://arxiv.org/abs/1605.06211)
+
+    Parameters
+    ----------
+    shape : tuple of int
+        The shape of the filters, [height, width, output_channels, in_channels].
+        It must match the shape passed to DeConv2dLayer.
+
+    Returns
+    -------
+    ``tf.constant_initializer``
+        A constant initializer with weights set to correspond to per channel bilinear upsampling
+        when passed as W_int in DeConv2dLayer
+
+    Examples
+    --------
+    - Upsampling by a factor of 2, ie e.g 100->200
+    >>> import tensorflow as tf
+    >>> import tensorlayer as tl
+    >>> rescale_factor = 2
+    >>> imsize = 128
+    >>> num_channels = 3
+    >>> filter_shape = (5, 5)
+    >>> filter_size = (2 * rescale_factor - rescale_factor % 2) #Corresponding bilinear filter size
+    >>> num_in_channels = 3
+    >>> num_out_channels = 3
+    >>> deconv_filter_shape = (filter_size, filter_size, num_out_channels, num_in_channels)
+    >>> x = tf.placeholder(tf.float32, (1, imsize, imsize, num_channels))
+    >>> net = tl.layers.InputLayer(x, name='input_layer')
+    >>> bilinear_init = deconv2d_bilinear_upsampling_initializer(shape=filter_shape)
+    >>> net = tl.layers.DeConv2dLayer(net,
+    ...                    shape=filter_shape,
+    ...                    output_shape=(1, imsize*rescale_factor, imsize*rescale_factor, num_out_channels),
+    ...                    strides=(1, rescale_factor, rescale_factor, 1),
+    ...                    W_init=bilinear_init,
+    ...                    padding='SAME',
+    ...                    act=None, name='g/h1/decon2d')
+
+    """
+    if shape[0] != shape[1]:
+        raise Exception('deconv2d_bilinear_upsampling_initializer only supports symmetrical filter sizes')
+
+    if shape[3] < shape[2]:
+        raise Exception(
+            'deconv2d_bilinear_upsampling_initializer behaviour is not defined for num_in_channels < num_out_channels '
+        )
+
+    filter_size = shape[0]
+    num_out_channels = shape[2]
+    num_in_channels = shape[3]
+
+    # Create bilinear filter kernel as numpy array
+    bilinear_kernel = np.zeros([filter_size, filter_size], dtype=np.float32)
+    scale_factor = (filter_size + 1) // 2
+    if filter_size % 2 == 1:
+        center = scale_factor - 1
+    else:
+        center = scale_factor - 0.5
+    for x in range(filter_size):
+        for y in range(filter_size):
+            bilinear_kernel[x, y] = (1 - abs(x - center) / scale_factor) * (1 - abs(y - center) / scale_factor)
+    weights = np.zeros((filter_size, filter_size, num_out_channels, num_in_channels))
+    for i in range(num_out_channels):
+        weights[:, :, i, i] = bilinear_kernel
+
+    # assign numpy array to constant_initalizer and pass to get_variable
+    return tf.constant_initializer(value=weights, dtype=LayersConfig.tf_dtype)