[maintain] for TF 0.12.1, check examples

Author: zsdonghao

README.md

@@ -92,8 +92,8 @@ train_params = network.all_params
 train_op = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.9, beta2=0.999,
                             epsilon=1e-08, use_locking=False).minimize(cost, var_list=train_params)
 
-# Initialize all variables
-sess.run(tf.initialize_all_variables())
+# Initialize all variables in the session
+tl.layers.initialize_global_variables(sess)
 
 # Print network information
 network.print_params()

docs/modules/layers.rst

@@ -20,7 +20,7 @@ All TensorLayer layers have a number of properties in common:
 
 All TensorLayer layers have a number of methods in common:
 
- - ``layer.print_params()`` : print the network variables information in order (after ``sess.run(tf.initialize_all_variables())``). alternatively, print all variables by ``tl.layers.print_all_variables()``.
+ - ``layer.print_params()`` : print the network variables information in order (after ``tl.layers.initialize_global_variables(sess)``). alternatively, print all variables by ``tl.layers.print_all_variables()``.
  - ``layer.print_layers()`` : print the network layers information in order.
  - ``layer.count_params()`` : print the number of parameters in the network.
 
@@ -71,7 +71,7 @@ To count the number of parameters in a network, run ``network.count_params()``.
   train_op = tf.train.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.999,
                               epsilon=1e-08, use_locking=False).minimize(cost, var_list = train_params)
 
-  sess.run(tf.initialize_all_variables())
+  tl.layers.initialize_global_variables(sess)
 
   network.print_params()
   network.print_layers()
@@ -260,6 +260,7 @@ Layer list
    get_variables_with_name
    set_name_reuse
    print_all_variables
+   initialize_global_variables
 
    Layer
 
@@ -313,7 +314,7 @@ Layer list
    MultiplexerLayer
 
    EmbeddingAttentionSeq2seqWrapper
-   
+
    flatten_reshape
    clear_layers_name
    initialize_rnn_state
@@ -338,10 +339,15 @@ Print variables
 ^^^^^^^^^^^^^^^^^^
 .. autofunction:: print_all_variables
 
+Initialize variables
+^^^^^^^^^^^^^^^^^^^^^^
+.. autofunction:: initialize_global_variables
+
 Basic layer
 -----------
 .. autoclass:: Layer
 
+
 Input layer
 ------------
 .. autoclass:: InputLayer

docs/user/tutorial.rst

@@ -88,8 +88,8 @@ TensorFlow's methods like ``sess.run()``, see ``tutorial_mnist.py`` for more det
   train_op = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.9, beta2=0.999,
                               epsilon=1e-08, use_locking=False).minimize(cost, var_list=train_params)
 
-  # initialize all variables
-  sess.run(tf.initialize_all_variables())
+  # initialize all variables in the session
+  tl.layers.initialize_global_variables(sess)
 
   # print network information
   network.print_params()
@@ -432,6 +432,7 @@ max-pooling of factor 2 in both dimensions. And then apply a ``Conv2dLayer`` wit
 the 4D output to 1D vector by using ``FlattenLayer``, and apply a dropout with 50%
 to last hidden layer. The ``?`` represents arbitrary batch_size.
 
+Note, ``tutorial_mnist.py`` introduces the simplified CNN API for beginner.
 
 .. code-block:: python
 
@@ -949,7 +950,7 @@ directories as follow.
                   embedding_size = embedding_size,
                   name ='embedding_layer')
 
-  sess.run(tf.initialize_all_variables())
+  tl.layers.initialize_global_variables(sess)
 
   tl.files.assign_params(sess, [load_params[0]], emb_net)
 

example/tutorial_atari_pong.py

@@ -76,8 +76,9 @@ def prepro(I):
 train_op = tf.train.RMSPropOptimizer(learning_rate, decay_rate).minimize(loss)
 
 with tf.Session() as sess:
-    init = tf.initialize_all_variables()
-    sess.run(init)
+    # init = tf.initialize_all_variables()
+    # sess.run(init)
+    tl.layers.initialize_global_variables(sess)
     if resume:
         load_params = tl.files.load_npz(name=model_file_name+'.npz')
         tl.files.assign_params(sess, load_params, network)

example/tutorial_cifar10_tfrecord.py

@@ -182,54 +182,43 @@ def read_and_decode(filename, is_train=None):
                                                     num_threads=32)
 
     def inference(x_crop, y_, reuse):
+        """
+        For simplified CNN API, check tensorlayer.org
+        """
+        W_init = tf.truncated_normal_initializer(stddev=5e-2)
+        b_init = tf.constant_initializer(value=0.0)
+        W_init2 = tf.truncated_normal_initializer(stddev=0.04)
+        b_init2 = tf.constant_initializer(value=0.1)
         with tf.variable_scope("model", reuse=reuse):
             tl.layers.set_name_reuse(reuse)
             network = tl.layers.InputLayer(x_crop, name='input_layer')
-            network = tl.layers.Conv2dLayer(network,
-                                act = tf.nn.relu,
-                                shape = [5, 5, 3, 64],  # 64 features for each 5x5x3 patch
-                                strides=[1, 1, 1, 1],
-                                padding='SAME',
-                                W_init=tf.truncated_normal_initializer(stddev=5e-2),
-                                b_init=tf.constant_initializer(value=0.0),
-                                name ='cnn_layer1')     # output: (batch_size, 24, 24, 64)
-            network = tl.layers.PoolLayer(network,
-                                ksize=[1, 3, 3, 1],
-                                strides=[1, 2, 2, 1],
-                                padding='SAME',
-                                pool = tf.nn.max_pool,
-                                name ='pool_layer1',)   # output: (batch_size, 12, 12, 64)
+            network = tl.layers.Conv2dLayer(network, act=tf.nn.relu,
+                        shape=[5, 5, 3, 64], strides=[1, 1, 1, 1], padding='SAME', # 64 features for each 5x5x3 patch
+                        W_init=W_init, b_init=b_init, name ='cnn_layer1')       # output: (batch_size, 24, 24, 64)
+            network = tl.layers.PoolLayer(network, ksize=[1, 3, 3, 1],
+                        strides=[1, 2, 2, 1], padding='SAME',
+                        pool = tf.nn.max_pool, name ='pool_layer1',)            # output: (batch_size, 12, 12, 64)
+            # you can also use tl.layers.LocalResponseNormLayer
             network.outputs = tf.nn.lrn(network.outputs, 4, bias=1.0, alpha=0.001 / 9.0,
                                                             beta=0.75, name='norm1')
-            network = tl.layers.Conv2dLayer(network,
-                                act = tf.nn.relu,
-                                shape = [5, 5, 64, 64], # 64 features for each 5x5 patch
-                                strides=[1, 1, 1, 1],
-                                padding='SAME',
-                                W_init=tf.truncated_normal_initializer(stddev=5e-2),
-                                b_init=tf.constant_initializer(value=0.1),
-                                name ='cnn_layer2')     # output: (batch_size, 12, 12, 64)
+
+            network = tl.layers.Conv2dLayer(network, act=tf.nn.relu,
+                        shape=[5, 5, 64, 64], strides=[1, 1, 1, 1], padding='SAME',# 64 features for each 5x5 patch
+                        W_init=W_init, b_init=b_init, name ='cnn_layer2')       # output: (batch_size, 12, 12, 64)
             network.outputs = tf.nn.lrn(network.outputs, 4, bias=1.0, alpha=0.001 / 9.0,
                                                             beta=0.75, name='norm2')
-            network = tl.layers.PoolLayer(network,
-                                ksize=[1, 3, 3, 1],
-                                strides=[1, 2, 2, 1],
-                                padding='SAME',
-                                pool = tf.nn.max_pool,
-                                name ='pool_layer2')   # output: (batch_size, 6, 6, 64)
-            network = tl.layers.FlattenLayer(network, name='flatten_layer')      # output: (batch_size, 2304)
-            network = tl.layers.DenseLayer(network, n_units=384, act = tf.nn.relu,
-                                W_init=tf.truncated_normal_initializer(stddev=0.04),
-                                b_init=tf.constant_initializer(value=0.1),
-                                name='relu1')       # output: (batch_size, 384)
-            network = tl.layers.DenseLayer(network, n_units=192, act = tf.nn.relu,
-                                W_init=tf.truncated_normal_initializer(stddev=0.04),
-                                b_init=tf.constant_initializer(value=0.1),
-                                name='relu2')       # output: (batch_size, 192)
-            network = tl.layers.DenseLayer(network, n_units=10, act = tf.identity,
-                                W_init=tf.truncated_normal_initializer(stddev=1/192.0),
-                                b_init = tf.constant_initializer(value=0.0),
-                                name='output_layer')    # output: (batch_size, 10)
+            network = tl.layers.PoolLayer(network, ksize=[1, 3, 3, 1],
+                        strides=[1, 2, 2, 1], padding='SAME',
+                        pool = tf.nn.max_pool, name ='pool_layer2')             # output: (batch_size, 6, 6, 64)
+            network = tl.layers.FlattenLayer(network, name='flatten_layer')     # output: (batch_size, 2304)
+            network = tl.layers.DenseLayer(network, n_units=384, act=tf.nn.relu,
+                        W_init=W_init2, b_init=b_init2, name='relu1')           # output: (batch_size, 384)
+            network = tl.layers.DenseLayer(network, n_units=192, act=tf.nn.relu,
+                        W_init=W_init2, b_init=b_init2, name='relu2')           # output: (batch_size, 192)
+            network = tl.layers.DenseLayer(network, n_units=10, act=tf.identity,
+                        W_init=tf.truncated_normal_initializer(stddev=1/192.0),
+                        b_init = tf.constant_initializer(value=0.0),
+                        name='output_layer')    # output: (batch_size, 10)
             y = network.outputs
 
             ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(y, y_))
@@ -248,59 +237,44 @@ def inference_batch_norm(x_crop, y_, reuse, is_train):
         """
         For batch normalization, the normalization should be placed after cnn
         with linear activation.
+
+        For simplified CNN API, check tensorlayer.org
         """
+        W_init = tf.truncated_normal_initializer(stddev=5e-2)
+        W_init2 = tf.truncated_normal_initializer(stddev=0.04)
+        b_init2 = tf.constant_initializer(value=0.1)
         with tf.variable_scope("model", reuse=reuse):
             tl.layers.set_name_reuse(reuse)
             network = tl.layers.InputLayer(x_crop, name='input_layer')
-            network = tl.layers.Conv2dLayer(network,
-                                act = tf.identity,
-                                shape = [5, 5, 3, 64],  # 64 features for each 5x5x3 patch
-                                strides=[1, 1, 1, 1],
-                                padding='SAME',
-                                W_init=tf.truncated_normal_initializer(stddev=5e-2),
-                                # b_init=tf.constant_initializer(value=0.0),
-                                b_init=None,
-                                name ='cnn_layer1')     # output: (batch_size, 24, 24, 64)
-            network = tl.layers.BatchNormLayer(network, is_train=is_train, name='batch_norm1')
-            network.outputs = tf.nn.relu(network.outputs, name='relu1')
-            network = tl.layers.PoolLayer(network,
-                                ksize=[1, 3, 3, 1],
-                                strides=[1, 2, 2, 1],
-                                padding='SAME',
-                                pool = tf.nn.max_pool,
-                                name ='pool_layer1',)   # output: (batch_size, 12, 12, 64)
-
-            network = tl.layers.Conv2dLayer(network,
-                                act = tf.identity,
-                                shape = [5, 5, 64, 64], # 64 features for each 5x5 patch
-                                strides=[1, 1, 1, 1],
-                                padding='SAME',
-                                W_init=tf.truncated_normal_initializer(stddev=5e-2),
-                                # b_init=tf.constant_initializer(value=0.1),
-                                b_init=None,
-                                name ='cnn_layer2')     # output: (batch_size, 12, 12, 64)
-
-            network = tl.layers.BatchNormLayer(network, is_train=is_train, name='batch_norm2')
-            network.outputs = tf.nn.relu(network.outputs, name='relu2')
-            network = tl.layers.PoolLayer(network,
-                                ksize=[1, 3, 3, 1],
-                                strides=[1, 2, 2, 1],
-                                padding='SAME',
-                                pool = tf.nn.max_pool,
-                                name ='pool_layer2')   # output: (batch_size, 6, 6, 64)
-            network = tl.layers.FlattenLayer(network, name='flatten_layer')    # output: (batch_size, 2304)
-            network = tl.layers.DenseLayer(network, n_units=384, act = tf.nn.relu,
-                                W_init=tf.truncated_normal_initializer(stddev=0.04),
-                                b_init=tf.constant_initializer(value=0.1),
-                                name='relu1')       # output: (batch_size, 384)
+            network = tl.layers.Conv2dLayer(network, act=tf.identity,
+                        shape=[5, 5, 3, 64], strides=[1, 1, 1, 1], padding='SAME', # 64 features for each 5x5x3 patch
+                        W_init=W_init, b_init=None, name='cnn_layer1')                            # output: (batch_size, 24, 24, 64)
+            network = tl.layers.BatchNormLayer(network, is_train=is_train,
+                        act=tf.nn.relu, name='batch_norm1')
+
+            network = tl.layers.PoolLayer(network, ksize=[1, 3, 3, 1],
+                        strides=[1, 2, 2, 1], padding='SAME',
+                        pool=tf.nn.max_pool, name='pool_layer1',)               # output: (batch_size, 12, 12, 64)
+
+            network = tl.layers.Conv2dLayer(network, act=tf.identity,
+                        shape=[5, 5, 64, 64], strides=[1, 1, 1, 1], padding='SAME',# 64 features for each 5x5 patch
+                        W_init=W_init, b_init=None, name ='cnn_layer2')         # output: (batch_size, 12, 12, 64)
+
+            network = tl.layers.BatchNormLayer(network, is_train=is_train,
+                        act=tf.nn.relu, name='batch_norm2')
+
+            network = tl.layers.PoolLayer(network, ksize=[1, 3, 3, 1],
+                        strides=[1, 2, 2, 1], padding='SAME',
+                        pool = tf.nn.max_pool, name ='pool_layer2')             # output: (batch_size, 6, 6, 64)
+            network = tl.layers.FlattenLayer(network, name='flatten_layer')     # output: (batch_size, 2304)
+            network = tl.layers.DenseLayer(network, n_units=384, act=tf.nn.relu,
+                        W_init=W_init2, b_init=b_init2, name='relu1')           # output: (batch_size, 384)
             network = tl.layers.DenseLayer(network, n_units=192, act = tf.nn.relu,
-                                W_init=tf.truncated_normal_initializer(stddev=0.04),
-                                b_init=tf.constant_initializer(value=0.1),
-                                name='relu2')       # output: (batch_size, 192)
+                        W_init=W_init2, b_init=b_init2, name='relu2')           # output: (batch_size, 192)
             network = tl.layers.DenseLayer(network, n_units=10, act = tf.identity,
-                                W_init=tf.truncated_normal_initializer(stddev=1/192.0),
-                                b_init = tf.constant_initializer(value=0.0),
-                                name='output_layer')    # output: (batch_size, 10)
+                        W_init=tf.truncated_normal_initializer(stddev=1/192.0),
+                        b_init = tf.constant_initializer(value=0.0),
+                        name='output_layer')                                    # output: (batch_size, 10)
             y = network.outputs
 
             ce = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(y, y_))
@@ -321,7 +295,7 @@ def inference_batch_norm(x_crop, y_, reuse, is_train):
     # y_ = tf.placeholder(tf.int32, shape=[batch_size,])
     # cost, acc, network = inference(x_crop, y_, None)
 
-    with tf.device('/gpu:0'):
+    with tf.device('/gpu:0'): # <-- remove it if you don't have GPU
         # network in gpu
         cost, acc, network = inference(x_train_batch, y_train_batch, None)
         cost_test, acc_test, _ = inference(x_test_batch, y_test_batch, True)
@@ -336,7 +310,7 @@ def inference_batch_norm(x_crop, y_, reuse, is_train):
     n_step_epoch = int(len(y_train)/batch_size)
     n_step = n_epoch * n_step_epoch
 
-    with tf.device('/gpu:0'):
+    with tf.device('/gpu:0'):   # <-- remove it if you don't have GPU
         # train in gpu
         train_params = network.all_params
         train_op = tf.train.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.999,
@@ -357,7 +331,6 @@ def inference_batch_norm(x_crop, y_, reuse, is_train):
 
     coord = tf.train.Coordinator()
     threads = tf.train.start_queue_runners(sess=sess, coord=coord)
-    # for step in range(n_step):
     step = 0
     for epoch in range(n_epoch):
         start_time = time.time()

example/tutorial_generate_text.py

@@ -154,7 +154,8 @@ def main_restore_embedding_layer():
                     embedding_size = embedding_size,
                     name ='embedding_layer')
 
-    sess.run(tf.initialize_all_variables())
+    # sess.run(tf.initialize_all_variables())
+    tl.layers.initialize_global_variables(sess)
 
     tl.files.assign_params(sess, [load_params[0]], emb_net)
 

example/tutorial_inceptionV3_tfslim.py

@@ -64,7 +64,7 @@ def print_prob(prob):
     return top1
 
 
-## Alexnet_v2 / All Slim nets can be merged into TensorLayer
+## Alexnet_v2 / All TF-Slim nets can be merged into TensorLayer
 # x = tf.placeholder(tf.float32, shape=[None, 299, 299, 3])
 # net_in = tl.layers.InputLayer(x, name='input_layer')
 # network = tl.layers.SlimNetsLayer(layer=net_in, slim_layer=alexnet_v2,
@@ -74,14 +74,16 @@ def print_prob(prob):
 #                                        'dropout_keep_prob' : 0.5,
 #                                        'spatial_squeeze' : True,
 #                                        'scope' : 'alexnet_v2'
-#                                         }
+#                                         },
+#                                     name='alexnet_v2'  # <-- the name should be the same with the ckpt model
 #                                     )
 # sess = tf.InteractiveSession()
-# sess.run(tf.initialize_all_variables())
+# # sess.run(tf.initialize_all_variables())
+# tl.layers.initialize_global_variables(sess)
 # network.print_params()
-# exit()
 
-# InceptionV3
+
+## InceptionV3 / All TF-Slim nets can be merged into TensorLayer
 x = tf.placeholder(tf.float32, shape=[None, 299, 299, 3])
 net_in = tl.layers.InputLayer(x, name='input_layer')
 with slim.arg_scope(inception_v3_arg_scope()):
@@ -92,7 +94,7 @@ def print_prob(prob):
                                     slim_args= {
                                              'num_classes' : 1001,
                                              'is_training' : False,
-                                            #  'dropout_keep_prob' : 0.8,   # for training
+                                            #  'dropout_keep_prob' : 0.8,       # for training
                                             #  'min_depth' : 16,
                                             #  'depth_multiplier' : 1.0,
                                             #  'prediction_fn' : slim.softmax,
@@ -102,16 +104,17 @@ def print_prob(prob):
                                             },
                                         name='InceptionV3'  # <-- the name should be the same with the ckpt model
                                         )
-saver = tf.train.Saver()
 
 sess = tf.InteractiveSession()
-sess.run(tf.initialize_all_variables())
 
-# with tf.Session() as sess:
-saver.restore(sess, "inception_v3.ckpt")    # download from https://github.com/tensorflow/models/tree/master/slim#Install
-print("Model Restored")
 network.print_params(False)
 
+saver = tf.train.Saver()
+try:    # TF12
+    saver.restore(sess, "./inception_v3.ckpt")    # download from https://github.com/tensorflow/models/tree/master/slim#Install
+except: # TF11
+    saver.restore(sess, "inception_v3.ckpt")
+print("Model Restored")
 
 from scipy.misc import imread, imresize
 y = network.outputs