Merge branch 'master' of https://github.com/tensorlayer/tensorlayer

Author: warshallrho

docs/user/get_involved.rst

@@ -4,8 +4,20 @@
 Get Involved in Research
 =========================
 
+Ph.D. Postition @ PKU
+=============================================================
+
+
+Hi, I am Hao Dong, a new faculty member in EECS, Peking University. I now have a few Ph.D. positions per year open for international students who would like to study AI. If you or your friends are interested in it, please contact me ASAP.
+PKU is a top 30 university in the global ranking. The application is competitive, apply early is recommended. For the application of next year, please note that the DDL of Chinese Government Scholarship is in the end of each year, please check this `link <http://www.isd.pku.edu.cn/info/1503/5676.htm>`__ for more details.
+
+My homepage: https://zsdonghao.github.io 
+
+Contact: hao.dong11[AT]imperial.ac.uk
+
+
 
-AP Opportunity, Peking University
+Faculty Postition @ PKU
 =============================================================
 
 The Center on Frontiers of Computing Studies (CFCS), Peking University (PKU), China, is a university new initiative co-founded by Professors John Hopcroft (Turing Awardee) and Wen Gao (CAE, ACM/IEEE Fellow). The center aims at developing the excellence on two fronts: research and education. On the research front, the center will provide a world-class research environment, where innovation and impactful research is the central aim, measured by professional reputation among world scholars, not by counting the number of publications and research funding. On the education front, the center deeply involves in the Turing Class, an elite undergraduate program that draws the cream of the crop from the PKU undergraduate talent pool. New curriculum and pedagogy are designed and practiced in this program, with the aim to cultivate a new generation of computer scientist/engineers that are solid in both theories and practices. 
@@ -27,7 +39,7 @@ Application for a postdoctoral position should include a curriculum vita, brief
 We conduct review of applications monthly, immediately upon the recipient of all application materials at the beginning of each month. However, it is highly recommended that applicants submit complete applications sooner than later, as the positions are to be filled quickly. 
 
 
-Postdoc Opportunity, Imperial College London
+Postdoc Postition @ ICL
 ==================================================
 
 Data science is therefore by nature at the core of all modern transdisciplinary scientific activities, as it involves the whole life cycle of data, from acquisition and exploration to analysis and communication of the results. Data science is not only concerned with the tools and methods to obtain, manage and analyse data: it is also about extracting value from data and translating it from asset to product.
@@ -48,3 +60,17 @@ and other ways to
 `get involved <https://www.imperial.ac.uk/data-science/get-involved/>`__
 , or feel free to
 `contact us <https://www.imperial.ac.uk/data-science/get-involved/contact-us/>`__.
+
+Software [email protected]
+=============================================================
+SurgicalAI is a startup founded by the data scientists and surgical robot experts from Imperial College. Our objective is AI democratise Surgery. By combining 5G, AI and Cloud Computing, SurgicalAI is building a platform enable junor surgeons to perfom complex procedures. As one of the most impactful startup, SurgicalAI is supported by Nvidia, AWS and top surgeons around the world.
+
+Currently based in Hangzhou, China, we are building digital solution for cardiac surgery like TAVR, LAA and Orthopedidcs like TKA and UNA. A demo can be found at here <http://demo5g.surgicalai.cn>
+
+We are activly looking for experts in robotic navigation, computer graphics and  medical image analysis experts to join us, building digitalized surgical service platform for the aging world.
+
+Home Page: http://www.surgicalai.cn
+
+Demo Page: http://demo5g.surgicalai.cn
+
+Contact: [email protected]

docs/user/get_start_model.rst

@@ -19,12 +19,12 @@ Static model
   def get_model(inputs_shape):
       ni = Input(inputs_shape)
       nn = Dropout(keep=0.8)(ni)
-      nn = Dense(n_units=800, act=tf.nn.relu, name="dense1")(nn)
+      nn = Dense(n_units=800, act=tf.nn.relu, name="dense1")(nn) # “name" is optional
       nn = Dropout(keep=0.8)(nn)
       nn = Dense(n_units=800, act=tf.nn.relu)(nn)
       nn = Dropout(keep=0.8)(nn)
-      nn = Dense(n_units=10, act=tf.nn.relu)(nn)
-      M = Model(inputs=ni, outputs=nn, name="mlp")
+      nn = Dense(n_units=10, act=None)(nn)
+      M = Model(inputs=ni, outputs=nn, name="mlp") # “name" is optional
       return M
 
   MLP = get_model([None, 784])
@@ -46,10 +46,10 @@ In this case, you need to manually input the output shape of the previous layer
 
           self.dropout1 = Dropout(keep=0.8)
           self.dense1 = Dense(n_units=800, act=tf.nn.relu, in_channels=784)
-          self.dropout2 = Dropout(keep=0.8)#(self.dense1)
+          self.dropout2 = Dropout(keep=0.8)
           self.dense2 = Dense(n_units=800, act=tf.nn.relu, in_channels=800)
-          self.dropout3 = Dropout(keep=0.8)#(self.dense2)
-          self.dense3 = Dense(n_units=10, act=tf.nn.relu, in_channels=800)
+          self.dropout3 = Dropout(keep=0.8)
+          self.dense3 = Dense(n_units=10, act=None, in_channels=800)
 
       def forward(self, x, foo=False):
           z = self.dropout1(x)
@@ -59,7 +59,7 @@ In this case, you need to manually input the output shape of the previous layer
           z = self.dropout3(z)
           out = self.dense3(z)
           if foo:
-              out = tf.nn.relu(out)
+              out = tf.nn.softmax(out)
           return out
 
   MLP = CustomModel()
@@ -156,7 +156,7 @@ Print model information
   #   (dropout_1): Dropout(keep=0.8, name='dropout_1')
   #   (dense_1): Dense(n_units=800, relu, in_channels='800', name='dense_1')
   #   (dropout_2): Dropout(keep=0.8, name='dropout_2')
-  #   (dense_2): Dense(n_units=10, relu, in_channels='800', name='dense_2')
+  #   (dense_2): Dense(n_units=10, None, in_channels='800', name='dense_2')
   # )
   
   import pprint
@@ -195,7 +195,7 @@ Print model information
   #                                   'name': 'dropout_3'},
   #                          'class': 'Dropout',
   #                          'prev_layer': ['dense_2_node_0']},
-  #                         {'args': {'act': 'relu',
+  #                         {'args': {'act': None,
   #                                   'layer_type': 'normal',
   #                                   'n_units': 10,
   #                                   'name': 'dense_3'},

examples/basic_tutorials/tutorial_cifar10_cnn_static.py

@@ -1,11 +1,9 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import multiprocessing
 import time
-
 import numpy as np
-
+import multiprocessing
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import (BatchNorm, Conv2d, Dense, Flatten, Input, LocalResponseNorm, MaxPool2d)
@@ -80,14 +78,11 @@ def get_model_batchnorm(inputs_shape):
 print_freq = 5
 n_step_epoch = int(len(y_train) / batch_size)
 n_step = n_epoch * n_step_epoch
-shuffle_buffer_size = 128  # 100
-# init_learning_rate = 0.1
-# learning_rate_decay_factor = 0.1
-# num_epoch_decay = 350
+shuffle_buffer_size = 128
 
 train_weights = net.trainable_weights
-# learning_rate = tf.Variable(init_learning_rate)
 optimizer = tf.optimizers.Adam(learning_rate)
+# looking for decay learning rate? see https://github.com/tensorlayer/srgan/blob/master/train.py
 
 
 def generator_train():
@@ -182,14 +177,10 @@ def _map_fn_test(img, target):
 
     # use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         print("   train loss: {}".format(train_loss / n_iter))
         print("   train acc:  {}".format(train_acc / n_iter))
-
         net.eval()
-
         val_loss, val_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in test_ds:
             _logits = net(X_batch)  # is_train=False, disable dropout
@@ -199,10 +190,6 @@ def _map_fn_test(img, target):
         print("   val loss: {}".format(val_loss / n_iter))
         print("   val acc:  {}".format(val_acc / n_iter))
 
-    # FIXME : how to apply lr decay in eager mode?
-    # learning_rate.assign(tf.train.exponential_decay(init_learning_rate, epoch, num_epoch_decay,
-    #                                                 learning_rate_decay_factor))
-
 # use testing data to evaluate the model
 net.eval()
 test_loss, test_acc, n_iter = 0, 0, 0

examples/basic_tutorials/tutorial_mnist_mlp_dynamic.py

@@ -1,7 +1,5 @@
 import time
-
 import numpy as np
-
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import Dense, Dropout, Input
@@ -19,7 +17,6 @@ class CustomModel(Model):
 
     def __init__(self):
         super(CustomModel, self).__init__()
-
         self.dropout1 = Dropout(keep=0.8)  #(self.innet)
         self.dense1 = Dense(n_units=800, act=tf.nn.relu, in_channels=784)  #(self.dropout1)
         self.dropout2 = Dropout(keep=0.8)  #(self.dense1)
@@ -52,27 +49,20 @@ def forward(self, x, foo=None):
 for epoch in range(n_epoch):  ## iterate the dataset n_epoch times
     start_time = time.time()
     ## iterate over the entire training set once (shuffle the data via training)
-
     for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
-
         MLP.train()  # enable dropout
-
         with tf.GradientTape() as tape:
             ## compute outputs
             _logits = MLP(X_batch, foo=1)
             ## compute loss and update model
             _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
-
         grad = tape.gradient(_loss, train_weights)
         optimizer.apply_gradients(zip(grad, train_weights))
 
     ## use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         MLP.eval()  # disable dropout
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         train_loss, train_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
             _logits = MLP(X_batch, foo=1)
@@ -81,7 +71,6 @@ def forward(self, x, foo=None):
             n_iter += 1
         print("   train foo=1 loss: {}".format(train_loss / n_iter))
         print("   train foo=1 acc:  {}".format(train_acc / n_iter))
-
         val_loss, val_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=False):
             _logits = MLP(X_batch, foo=1)  # is_train=False, disable dropout
@@ -90,7 +79,6 @@ def forward(self, x, foo=None):
             n_iter += 1
         print("   val foo=1 loss: {}".format(val_loss / n_iter))
         print("   val foo=1 acc:  {}".format(val_acc / n_iter))
-
         val_loss, val_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=False):
             _logits = MLP(X_batch)  # is_train=False, disable dropout

examples/basic_tutorials/tutorial_mnist_mlp_dynamic_2.py

@@ -1,7 +1,5 @@
 import time
-
 import numpy as np
-
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import Dense, Dropout, Input, LayerList
@@ -19,17 +17,14 @@ class CustomModelHidden(Model):
 
     def __init__(self):
         super(CustomModelHidden, self).__init__()
-
         self.dropout1 = Dropout(keep=0.8)  #(self.innet)
-
         self.seq = LayerList(
             [
                 Dense(n_units=800, act=tf.nn.relu, in_channels=784),
                 Dropout(keep=0.8),
                 Dense(n_units=800, act=tf.nn.relu, in_channels=800),
             ]
         )
-
         self.dropout3 = Dropout(keep=0.8)  #(self.seq)
 
     def forward(self, x):
@@ -43,7 +38,6 @@ class CustomModelOut(Model):
 
     def __init__(self):
         super(CustomModelOut, self).__init__()
-
         self.dense3 = Dense(n_units=10, act=tf.nn.relu, in_channels=800)
 
     def forward(self, x, foo=None):
@@ -74,30 +68,23 @@ def forward(self, x, foo=None):
 for epoch in range(n_epoch):  ## iterate the dataset n_epoch times
     start_time = time.time()
     ## iterate over the entire training set once (shuffle the data via training)
-
     for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
-
         MLP1.train()  # enable dropout
         MLP2.train()
-
         with tf.GradientTape() as tape:
             ## compute outputs
             _hidden = MLP1(X_batch)
             _logits = MLP2(_hidden, foo=1)
             ## compute loss and update model
             _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
-
         grad = tape.gradient(_loss, train_weights)
         optimizer.apply_gradients(zip(grad, train_weights))
 
     ## use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         MLP1.eval()  # disable dropout
         MLP2.eval()
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         train_loss, train_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
             _hidden = MLP1(X_batch)

examples/basic_tutorials/tutorial_mnist_mlp_static.py

@@ -1,7 +1,5 @@
 import time
-
 import numpy as np
-
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import Dense, Dropout, Input
@@ -21,7 +19,7 @@
 def get_model(inputs_shape):
     ni = Input(inputs_shape)
     nn = Dropout(keep=0.8)(ni)
-    nn = Dense(n_units=800, act=tf.nn.relu)(nn)  
+    nn = Dense(n_units=800, act=tf.nn.relu)(nn)
     nn = Dropout(keep=0.8)(nn)
     nn = Dense(n_units=800, act=tf.nn.relu)(nn)
     nn = Dropout(keep=0.8)(nn)
@@ -45,31 +43,23 @@ def get_model(inputs_shape):
 for epoch in range(n_epoch):  ## iterate the dataset n_epoch times
     start_time = time.time()
     ## iterate over the entire training set once (shuffle the data via training)
-
     for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
-
         MLP.train()  # enable dropout
-
         with tf.GradientTape() as tape:
             ## compute outputs
-            _logits = MLP(X_batch)  # alternatively, you can use MLP(x, is_train=True) and remove MLP.train()
+            _logits = MLP(X_batch)
             ## compute loss and update model
             _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
-
         grad = tape.gradient(_loss, train_weights)
         optimizer.apply_gradients(zip(grad, train_weights))
 
     ## use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         MLP.eval()  # disable dropout
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         train_loss, train_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
-
-            _logits = MLP(X_batch)  # alternatively, you can use MLP(x, is_train=False) and remove MLP.eval()
+            _logits = MLP(X_batch)
             train_loss += tl.cost.cross_entropy(_logits, y_batch, name='eval_loss')
             train_acc += np.mean(np.equal(np.argmax(_logits, 1), y_batch))
             n_iter += 1

examples/basic_tutorials/tutorial_mnist_mlp_static_2.py

@@ -1,7 +1,5 @@
 import time
-
 import numpy as np
-
 import tensorflow as tf
 import tensorlayer as tl
 from tensorlayer.layers import Dense, Dropout, Input
@@ -55,27 +53,20 @@ def get_model(inputs_shape, hmodel):
 for epoch in range(n_epoch):  ## iterate the dataset n_epoch times
     start_time = time.time()
     ## iterate over the entire training set once (shuffle the data via training)
-
     for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=True):
-
         MLP.train()  # enable dropout
-
         with tf.GradientTape() as tape:
             ## compute outputs
             _logits = MLP(X_batch)  # alternatively, you can use MLP(x, is_train=True) and remove MLP.train()
             ## compute loss and update model
             _loss = tl.cost.cross_entropy(_logits, y_batch, name='train_loss')
-
         grad = tape.gradient(_loss, train_weights)
         optimizer.apply_gradients(zip(grad, train_weights))
 
     ## use training and evaluation sets to evaluate the model every print_freq epoch
     if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
-
         MLP.eval()  # disable dropout
-
         print("Epoch {} of {} took {}".format(epoch + 1, n_epoch, time.time() - start_time))
-
         train_loss, train_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_train, y_train, batch_size, shuffle=False):
 
@@ -85,7 +76,6 @@ def get_model(inputs_shape, hmodel):
             n_iter += 1
         print("   train loss: {}".format(train_loss / n_iter))
         print("   train acc:  {}".format(train_acc / n_iter))
-
         val_loss, val_acc, n_iter = 0, 0, 0
         for X_batch, y_batch in tl.iterate.minibatches(X_val, y_val, batch_size, shuffle=False):
             _logits = MLP(X_batch)  # is_train=False, disable dropout