Skip to content

Commit f26de08

Browse files
zsdonghaozsdonghao
committed
DEBUG - BatchNormLayer
1 parent f53ec4a commit f26de08

File tree

2 files changed

+87
-55
lines changed

2 files changed

+87
-55
lines changed

‎example/tutorial_cifar10_tfrecord.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -246,26 +246,25 @@ def inference_batch_norm(x_crop, y_, reuse, is_train):
246246
with tf.variable_scope("model", reuse=reuse):
247247
tl.layers.set_name_reuse(reuse)
248248
network = tl.layers.InputLayer(x_crop, name='input_layer')
249+
249250
network = tl.layers.Conv2dLayer(network, act=tf.identity,
250251
shape=[5, 5, 3, 64], strides=[1, 1, 1, 1], padding='SAME', # 64 features for each 5x5x3 patch
251252
W_init=W_init, b_init=None, name='cnn_layer1') # output: (batch_size, 24, 24, 64)
252253
network = tl.layers.BatchNormLayer(network, is_train=is_train,
253254
act=tf.nn.relu, name='batch_norm1')
254-
255255
network = tl.layers.PoolLayer(network, ksize=[1, 3, 3, 1],
256256
strides=[1, 2, 2, 1], padding='SAME',
257257
pool=tf.nn.max_pool, name='pool_layer1',) # output: (batch_size, 12, 12, 64)
258258

259259
network = tl.layers.Conv2dLayer(network, act=tf.identity,
260260
shape=[5, 5, 64, 64], strides=[1, 1, 1, 1], padding='SAME',# 64 features for each 5x5 patch
261261
W_init=W_init, b_init=None, name ='cnn_layer2') # output: (batch_size, 12, 12, 64)
262-
263262
network = tl.layers.BatchNormLayer(network, is_train=is_train,
264263
act=tf.nn.relu, name='batch_norm2')
265-
266264
network = tl.layers.PoolLayer(network, ksize=[1, 3, 3, 1],
267265
strides=[1, 2, 2, 1], padding='SAME',
268266
pool = tf.nn.max_pool, name ='pool_layer2') # output: (batch_size, 6, 6, 64)
267+
269268
network = tl.layers.FlattenLayer(network, name='flatten_layer') # output: (batch_size, 2304)
270269
network = tl.layers.DenseLayer(network, n_units=384, act=tf.nn.relu,
271270
W_init=W_init2, b_init=b_init2, name='relu1') # output: (batch_size, 384)

‎tensorlayer/layers.py‎

Lines changed: 85 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -868,10 +868,10 @@ def __init__(
868868

869869
# The name of placeholder for keep_prob is the same with the name
870870
# of the Layer.
871-
set_keep[name] = tf.placeholder(tf.float32)
872871
if is_fix:
873872
self.outputs = tf.nn.dropout(self.inputs, keep, name=name)
874873
else:
874+
set_keep[name] = tf.placeholder(tf.float32)
875875
self.outputs = tf.nn.dropout(self.inputs, set_keep[name], name=name) # 1.2
876876

877877
self.all_layers = list(layer.all_layers)
@@ -1688,6 +1688,7 @@ def __init__(
16881688
self.all_drop = dict(layer.all_drop)
16891689
self.all_layers.extend( [self.outputs] )
16901690

1691+
16911692
class BatchNormLayer(Layer):
16921693
"""
16931694
The :class:`BatchNormLayer` class is a normalization layer, see ``tf.nn.batch_normalization`` and ``tf.nn.moments``.
@@ -1725,7 +1726,8 @@ def __init__(
17251726
act = tf.identity,
17261727
is_train = None,
17271728
beta_init = tf.zeros_initializer,
1728-
gamma_init = tf.ones_initializer,
1729+
# gamma_init = tf.ones_initializer,
1730+
gamma_init = tf.random_normal_initializer(mean=1.0, stddev=0.002),
17291731
name ='batchnorm_layer',
17301732
):
17311733
Layer.__init__(self, name=name)
@@ -1735,26 +1737,26 @@ def __init__(
17351737
x_shape = self.inputs.get_shape()
17361738
params_shape = x_shape[-1:]
17371739

1738-
def _get_variable(name,
1739-
shape,
1740-
initializer,
1741-
weight_decay=0.0,
1742-
dtype='float',
1743-
trainable=True):
1744-
"A little wrapper around tf.get_variable to do weight decay and add to"
1745-
"resnet collection"
1746-
if weight_decay > 0:
1747-
regularizer = tf.contrib.layers.l2_regularizer(weight_decay)
1748-
else:
1749-
regularizer = None
1750-
# collections = [TF_GRAPHKEYS_VARIABLES, RESNET_VARIABLES]
1751-
return tf.get_variable(name,
1752-
shape=shape,
1753-
initializer=initializer,
1754-
dtype=dtype,
1755-
regularizer=regularizer,
1756-
# collections=collections,
1757-
trainable=trainable)
1740+
# def _get_variable(name,
1741+
# shape,
1742+
# initializer,
1743+
# weight_decay=0.0,
1744+
# dtype='float',
1745+
# trainable=True):
1746+
# "A little wrapper around tf.get_variable to do weight decay and add to"
1747+
# "resnet collection"
1748+
# if weight_decay > 0:
1749+
# regularizer = tf.contrib.layers.l2_regularizer(weight_decay)
1750+
# else:
1751+
# regularizer = None
1752+
# # collections = [TF_GRAPHKEYS_VARIABLES, RESNET_VARIABLES]
1753+
# return tf.get_variable(name,
1754+
# shape=shape,
1755+
# initializer=initializer,
1756+
# dtype=dtype,
1757+
# regularizer=regularizer,
1758+
# # collections=collections,
1759+
# trainable=trainable)
17581760

17591761
from tensorflow.python.training import moving_averages
17601762
from tensorflow.python.ops import control_flow_ops
@@ -1767,42 +1769,69 @@ def _get_variable(name,
17671769

17681770
axis = list(range(len(x_shape) - 1))
17691771

1770-
beta = _get_variable('beta',
1771-
params_shape,
1772-
initializer=beta_init)
1773-
try: # TF12
1774-
gamma = _get_variable('gamma',
1775-
params_shape,
1776-
initializer=gamma_init())
1777-
except: # TF11
1778-
gamma = _get_variable('gamma',
1779-
params_shape,
1780-
initializer=gamma_init)
1772+
# beta = _get_variable('beta',
1773+
# params_shape,
1774+
# initializer=beta_init)
1775+
beta = tf.get_variable('beta', shape=params_shape,
1776+
initializer=beta_init,
1777+
trainable=is_train)#, restore=restore)
1778+
# try: # TF12
1779+
# gamma = _get_variable('gamma',
1780+
# params_shape,
1781+
# initializer=gamma_init())
1782+
# except: # TF11
1783+
# gamma = _get_variable('gamma',
1784+
# params_shape,
1785+
# initializer=gamma_init)
1786+
# print("x"*100)
1787+
1788+
gamma = tf.get_variable('gamma', shape=params_shape,
1789+
initializer=gamma_init, trainable=is_train,
1790+
)#restore=restore)
17811791

17821792
# trainable=False means : it prevent TF from updating this variable
17831793
# from the gradient, we have to update this from the mean computed
17841794
# from each batch during training
1785-
moving_mean = _get_variable('moving_mean',
1786-
params_shape,
1787-
initializer=tf.zeros_initializer,
1788-
trainable=False)
1789-
try: # TF12
1790-
moving_variance = _get_variable('moving_variance',
1791-
params_shape,
1792-
initializer=tf.ones_initializer(),
1793-
trainable=False)
1794-
except: # TF11
1795-
moving_variance = _get_variable('moving_variance',
1796-
params_shape,
1797-
initializer=tf.ones_initializer,
1798-
trainable=False)
1795+
# moving_mean = _get_variable('moving_mean',
1796+
# params_shape,
1797+
# initializer=tf.zeros_initializer,
1798+
# trainable=False)
1799+
# try: # TF12
1800+
# moving_variance = _get_variable('moving_variance',
1801+
# params_shape,
1802+
# initializer=tf.ones_initializer(),
1803+
# trainable=False)
1804+
# except: # TF11
1805+
# moving_variance = _get_variable('moving_variance',
1806+
# params_shape,
1807+
# initializer=tf.ones_initializer,
1808+
# trainable=False)
1809+
1810+
moving_mean = tf.get_variable('moving_mean',
1811+
params_shape,
1812+
initializer=tf.zeros_initializer,
1813+
trainable=False,)# restore=restore)
1814+
moving_variance = tf.get_variable('moving_variance',
1815+
params_shape,
1816+
initializer=tf.constant_initializer(1.),
1817+
trainable=False,)# restore=restore)
17991818

18001819
# These ops will only be preformed when training.
18011820
mean, variance = tf.nn.moments(self.inputs, axis)
1802-
update_moving_mean = moving_averages.assign_moving_average(moving_mean,
1803-
mean, decay)
1804-
update_moving_variance = moving_averages.assign_moving_average(
1805-
moving_variance, variance, decay)
1821+
try: # TF12
1822+
update_moving_mean = moving_averages.assign_moving_average(
1823+
moving_mean, mean, decay, zero_debias=True) # if zero_debias=True, has bias
1824+
update_moving_variance = moving_averages.assign_moving_average(
1825+
moving_variance, variance, decay, zero_debias=True) # if zero_debias=True, has bias
1826+
# print("TF12 moving")
1827+
except Exception as e: # TF11
1828+
update_moving_mean = moving_averages.assign_moving_average(
1829+
moving_mean, mean, decay)
1830+
update_moving_variance = moving_averages.assign_moving_average(
1831+
moving_variance, variance, decay)
1832+
# print("TF11 moving")
1833+
1834+
18061835
# tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_mean)
18071836
# tf.add_to_collection(UPDATE_OPS_COLLECTION, update_moving_variance)
18081837

@@ -1822,7 +1851,9 @@ def mean_var_with_update():
18221851

18231852
self.outputs = act( tf.nn.batch_normalization(self.inputs, mean, variance, beta, gamma, epsilon) )
18241853
#x.set_shape(inputs.get_shape()) ??
1825-
variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name)
1854+
variables = tf.get_collection(TF_GRAPHKEYS_VARIABLES, scope=vs.name) # 8 params in TF12 if zero_debias=True
1855+
# variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=vs.name) # 2 params beta, gamma
1856+
# variables = [beta, gamma, moving_mean, moving_variance]
18261857

18271858
# print(len(variables))
18281859
# for idx, v in enumerate(variables):
@@ -1836,6 +1867,8 @@ def mean_var_with_update():
18361867
self.all_params.extend( variables )
18371868
# self.all_params.extend( [beta, gamma] )
18381869

1870+
1871+
18391872
# class BatchNormLayer(Layer):
18401873
# """
18411874
# The :class:`BatchNormLayer` class is a normalization layer, see ``tf.nn.batch_normalization``.

0 commit comments

Comments
 (0)