switch to adam and set affine to false for the input batchnorm layer.

Author: csukuangfj

egs/aishell/s10/chain/model.py

@@ -136,7 +136,8 @@ def __init__(self,
             self.has_LDA = True
         else:
             logging.info('replace LDA with BatchNorm')
-            self.input_batch_norm = nn.BatchNorm1d(num_features=feat_dim * 3)
+            self.input_batch_norm = nn.BatchNorm1d(num_features=feat_dim * 3,
+                                                   affine=False)
             self.has_LDA = False
 
     def forward(self, x):
@@ -218,9 +219,9 @@ def constrain_orthonormal(self):
 
 if __name__ == '__main__':
     logging.basicConfig(
-                level=logging.DEBUG,
-                format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
-            )
+        level=logging.DEBUG,
+        format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
+    )
     feat_dim = 43
     output_dim = 4344
     model = ChainModel(feat_dim=feat_dim, output_dim=output_dim)

egs/aishell/s10/chain/tdnnf_layer.py

@@ -187,13 +187,11 @@ def forward(self, x):
 
         # save it for skip connection
         input_x = x
-        logging.debug(f"input_x shape is {input_x.shape}")
+
         x = self.linear(x)
-        logging.debug(f"x shape after linear is {x.shape}")
         # at this point, x is [N, C, T]
 
         x = self.affine(x)
-        logging.debug(f"x shape after affine is {x.shape}")
         # at this point, x is [N, C, T]
 
         x = F.relu(x)
@@ -298,7 +296,6 @@ def _test_factorized_tdnn():
     assert y.size(2) == math.ceil(math.ceil((T - 3)) - 3)
 
 
-
 if __name__ == '__main__':
     torch.manual_seed(20200130)
     _test_factorized_tdnn()

egs/aishell/s10/chain/train.py

@@ -183,12 +183,11 @@ def main():
                                     egs_left_context=args.egs_left_context,
                                     egs_right_context=args.egs_right_context)
 
-    optimizer = optim.SGD(model.parameters(),
-                          lr=learning_rate,
-                          momentum=0.9,
-                          weight_decay=args.l2_regularize)
+    optimizer = optim.Adam(model.parameters(),
+                           lr=learning_rate,
+                           weight_decay=args.l2_regularize)
 
-    scheduler = MultiStepLR(optimizer, milestones=[1, 3, 5], gamma=0.5)
+    scheduler = MultiStepLR(optimizer, milestones=[1, 2, 3, 4, 5], gamma=0.5)
     criterion = KaldiChainObjfFunction.apply
 
     tf_writer = SummaryWriter(log_dir='{}/tensorboard'.format(args.dir))