behavior version 2: disallowing boolean optimizer (#563)

Author: JackTemaki

demos/demo-hyper-param-tuning.config

@@ -36,7 +36,7 @@ network = {
 }
 
 # training
-adam = True
+optimizer = {"class": "adam"}
 optimizer_epsilon = HyperParam(float, [1e-16, 1], log=True, default=1e-16)
 decouple_constraints = HyperParam(bool)
 learning_rate = HyperParam(float, [1e-6, 1], log=True, default=0.01)

demos/demo-returnn-as-framework.py

@@ -37,7 +37,7 @@
   },
 
   # training
-  adam=True,
+  optimizer={'class': 'adam'},
   learning_rate=0.01,
   num_epochs=5,
   debug_add_check_numerics_ops=True,

demos/demo-rhn-enwik8.config

@@ -57,7 +57,7 @@ model = "/tmp/%s/returnn/%s/model" % (get_login_username(), demo_name)
 cleanup_old_models = True
 gradient_clip = 0
 #gradient_clip_global_norm = 1.0
-adam = True
+optimizer = {"class": "adam"}
 optimizer_epsilon = 1e-8
 #debug_add_check_numerics_ops = True
 #debug_add_check_numerics_on_output = True

demos/demo-tf-att-copy.config

@@ -39,7 +39,7 @@ network = {
     "output": {"class": "softmax", "from": ["decoder"], "loss": "ce", "grad_filter": 1.0}
 }
 
-adam = True
+optimizer = {"class": "adam"}
 learning_rate = 0.01
 gradient_noise = 0.3
 gradient_clip = 2

demos/demo-tf-attention.config

@@ -53,7 +53,7 @@ chunking = "0"
 truncation = -1
 #gradient_clip = 10
 gradient_nan_inf_filter = True
-adam = True
+optimizer = {"class": "adam"}
 gradient_noise = 0.3
 learning_rate = 0.0005
 learning_rate_control = "newbob"

demos/demo-tf-chunking-blstm.12ax.config

@@ -34,7 +34,7 @@ network = {
 }
 
 # training
-adam = True
+optimizer = {"class": "adam"}
 learning_rate = 0.01
 model = "/tmp/%s/returnn/%s/model" % (get_login_username(), demo_name)  # https://github.com/tensorflow/tensorflow/issues/6537
 num_epochs = 100

demos/demo-tf-contribrnn-lstm.12ax.config

@@ -25,7 +25,7 @@ network = {
 }
 
 # training
-adam = True
+optimizer = {"class": "adam"}
 learning_rate = 0.01
 model = "/tmp/%s/returnn/%s/model" % (os.getlogin(), demo_name)  # https://github.com/tensorflow/tensorflow/issues/6537
 num_epochs = 100

demos/demo-tf-enc-dec.config

@@ -74,7 +74,7 @@ batching = "random"
 batch_size = 5000
 max_seqs = 40
 chunking = "0"
-adam = True
+optimizer = {"class": "adam"}
 learning_rate = 0.0005
 learning_rate_control = "newbob"
 learning_rate_control_relative_error_relative_lr = True

demos/demo-tf-hard-att-copy.config

@@ -245,7 +245,7 @@ def pretrain_construct(idx, net_dict):
 pretrain = {"construction_algo": pretrain_construct}
 
 stop_on_nonfinite_train_score = False
-adam = True
+optimizer = {"class": "adam"}
 learning_rate = 0.01
 #learning_rate = 0.001
 model = "/tmp/%s/returnn/%s/model" % (get_login_username(), demo_name)

demos/demo-tf-lstm-benchmark.py

@@ -121,7 +121,7 @@ def make_config_dict(lstm_unit, use_gpu):
     "max_seqs": base_settings["max_seqs"],
     "chunking": base_settings["chunking"],
     # optimization
-    "adam": True,
+    "optimizer": {"class": "adam"},
     "learning_rate": 0.01}