update: recipes

Author: kozistr

tests/constants.py

@@ -93,35 +93,35 @@
     (AdaBelief, {'lr': 5e-1, 'weight_decay': 1e-3, 'weight_decouple': False}, 10),
     (AdaBelief, {'lr': 5e-1, 'weight_decay': 1e-3, 'fixed_decay': True}, 10),
     (AdaBelief, {'lr': 5e-1, 'weight_decay': 1e-3, 'rectify': False}, 10),
-    (AdaBound, {'lr': 5e-1, 'gamma': 0.1, 'weight_decay': 1e-3}, 75),
-    (AdaBound, {'lr': 5e-1, 'gamma': 0.1, 'weight_decay': 1e-3, 'fixed_decay': True}, 75),
-    (AdaBound, {'lr': 5e-1, 'gamma': 0.1, 'weight_decay': 1e-3, 'weight_decouple': False}, 75),
-    (AdaBound, {'lr': 5e-1, 'gamma': 0.1, 'weight_decay': 1e-3, 'amsbound': True}, 75),
-    (Adai, {'lr': 2e-1, 'weight_decay': 0.0}, 25),
-    (Adai, {'lr': 2e-1, 'weight_decay': 0.0, 'use_gc': True}, 75),
-    (Adai, {'lr': 2e-1, 'weight_decay': 0.0, 'dampening': 0.9}, 25),
-    (Adai, {'lr': 2e-1, 'weight_decay': 1e-4, 'weight_decouple': False}, 25),
-    (Adai, {'lr': 2e-1, 'weight_decay': 1e-4, 'weight_decouple': True}, 25),
-    (Adai, {'lr': 2e-1, 'weight_decay': 1e-4, 'weight_decouple': False, 'stable_weight_decay': True}, 25),
-    (Adai, {'lr': 2e-1, 'weight_decay': 1e-4, 'weight_decouple': True, 'stable_weight_decay': True}, 25),
+    (AdaBound, {'lr': 5e-1, 'gamma': 0.1, 'weight_decay': 1e-3}, 50),
+    (AdaBound, {'lr': 5e-1, 'gamma': 0.1, 'weight_decay': 1e-3, 'fixed_decay': True}, 50),
+    (AdaBound, {'lr': 5e-1, 'gamma': 0.1, 'weight_decay': 1e-3, 'weight_decouple': False}, 50),
+    (AdaBound, {'lr': 5e-1, 'gamma': 0.1, 'weight_decay': 1e-3, 'amsbound': True}, 50),
+    (Adai, {'lr': 5e-1, 'weight_decay': 0.0}, 5),
+    (Adai, {'lr': 5e-1, 'weight_decay': 0.0, 'use_gc': True}, 50),
+    (Adai, {'lr': 5e-1, 'weight_decay': 0.0, 'dampening': 0.9}, 5),
+    (Adai, {'lr': 5e-1, 'weight_decay': 1e-4, 'weight_decouple': False}, 5),
+    (Adai, {'lr': 5e-1, 'weight_decay': 1e-4, 'weight_decouple': True}, 5),
+    (Adai, {'lr': 5e-1, 'weight_decay': 1e-4, 'weight_decouple': False, 'stable_weight_decay': True}, 5),
+    (Adai, {'lr': 5e-1, 'weight_decay': 1e-4, 'weight_decouple': True, 'stable_weight_decay': True}, 5),
     (AdamP, {'lr': 5e-1, 'weight_decay': 1e-3}, 5),
     (AdamP, {'lr': 5e-1, 'weight_decay': 1e-3, 'use_gc': True}, 10),
     (AdamP, {'lr': 5e-1, 'weight_decay': 1e-3, 'nesterov': True}, 5),
     (DiffGrad, {'lr': 5e-2, 'weight_decay': 1e-3}, 10),
     (DiffGrad, {'lr': 5e-2, 'weight_decay': 1e-3, 'amsgrad': True}, 10),
     (DiffGrad, {'lr': 5e-1, 'weight_decay': 1e-3, 'rectify': True}, 20),
-    (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3}, 25),
-    (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3, 'pre_norm': True, 'max_grad_norm': 0.0}, 25),
-    (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3, 'grad_averaging': False}, 25),
+    (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3}, 20),
+    (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3, 'pre_norm': True, 'max_grad_norm': 0.0}, 20),
+    (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3, 'grad_averaging': False}, 20),
     (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3, 'adam': True, 'eps': 1e-8}, 10),
-    (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3, 'pre_norm': True, 'eps': 1e-8}, 25),
+    (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3, 'pre_norm': True, 'eps': 1e-8}, 20),
     (Lamb, {'lr': 5e-1, 'weight_decay': 1e-3, 'rectify': True, 'degenerated_to_sgd': True}, 10),
-    (LARS, {'lr': 5e-1, 'weight_decay': 1e-3}, 25),
-    (LARS, {'lr': 5e-1, 'nesterov': True}, 25),
-    (MADGRAD, {'lr': 5e-2, 'weight_decay': 1e-3}, 25),
-    (MADGRAD, {'lr': 5e-2, 'weight_decay': 1e-3, 'eps': 0.0}, 25),
-    (MADGRAD, {'lr': 1e-2, 'weight_decay': 1e-3, 'momentum': 0.0}, 25),
-    (MADGRAD, {'lr': 5e-2, 'weight_decay': 1e-3, 'decouple_decay': True}, 25),
+    (LARS, {'lr': 5e-1, 'weight_decay': 1e-3}, 20),
+    (LARS, {'lr': 5e-1, 'nesterov': True}, 20),
+    (MADGRAD, {'lr': 5e-2, 'weight_decay': 1e-3}, 20),
+    (MADGRAD, {'lr': 5e-2, 'weight_decay': 1e-3, 'eps': 0.0}, 20),
+    (MADGRAD, {'lr': 1e-2, 'weight_decay': 1e-3, 'momentum': 0.0}, 20),
+    (MADGRAD, {'lr': 5e-2, 'weight_decay': 1e-3, 'decouple_decay': True}, 20),
     (RAdam, {'lr': 5e-1, 'weight_decay': 1e-3}, 20),
     (RAdam, {'lr': 5e-1, 'weight_decay': 1e-3, 'degenerated_to_sgd': True}, 10),
     (SGDP, {'lr': 5e-1, 'weight_decay': 1e-4}, 10),
@@ -286,15 +286,15 @@
     (Adan, {'lr': 5e-1, 'weight_decay': 1e-3, 'weight_decouple': True}, 5),
     (DAdaptAdaGrad, {'lr': 2.0, 'weight_decay': 1e-3}, 50),
     (DAdaptAdaGrad, {'lr': 2.0, 'weight_decay': 1e-3, 'momentum': 0.1}, 50),
-    (DAdaptAdam, {'lr': 2.0, 'weight_decay': 1e-3}, 50),
+    (DAdaptAdam, {'lr': 2.0, 'weight_decay': 1e-3}, 25),
     (DAdaptAdam, {'lr': 1.0, 'weight_decay': 1e-3, 'weight_decouple': True}, 50),
     (DAdaptSGD, {'lr': 2.0, 'weight_decay': 1e-2}, 25),
     (DAdaptSGD, {'lr': 2.0, 'momentum': 0.9, 'weight_decay': 1e-3}, 25),
     (DAdaptAdan, {'lr': 1.0, 'weight_decay': 1e-2}, 25),
     (DAdaptAdan, {'lr': 1.0, 'weight_decay': 1e-2, 'weight_decouple': True}, 50),
-    (AdamS, {'lr': 1.0, 'weight_decay': 1e-3}, 20),
+    (AdamS, {'lr': 1.0, 'weight_decay': 1e-3}, 10),
     (AdamS, {'lr': 1.0, 'weight_decay': 1e-3, 'amsgrad': True}, 20),
-    (AdaFactor, {'lr': 5e-1, 'weight_decay': 1e-3, 'scale_parameter': False}, 100),
+    (AdaFactor, {'lr': 7.5e-1, 'weight_decay': 1e-3, 'scale_parameter': False}, 100),
     (AdaFactor, {'lr': 7.5e-1, 'weight_decay': 1e-3, 'amsgrad': True}, 125),
     (Apollo, {'lr': 5e-1, 'weight_decay': 1e-3}, 10),
     (Apollo, {'lr': 5e-1, 'weight_decay': 1e-3, 'rebound': 'belief'}, 10),