add div by 4

samsja · samsja · commit f5f5df1675b5 · 2024-08-20T08:49:15.000Z
diff --git a/open_diloco/train_fsdp.py b/open_diloco/train_fsdp.py
@@ -223,7 +223,8 @@ def get_lr_outer(optimizer, config: Config):
     lambda_lr = partial(
         _get_lr_outer,
         num_warmup_steps=config.warmup_steps,
-        num_training_steps=config.total_steps,
+        # num_training_steps=config.total_steps,
+        num_training_steps=config.total_steps / 4,
     )
     return LambdaLR(optimizer, lambda_lr, -1)
 

Original file line number	Diff line number	Diff line change
`@@ -223,7 +223,8 @@ def get_lr_outer(optimizer, config: Config):`
`223`	`223`	`lambda_lr = partial(`
`224`	`224`	`_get_lr_outer,`
`225`	`225`	`num_warmup_steps=config.warmup_steps,`
`226`		`- num_training_steps=config.total_steps,`
	`226`	`+ # num_training_steps=config.total_steps,`
	`227`	`+ num_training_steps=config.total_steps / 4,`
`227`	`228`	`)`
`228`	`229`	`return LambdaLR(optimizer, lambda_lr, -1)`
`229`	`230`