change div

samsja · samsja · commit 392a0159aa6e · 2024-08-22T09:41:02.000Z
diff --git a/open_diloco/train_fsdp.py b/open_diloco/train_fsdp.py
@@ -226,7 +226,7 @@ def get_lr_outer(optimizer, config: Config):
         _get_lr_outer,
         num_warmup_steps=config.warmup_steps,
         # num_training_steps=config.total_steps,
-        num_training_steps=config.total_steps / 4,
+        num_training_steps=config.total_steps / 5,
         num_cycles=0.5,
     )
     return LambdaLR(optimizer, lambda_lr, -1)

Original file line number	Diff line number	Diff line change
`@@ -226,7 +226,7 @@ def get_lr_outer(optimizer, config: Config):`
`226`	`226`	`_get_lr_outer,`
`227`	`227`	`num_warmup_steps=config.warmup_steps,`
`228`	`228`	`# num_training_steps=config.total_steps,`
`229`		`- num_training_steps=config.total_steps / 4,`
	`229`	`+ num_training_steps=config.total_steps / 5,`
`230`	`230`	`num_cycles=0.5,`
`231`	`231`	`)`
`232`	`232`	`return LambdaLR(optimizer, lambda_lr, -1)`