Merge pull request #84 from zhipuch/main

glide-the · web-flow · commit 62a82a86474d · 2024-11-27T17:02:28.000+08:00
sft with multigpu gradient_norm_before_clip error update
diff --git a/training/cogvideox_text_to_video_sft.py b/training/cogvideox_text_to_video_sft.py
@@ -744,7 +744,7 @@ def load_model_hook(models, input_dir):
             last_lr = lr_scheduler.get_last_lr()[0] if lr_scheduler is not None else args.learning_rate
             logs = {"loss": loss.detach().item(), "lr": last_lr}
             # gradnorm + deepspeed: https://github.com/microsoft/DeepSpeed/issues/4555
-            if accelerator.distributed_type != DistributedType.DEEPSPEED:
+            if accelerator.sync_gradients and accelerator.distributed_type != DistributedType.DEEPSPEED:
                 logs.update(
                     {
                         "gradient_norm_before_clip": gradient_norm_before_clip,

Original file line number	Diff line number	Diff line change
`@@ -744,7 +744,7 @@ def load_model_hook(models, input_dir):`
`744`	`744`	`last_lr = lr_scheduler.get_last_lr()[0] if lr_scheduler is not None else args.learning_rate`
`745`	`745`	`logs = {"loss": loss.detach().item(), "lr": last_lr}`
`746`	`746`	`# gradnorm + deepspeed: https://github.com/microsoft/DeepSpeed/issues/4555`
`747`		`- if accelerator.distributed_type != DistributedType.DEEPSPEED:`
	`747`	`+ if accelerator.sync_gradients and accelerator.distributed_type != DistributedType.DEEPSPEED:`
`748`	`748`	`logs.update(`
`749`	`749`	`{`
`750`	`750`	`"gradient_norm_before_clip": gradient_norm_before_clip,`