sft with multigpu

zhipuch · zhipuch · commit ac5f41755500 · 2024-11-12T15:20:41.000+08:00
diff --git a/training/cogvideox_text_to_video_sft.py b/training/cogvideox_text_to_video_sft.py
@@ -743,7 +743,7 @@ def load_model_hook(models, input_dir):
             last_lr = lr_scheduler.get_last_lr()[0] if lr_scheduler is not None else args.learning_rate
             logs = {"loss": loss.detach().item(), "lr": last_lr}
             # gradnorm + deepspeed: https://github.com/microsoft/DeepSpeed/issues/4555
-            if accelerator.distributed_type != DistributedType.DEEPSPEED:
+            if accelerator.sync_gradients and accelerator.distributed_type != DistributedType.DEEPSPEED:
                 logs.update(
                     {
                         "gradient_norm_before_clip": gradient_norm_before_clip,

Original file line number	Diff line number	Diff line change
`@@ -743,7 +743,7 @@ def load_model_hook(models, input_dir):`
`743`	`743`	`last_lr = lr_scheduler.get_last_lr()[0] if lr_scheduler is not None else args.learning_rate`
`744`	`744`	`logs = {"loss": loss.detach().item(), "lr": last_lr}`
`745`	`745`	`# gradnorm + deepspeed: https://github.com/microsoft/DeepSpeed/issues/4555`
`746`		`- if accelerator.distributed_type != DistributedType.DEEPSPEED:`
	`746`	`+ if accelerator.sync_gradients and accelerator.distributed_type != DistributedType.DEEPSPEED:`
`747`	`747`	`logs.update(`
`748`	`748`	`{`
`749`	`749`	`"gradient_norm_before_clip": gradient_norm_before_clip,`