in-place sums

Author: bm-synth

src/diffusers/models/transformers/cogvideox_transformer_3d.py

@@ -138,8 +138,8 @@ def forward(
             **attention_kwargs,
         )
 
-        hidden_states = hidden_states + gate_msa * attn_hidden_states
-        encoder_hidden_states = encoder_hidden_states + enc_gate_msa * attn_encoder_hidden_states
+        hidden_states.add_(gate_msa * attn_hidden_states)
+        encoder_hidden_states.add_(enc_gate_msa * attn_encoder_hidden_states)
 
         # norm & modulate
         norm_hidden_states, norm_encoder_hidden_states, gate_ff, enc_gate_ff = self.norm2(
@@ -150,8 +150,8 @@ def forward(
         norm_hidden_states = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim=1)
         ff_output = self.ff(norm_hidden_states)
 
-        hidden_states = hidden_states + gate_ff * ff_output[:, text_seq_length:]
-        encoder_hidden_states = encoder_hidden_states + enc_gate_ff * ff_output[:, :text_seq_length]
+        hidden_states.add_(gate_ff * ff_output[:, text_seq_length:])
+        encoder_hidden_states.add_(enc_gate_ff * ff_output[:, :text_seq_length])
 
         return hidden_states, encoder_hidden_states
 

src/diffusers/models/transformers/transformer_cogview3plus.py

@@ -106,8 +106,8 @@ def forward(
             hidden_states=norm_hidden_states, encoder_hidden_states=norm_encoder_hidden_states
         )
 
-        hidden_states = hidden_states + gate_msa.unsqueeze(1) * attn_hidden_states
-        encoder_hidden_states = encoder_hidden_states + c_gate_msa.unsqueeze(1) * attn_encoder_hidden_states
+        hidden_states.add_(gate_msa.unsqueeze(1) * attn_hidden_states)
+        encoder_hidden_states.add_(c_gate_msa.unsqueeze(1) * attn_encoder_hidden_states)
 
         # norm & modulate
         norm_hidden_states = self.norm2(hidden_states)
@@ -120,8 +120,8 @@ def forward(
         norm_hidden_states = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim=1)
         ff_output = self.ff(norm_hidden_states)
 
-        hidden_states = hidden_states + gate_mlp.unsqueeze(1) * ff_output[:, text_seq_length:]
-        encoder_hidden_states = encoder_hidden_states + c_gate_mlp.unsqueeze(1) * ff_output[:, :text_seq_length]
+        hidden_states.add_(gate_mlp.unsqueeze(1) * ff_output[:, text_seq_length:])
+        encoder_hidden_states.add_(c_gate_mlp.unsqueeze(1) * ff_output[:, :text_seq_length])
 
         if hidden_states.dtype == torch.float16:
             hidden_states = hidden_states.clip(-65504, 65504)