Fix enable_sequential_cpu_offload in CogView4Pipeline (huggingface#11195)

* Fix enable_sequential_cpu_offload in CogView4Pipeline

* make fix-copies

Author: hlky

src/diffusers/pipelines/cogview4/pipeline_cogview4.py

@@ -213,9 +213,7 @@ def _get_glm_embeds(
                 device=text_input_ids.device,
             )
             text_input_ids = torch.cat([pad_ids, text_input_ids], dim=1)
-        prompt_embeds = self.text_encoder(
-            text_input_ids.to(self.text_encoder.device), output_hidden_states=True
-        ).hidden_states[-2]
+        prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=True).hidden_states[-2]
 
         prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
         return prompt_embeds

src/diffusers/pipelines/cogview4/pipeline_cogview4_control.py

@@ -216,9 +216,7 @@ def _get_glm_embeds(
                 device=text_input_ids.device,
             )
             text_input_ids = torch.cat([pad_ids, text_input_ids], dim=1)
-        prompt_embeds = self.text_encoder(
-            text_input_ids.to(self.text_encoder.device), output_hidden_states=True
-        ).hidden_states[-2]
+        prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=True).hidden_states[-2]
 
         prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
         return prompt_embeds