diff --git a/tests/lora/test_lora_layers_cogvideox.py b/tests/lora/test_lora_layers_cogvideox.py
index c141ebc96b3e..623b06621d66 100644
--- a/tests/lora/test_lora_layers_cogvideox.py
+++ b/tests/lora/test_lora_layers_cogvideox.py
@@ -16,6 +16,7 @@
 import unittest
 
 import numpy as np
+import pytest
 import torch
 from transformers import AutoTokenizer, T5EncoderModel
 
@@ -29,6 +30,7 @@
 from diffusers.utils.testing_utils import (
     floats_tensor,
     is_peft_available,
+    is_torch_version,
     require_peft_backend,
     skip_mps,
     torch_device,
@@ -126,6 +128,11 @@ def get_dummy_inputs(self, with_generator=True):
         return noise, input_ids, pipeline_inputs
 
     @skip_mps
+    @pytest.mark.xfail(
+        condtion=torch.device(torch_device).type == "cpu" and is_torch_version(">=", "2.5"),
+        reason="Test currently fails on CPU and PyTorch 2.5.1 but not on PyTorch 2.4.1.",
+        strict=True,
+    )
     def test_lora_fuse_nan(self):
         for scheduler_cls in self.scheduler_classes:
             components, text_lora_config, denoiser_lora_config = self.get_dummy_components(scheduler_cls)
diff --git a/tests/lora/test_lora_layers_mochi.py b/tests/lora/test_lora_layers_mochi.py
index eb15124601c6..910b126c147b 100644
--- a/tests/lora/test_lora_layers_mochi.py
+++ b/tests/lora/test_lora_layers_mochi.py
@@ -16,6 +16,7 @@
 import unittest
 
 import numpy as np
+import pytest
 import torch
 from transformers import AutoTokenizer, T5EncoderModel
 
@@ -23,6 +24,7 @@
 from diffusers.utils.testing_utils import (
     floats_tensor,
     is_peft_available,
+    is_torch_version,
     require_peft_backend,
     skip_mps,
     torch_device,
@@ -105,6 +107,11 @@ def get_dummy_inputs(self, with_generator=True):
 
         return noise, input_ids, pipeline_inputs
 
+    @pytest.mark.xfail(
+        condtion=torch.device(torch_device).type == "cpu" and is_torch_version(">=", "2.5"),
+        reason="Test currently fails on CPU and PyTorch 2.5.1 but not on PyTorch 2.4.1.",
+        strict=True,
+    )
     def test_lora_fuse_nan(self):
         for scheduler_cls in self.scheduler_classes:
             components, text_lora_config, denoiser_lora_config = self.get_dummy_components(scheduler_cls)
diff --git a/tests/lora/utils.py b/tests/lora/utils.py
index 7cdb2d6f51d7..d8dc86d57007 100644
--- a/tests/lora/utils.py
+++ b/tests/lora/utils.py
@@ -19,6 +19,7 @@
 from itertools import product
 
 import numpy as np
+import pytest
 import torch
 
 from diffusers import (
@@ -32,6 +33,7 @@
 from diffusers.utils.testing_utils import (
     CaptureLogger,
     floats_tensor,
+    is_torch_version,
     require_peft_backend,
     require_peft_version_greater,
     require_transformers_version_greater,
@@ -1510,6 +1512,11 @@ def test_simple_inference_with_text_denoiser_multi_adapter_weighted(self):
             )
 
     @skip_mps
+    @pytest.mark.xfail(
+        condtion=torch.device(torch_device).type == "cpu" and is_torch_version(">=", "2.5"),
+        reason="Test currently fails on CPU and PyTorch 2.5.1 but not on PyTorch 2.4.1.",
+        strict=True,
+    )
     def test_lora_fuse_nan(self):
         for scheduler_cls in self.scheduler_classes:
             components, text_lora_config, denoiser_lora_config = self.get_dummy_components(scheduler_cls)