From f5729da4822f336f302c1485895407b2df64f243 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 8 Jul 2025 09:02:24 +0530 Subject: [PATCH 1/5] mark wanvace lora tests as flaky --- tests/lora/test_lora_layers_wanvace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lora/test_lora_layers_wanvace.py b/tests/lora/test_lora_layers_wanvace.py index a7eb74080499..05d0132d8ff9 100644 --- a/tests/lora/test_lora_layers_wanvace.py +++ b/tests/lora/test_lora_layers_wanvace.py @@ -46,6 +46,7 @@ @require_peft_backend @skip_mps +@is_flaky class WanVACELoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): pipeline_class = WanVACEPipeline scheduler_cls = FlowMatchEulerDiscreteScheduler @@ -217,6 +218,5 @@ def test_lora_exclude_modules_wanvace(self): "Lora outputs should match.", ) - @is_flaky def test_simple_inference_with_text_denoiser_lora_and_scale(self): super().test_simple_inference_with_text_denoiser_lora_and_scale() From 9498c6171139b31596fb6caca58e1456cbe3b483 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 8 Jul 2025 10:16:53 +0530 Subject: [PATCH 2/5] ability to apply is_flaky at a class-level --- src/diffusers/utils/testing_utils.py | 45 +++++++++++++++----------- tests/lora/test_lora_layers_wanvace.py | 2 +- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/diffusers/utils/testing_utils.py b/src/diffusers/utils/testing_utils.py index e5da39c1d865..5061e19b98ea 100644 --- a/src/diffusers/utils/testing_utils.py +++ b/src/diffusers/utils/testing_utils.py @@ -990,37 +990,44 @@ def summary_failures_short(tr): config.option.tbstyle = orig_tbstyle -# Copied from https://github.com/huggingface/transformers/blob/000e52aec8850d3fe2f360adc6fd256e5b47fe4c/src/transformers/testing_utils.py#L1905 -def is_flaky(max_attempts: int = 5, wait_before_retry: Optional[float] = None, description: Optional[str] = None): +# Adapted from https://github.com/huggingface/transformers/blob/000e52aec8850d3fe2f360adc6fd256e5b47fe4c/src/transformers/testing_utils.py#L1905 +def is_flaky( + max_attempts: int = 5, + wait_before_retry: Optional[float] = None, + description: Optional[str] = None, +): """ - To decorate flaky tests. They will be retried on failures. - - Args: - max_attempts (`int`, *optional*, defaults to 5): - The maximum number of attempts to retry the flaky test. - wait_before_retry (`float`, *optional*): - If provided, will wait that number of seconds before retrying the test. - description (`str`, *optional*): - A string to describe the situation (what / where / why is flaky, link to GH issue/PR comments, errors, - etc.) + To decorate flaky tests (methods or entire classes). They will be retried on failures. """ - def decorator(test_func_ref): - @functools.wraps(test_func_ref) + def decorator(obj): + # If decorating a class, wrap each test method on it + if inspect.isclass(obj): + for attr_name, attr_value in list(obj.__dict__.items()): + if callable(attr_value) and attr_name.startswith("test"): + # recursively decorate the method + setattr(obj, attr_name, decorator(attr_value)) + return obj + + # Otherwise we're decorating a single test function / method + @functools.wraps(obj) def wrapper(*args, **kwargs): retry_count = 1 - while retry_count < max_attempts: try: - return test_func_ref(*args, **kwargs) - + return obj(*args, **kwargs) except Exception as err: - print(f"Test failed with {err} at try {retry_count}/{max_attempts}.", file=sys.stderr) + msg = ( + f"[FLAKY] {description or obj.__name__!r} " + f"failed on attempt {retry_count}/{max_attempts}: {err}" + ) + print(msg, file=sys.stderr) if wait_before_retry is not None: time.sleep(wait_before_retry) retry_count += 1 - return test_func_ref(*args, **kwargs) + # final attempt, let any exception bubble up + return obj(*args, **kwargs) return wrapper diff --git a/tests/lora/test_lora_layers_wanvace.py b/tests/lora/test_lora_layers_wanvace.py index 05d0132d8ff9..a0c2c7474495 100644 --- a/tests/lora/test_lora_layers_wanvace.py +++ b/tests/lora/test_lora_layers_wanvace.py @@ -46,7 +46,7 @@ @require_peft_backend @skip_mps -@is_flaky +@is_flaky(max_attempts=3, wait_before_retry=1, description="very flaky class") class WanVACELoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): pipeline_class = WanVACEPipeline scheduler_cls = FlowMatchEulerDiscreteScheduler From e8c124d917e5f4d98ec37af19df7b67d4a58d1b9 Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Tue, 8 Jul 2025 10:20:27 +0530 Subject: [PATCH 3/5] update --- src/diffusers/utils/testing_utils.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/diffusers/utils/testing_utils.py b/src/diffusers/utils/testing_utils.py index 5061e19b98ea..270b9d3ac9bc 100644 --- a/src/diffusers/utils/testing_utils.py +++ b/src/diffusers/utils/testing_utils.py @@ -991,13 +991,18 @@ def summary_failures_short(tr): # Adapted from https://github.com/huggingface/transformers/blob/000e52aec8850d3fe2f360adc6fd256e5b47fe4c/src/transformers/testing_utils.py#L1905 -def is_flaky( - max_attempts: int = 5, - wait_before_retry: Optional[float] = None, - description: Optional[str] = None, -): +def is_flaky(max_attempts: int = 5, wait_before_retry: Optional[float] = None, description: Optional[str] = None): """ To decorate flaky tests (methods or entire classes). They will be retried on failures. + + Args: + max_attempts (`int`, *optional*, defaults to 5): + The maximum number of attempts to retry the flaky test. + wait_before_retry (`float`, *optional*): + If provided, will wait that number of seconds before retrying the test. + description (`str`, *optional*): + A string to describe the situation (what / where / why is flaky, link to GH issue/PR comments, errors, + etc.) """ def decorator(obj): @@ -1026,7 +1031,6 @@ def wrapper(*args, **kwargs): time.sleep(wait_before_retry) retry_count += 1 - # final attempt, let any exception bubble up return obj(*args, **kwargs) return wrapper From 472cef361603644450cd9568b16d5d4ca2aa4e5c Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 9 Jul 2025 09:39:29 +0530 Subject: [PATCH 4/5] increase max_attempt. --- tests/lora/test_lora_layers_wanvace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lora/test_lora_layers_wanvace.py b/tests/lora/test_lora_layers_wanvace.py index a0c2c7474495..784a70dff4a7 100644 --- a/tests/lora/test_lora_layers_wanvace.py +++ b/tests/lora/test_lora_layers_wanvace.py @@ -46,7 +46,7 @@ @require_peft_backend @skip_mps -@is_flaky(max_attempts=3, wait_before_retry=1, description="very flaky class") +@is_flaky(description="very flaky class") class WanVACELoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): pipeline_class = WanVACEPipeline scheduler_cls = FlowMatchEulerDiscreteScheduler From b167eeb53f37315f0806e2dfb730734a739104db Mon Sep 17 00:00:00 2001 From: sayakpaul Date: Wed, 9 Jul 2025 09:50:21 +0530 Subject: [PATCH 5/5] increase attemtp. --- tests/lora/test_lora_layers_wanvace.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lora/test_lora_layers_wanvace.py b/tests/lora/test_lora_layers_wanvace.py index 784a70dff4a7..f976577653b2 100644 --- a/tests/lora/test_lora_layers_wanvace.py +++ b/tests/lora/test_lora_layers_wanvace.py @@ -46,7 +46,7 @@ @require_peft_backend @skip_mps -@is_flaky(description="very flaky class") +@is_flaky(max_attempts=10, description="very flaky class") class WanVACELoRATests(unittest.TestCase, PeftLoraLoaderMixinTests): pipeline_class = WanVACEPipeline scheduler_cls = FlowMatchEulerDiscreteScheduler