Moved the is_fsdp_available function to import utils

Author: romitjain

src/transformers/cache_utils.py

@@ -7,8 +7,8 @@
 from transformers.pytorch_utils import is_torch_greater_or_equal_than_2_6
 
 from .configuration_utils import PretrainedConfig
-from .modeling_utils import is_fsdp_enabled
 from .utils import (
+    is_fsdp_enabled,
     is_hqq_available,
     is_quanto_greater,
     is_torch_greater_or_equal,

src/transformers/modeling_utils.py

@@ -110,6 +110,7 @@
     is_bitsandbytes_available,
     is_flash_attn_2_available,
     is_flash_attn_3_available,
+    is_fsdp_enabled,
     is_kernels_available,
     is_offline_mode,
     is_optimum_available,
@@ -124,7 +125,6 @@
     is_torch_xla_available,
     is_torch_xpu_available,
     logging,
-    strtobool,
 )
 from .utils.generic import _CAN_RECORD_REGISTRY, GeneralInterface, OutputRecorder
 from .utils.hub import create_and_tag_model_card, get_checkpoint_shard_files
@@ -182,15 +182,6 @@
     from torch.distributed.tensor import DTensor
 
 
-def is_fsdp_enabled():
-    return (
-        torch.distributed.is_available()
-        and torch.distributed.is_initialized()
-        and strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
-        and strtobool(os.environ.get("FSDP_CPU_RAM_EFFICIENT_LOADING", "False")) == 1
-    )
-
-
 def is_local_dist_rank_0():
     return (
         torch.distributed.is_available()

src/transformers/utils/__init__.py

@@ -161,6 +161,7 @@
     is_flute_available,
     is_fp_quant_available,
     is_fsdp_available,
+    is_fsdp_enabled,
     is_ftfy_available,
     is_g2p_en_available,
     is_galore_torch_available,

src/transformers/utils/import_utils.py

@@ -35,7 +35,7 @@
 
 from packaging import version
 
-from . import logging
+from . import logging, strtobool
 
 
 logger = logging.get_logger(__name__)  # pylint: disable=invalid-name
@@ -665,6 +665,18 @@ def is_torch_bf16_available():
     return is_torch_bf16_gpu_available()
 
 
+def is_fsdp_enabled():
+    if is_torch_available():
+        import torch
+
+        return (
+            torch.distributed.is_available()
+            and torch.distributed.is_initialized()
+            and strtobool(os.environ.get("ACCELERATE_USE_FSDP", "False")) == 1
+            and strtobool(os.environ.get("FSDP_CPU_RAM_EFFICIENT_LOADING", "False")) == 1
+        )
+
+
 @lru_cache
 def is_torch_fp16_available_on_device(device):
     if not is_torch_available():