Fix mutable default in Megatron init and IndexError on empty ModuleList (#3944)

* Fix mutable default argument in Megatron init and IndexError on empty ModuleList

In megatron_lm.py, initialize() used args_defaults={} as a default
parameter, which is shared across calls and can cause unexpected
state persistence.

In other.py, is_repeated_block() accessed module[0] without checking
if the ModuleList was empty, causing IndexError for empty ModuleLists.

* Apply style fixes

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: jashshah999

src/accelerate/utils/megatron_lm.py

@@ -873,7 +873,9 @@ def finish_mpu_init():
 
 
 # initialize megatron setup
-def initialize(accelerator, extra_args_provider=None, args_defaults={}):
+def initialize(accelerator, extra_args_provider=None, args_defaults=None):
+    if args_defaults is None:
+        args_defaults = {}
     accelerator.print("Initializing Megatron-LM")
     assert torch.cuda.is_available(), "Megatron requires CUDA."
 

src/accelerate/utils/other.py

@@ -82,7 +82,11 @@ def is_repeated_blocks(module: torch.nn.Module) -> bool:
     is useful to determine whether we should apply regional compilation to the module.
     """
 
-    return isinstance(module, torch.nn.ModuleList) and all(isinstance(m, module[0].__class__) for m in module)
+    return (
+        isinstance(module, torch.nn.ModuleList)
+        and len(module) > 0
+        and all(isinstance(m, module[0].__class__) for m in module)
+    )
 
 
 def has_repeated_blocks(module: torch.nn.Module) -> bool: