[megatron] Support deepspeed launcher (#5404)

Author: Jintao-Huang

swift/megatron/argument/megatron_args.py

@@ -10,7 +10,7 @@
 from transformers.utils.versions import require_version
 
 from swift.llm.argument.base_args import to_abspath
-from swift.utils import get_logger, json_parse_to_dict
+from swift.utils import get_dist_setting, get_logger, json_parse_to_dict
 
 logger = get_logger()
 
@@ -160,6 +160,7 @@ class MegatronArguments(ExtraMegatronArguments):
 
     # dist
     distributed_backend: Literal['nccl', 'gloo'] = 'nccl'
+    local_rank: Optional[int] = None
     use_distributed_optimizer: bool = True
     tensor_model_parallel_size: int = 1
     pipeline_model_parallel_size: int = 1
@@ -273,6 +274,8 @@ class MegatronArguments(ExtraMegatronArguments):
     def _set_default(self):
         if self.mlp_padding_free and self.sequence_parallel:
             raise ValueError('mlp_padding_free is not compatible with sequence_parallel.')
+        if self.local_rank is None:
+            self.local_rank = get_dist_setting()[1]
         if self.lr is None:
             if self.train_type == 'full':
                 self.lr = 1e-5