[megatron] compat megatron-core 0.11 (#4655)

Author: Jintao-Huang

examples/train/grpo/multi_node/train_dlc.sh

@@ -1,8 +1,8 @@
 # This script is used in DLC (Deep Learning Containers)
-# For more information, visit:https://www.aliyun.com/activity/bigdata/pai-dlc
+# For more information, visit: https://www.aliyun.com/activity/bigdata/pai-dlc
+# https://help.aliyun.com/zh/pai/user-guide/general-environment-variables
 NNODES=$WORLD_SIZE \
 NODE_RANK=$RANK \
-PYTHONPATH=. \
 torchrun \
     --nproc_per_node=8 \
     --nnodes=${WORLD_SIZE} \

examples/train/multi-node/dlc/train.sh

@@ -1,3 +1,4 @@
+# For more information, visit: https://www.aliyun.com/activity/bigdata/pai-dlc
 # https://help.aliyun.com/zh/pai/user-guide/general-environment-variables
 NNODES=$WORLD_SIZE \
 NODE_RANK=$RANK \

swift/megatron/argument/megatron_args.py

@@ -264,7 +264,10 @@ def _args_to_argv(self) -> Tuple[List[Any], Dict[str, Any]]:
         extra_megatron_kwargs = args_dict.pop('extra_megatron_kwargs')
         args_dict.update(extra_megatron_kwargs)
         use_core_011 = version.parse(megatron.core.__version__) < version.parse('0.12')
-        core_012_arguments = {'recompute_modules', 'moe_router_dtype', 'cross_entropy_fusion_impl', 'moe_enable_deepep'}
+        core_012_arguments = {
+            'recompute_modules', 'moe_router_dtype', 'cross_entropy_fusion_impl', 'moe_enable_deepep',
+            'optimizer_offload_fraction'
+        }
         for k, value in args_dict.items():
             if k not in MegatronArguments.__annotations__ and k not in extra_megatron_kwargs:
                 extra_args[k] = value