🔥 Remove GLM

Author: joyliu-q

slime/configs/base.py

@@ -6,8 +6,6 @@
 
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Optional
-import textwrap
 
 
 @dataclass
@@ -109,94 +107,3 @@ def generate_train_args(self, hf_model_path: str, checkpoints_path: Path, data_p
     --apply-chat-template --rollout-shuffle --rm-type math
 """
 
-# GLM-4.7 (358B MoE) model architecture args
-# Based on: https://huggingface.co/zai-org/GLM-4.7/blob/main/config.json
-GLM_4_7_MODEL_ARGS = """
-    --num-layers 92 --hidden-size 5120 --ffn-hidden-size 12288
-    --num-attention-heads 96 --group-query-attention --num-query-groups 8
-    --kv-channels 128 --vocab-size 151552
-    --normalization RMSNorm --norm-epsilon 1e-5 --swiglu
-    --add-qkv-bias --qk-layernorm
-    --untie-embeddings-and-output-weights
-    --use-rotary-position-embeddings --rotary-base 1000000
-    --num-experts 160
-    --moe-layer-freq "[0]*3+[1]*89"
-    --moe-shared-expert-intermediate-size 1536
-    --moe-router-topk 8
-    --moe-grouped-gemm --moe-permute-fusion
-    --moe-ffn-hidden-size 1536
-    --moe-router-score-function sigmoid
-    --moe-router-pre-softmax
-    --moe-router-enable-expert-bias
-    --moe-router-bias-update-rate 0
-    --moe-router-load-balancing-type seq_aux_loss
-    --moe-router-topk-scaling-factor 2.5
-    --moe-aux-loss-coeff 0
-    --moe-router-dtype fp32
-    --moe-token-dispatcher-type flex
-    --moe-enable-deepep
-"""
-
-# GLM-4.7-Flash (30B MoE with MLA) model architecture args
-# Based on: scripts/models/glm4.7-30B-A3B.sh
-GLM_4_7_FLASH_MODEL_ARGS = """
-    --num-layers 47 --hidden-size 2048 --ffn-hidden-size 10240
-    --num-attention-heads 20 --vocab-size 154880
-    --make-vocab-size-divisible-by 64
-    --normalization RMSNorm --norm-epsilon 1e-5 --swiglu
-    --disable-bias-linear --add-qkv-bias --qk-layernorm
-    --untie-embeddings-and-output-weights
-    --position-embedding-type rope --no-position-embedding
-    --use-rotary-position-embeddings --rotary-base 1000000 --no-rope-fusion
-    --multi-latent-attention
-    --q-lora-rank 768 --kv-lora-rank 512
-    --qk-head-dim 192 --v-head-dim 256 --kv-channels 192
-    --qk-pos-emb-head-dim 64
-    --num-experts 64
-    --moe-layer-freq "[0]*1+[1]*46"
-    --moe-shared-expert-intermediate-size 1536
-    --moe-router-topk 4
-    --moe-grouped-gemm --moe-permute-fusion
-    --moe-ffn-hidden-size 1536
-    --moe-router-score-function sigmoid
-    --moe-router-pre-softmax
-    --moe-router-enable-expert-bias
-    --moe-router-bias-update-rate 0
-    --moe-router-load-balancing-type aux_loss
-    --moe-router-topk-scaling-factor 1.8
-    --moe-aux-loss-coeff 0
-    --moe-router-dtype fp32
-    --moe-token-dispatcher-type flex
-    --moe-enable-deepep
-"""
-
-# GLM training args with MoE parallelism
-GLM_4_7_TRAINING_ARGS = """
-    --tensor-model-parallel-size 8 --pipeline-model-parallel-size 4
-    --context-parallel-size 2
-    --expert-model-parallel-size 16 --expert-tensor-parallel-size 1
-    --sequence-parallel
-    --decoder-last-pipeline-num-layers 23
-    --recompute-granularity full --recompute-method uniform --recompute-num-layers 1
-    --use-dynamic-batch-size --max-tokens-per-gpu 16384
-    --megatron-to-hf-mode bridge
-    --attention-dropout 0.0 --hidden-dropout 0.0
-    --attention-backend flash
-    --optimizer-cpu-offload --overlap-cpu-optimizer-d2h-h2d
-    --use-precision-aware-optimizer
-"""
-
-GLM_4_7_FLASH_TRAINING_ARGS = """
-    --tensor-model-parallel-size 4 --pipeline-model-parallel-size 2
-    --context-parallel-size 2
-    --expert-model-parallel-size 8 --expert-tensor-parallel-size 1
-    --sequence-parallel
-    --decoder-last-pipeline-num-layers 23
-    --recompute-granularity full --recompute-method uniform --recompute-num-layers 1
-    --use-dynamic-batch-size --max-tokens-per-gpu 32768
-    --megatron-to-hf-mode bridge
-    --attention-dropout 0.0 --hidden-dropout 0.0
-    --attention-backend flash
-    --optimizer-cpu-offload --overlap-cpu-optimizer-d2h-h2d
-    --use-precision-aware-optimizer
-"""

slime/configs/glm_4_7.py

@@ -346,7 +346,7 @@ def list_available_configs():
 
 @app.function(
     image=image,
-    gpu="H200:8",  # GLM-4.7 needs H200s for memory
+    gpu="H200:8",
     volumes={
         HF_CACHE_PATH.as_posix(): hf_cache_volume,
         CHECKPOINTS_PATH.as_posix(): checkpoints_volume,
@@ -362,7 +362,7 @@ def list_available_configs():
 )
 @modal.experimental.clustered(
     4, rdma=True
-)  # 12 nodes for GLM-4.7 (8 train + 4 rollout)
+)
 async def train_multi_node(config: str = "qwen-0.5b-sync"):
     """Main entry point for multi-node GRPO training on Modal.