add default lmdeploy/memory/ray log dir in RL scripts (#1374)

add default lmdeploy/memory/ray log dir in scripts

Author: YanhuiDua

examples/v1/scripts/run_rl.sh

@@ -1,4 +1,5 @@
 set -ex
+ray stop --force
 # examples of usage:
 # qwen3_8B_grpo_gsm8k training: 
 # bash examples/v1/scripts/run_rl.sh examples/v1/config/rl_qwen3_8B_grpo.py "sglang" $MODEL_PATH $DATA_PATH $EVAL_DATA_PATH
@@ -54,23 +55,33 @@ current_time=$(date "+%m%d%H")
 # 取模型路径的最后一级作为model_name，取数据路径的倒数第二级作为data_name
 model_dir_name=$(basename "$MODEL_PATH")
 data_dir_name=$(basename "$(dirname "$DATA_PATH")")
-export WORK_DIR="work_dirs/${model_dir_name}_${data_dir_name}_${infer_backend_lower}"
-
+DIR=$(pwd)
+export WORK_DIR="${DIR}/work_dirs/${model_dir_name}_${data_dir_name}_${infer_backend_lower}"
+if [ ! -d "$WORK_DIR" ]; then
+  mkdir -p "$WORK_DIR"
+fi
+export LMDEPLOY_LOG_FILE="${WORK_DIR}/lmdeploy_log_${current_time}.txt"
+export XTUNER_RL_MEM_DIR="${WORK_DIR}/mem_${current_time}"
 
 # 2. Launch Ray cluster
 # 根据 NODE_COUNT 分配 num_cpus, 防止内存OOM
 node_count=${NODE_COUNT:-1}
 total_cpus=$((node_count * 128))
 
 if [ "$RAY_RANK" -eq 0 ]; then
+  rm -rf /tmp/ray_log
+  export RAY_LOG_DIR="${WORK_DIR}/ray_${current_time}/"
+  mkdir -p ${RAY_LOG_DIR}
+  ln -sfn "${RAY_LOG_DIR}" /tmp/ray_log
   ray start --head \
     --node-ip-address="$RAY_MASTER_ADDR" \
     --port="$RAY_HEAD_PORT" \
     --dashboard-host=0.0.0.0 \
     --dashboard-port=$RAY_DASHBOARD_PORT \
     --include-dashboard=true \
     --disable-usage-stats \
-    --num-cpus=$total_cpus
+    --num-cpus=$total_cpus \
+    --temp-dir="/tmp/ray_log/"
 else
   while true; do
     if curl --connect-timeout 2 "http://${RAY_MASTER_ADDR}:${RAY_DASHBOARD_PORT}" >/dev/null 2>&1; then
@@ -95,11 +106,6 @@ while true; do
   fi
 done
 
-# 3. start training job
-if [ ! -d "$WORK_DIR" ]; then
-  mkdir -p "$WORK_DIR"
-fi
-
 SCRIPT_NAME=$(basename "$0")
 cp "$0" "${WORK_DIR}/${SCRIPT_NAME}"
 cp "$CONFIG_PATH" "${WORK_DIR}/config.py"

examples/v1/scripts/run_rl_submit.sh

@@ -1,4 +1,5 @@
 set -ex
+ray stop --force
 # examples of usage:
 # qwen3_8B_grpo_gsm8k training: bash examples/v1/scripts/run_rl.sh examples/v1/config/rl_qwen3_8B_grpo.py "sglang" $MODEL_PATH $DATA_PATH $EVAL_DATA_PATH
 # qwen2.5_7B_dapo_math training: bash examples/v1/scripts/run_rl.sh  examples/v1/config/rl_qwen25_7B_dapo.py "sglang" $MODEL_PATH $DATA_PATH $EVAL_DATA_PATH
@@ -29,7 +30,7 @@ export DATA_PATH=$DATA_PATH
 export EVAL_DATA_PATH=$EVAL_DATA_PATH
 export XTUNER_USE_FA3=${XTUNER_USE_FA3:-1}
 export XTUNER_LOG_LEVEL=${XTUNER_LOG_LEVEL:-"INFO"}
-
+export PYTHONUNBUFFERED=1
 
 infer_backend_lower=$(echo "$INFER_BACKEND" | tr '[:upper:]' '[:lower:]')
 if [ "$infer_backend_lower" = "sglang" ]; then
@@ -48,20 +49,37 @@ else
   exit 1
 fi 
 
+current_time=$(date "+%m%d%H")
+# 取模型路径的最后一级作为model_name，取数据路径的倒数第二级作为data_name
+model_dir_name=$(basename "$MODEL_PATH")
+data_dir_name=$(basename "$(dirname "$DATA_PATH")")
+DIR=$(pwd)
+export WORK_DIR="${DIR}/work_dirs/${model_dir_name}_${data_dir_name}_${infer_backend_lower}"
+if [ ! -d "$WORK_DIR" ]; then
+  mkdir -p "$WORK_DIR"
+fi
+export LMDEPLOY_LOG_FILE="${WORK_DIR}/lmdeploy_log_${current_time}.txt"
+export XTUNER_RL_MEM_DIR="${WORK_DIR}/mem_${current_time}"
+
 # 2. Launch Ray cluster
 # 根据 NODE_COUNT 分配 num_cpus, 防止内存OOM
 node_count=${NODE_COUNT:-1}
 total_cpus=$((node_count * 128))
 
 if [ "$RAY_RANK" -eq 0 ]; then
+  rm -rf /tmp/ray_log
+  export RAY_LOG_DIR="${WORK_DIR}/ray_${current_time}/"
+  mkdir -p ${RAY_LOG_DIR}
+  ln -sfn "${RAY_LOG_DIR}" /tmp/ray_log 
   ray start --head \
     --node-ip-address="$RAY_MASTER_ADDR" \
     --port="$RAY_HEAD_PORT" \
     --dashboard-host=0.0.0.0 \
     --dashboard-port=$RAY_DASHBOARD_PORT \
     --include-dashboard=true \
     --disable-usage-stats \
-    --num-cpus=$total_cpus
+    --num-cpus=$total_cpus \
+    --temp-dir="/tmp/ray_log/"
 else
   while true; do
     if curl --connect-timeout 2 "http://${RAY_MASTER_ADDR}:${RAY_DASHBOARD_PORT}" >/dev/null 2>&1; then
@@ -86,23 +104,12 @@ while true; do
   fi
 done
 
-# 3. Prepare work directory and log file
-current_time=$(date "+%m%d%H")
-# 取模型路径的最后一级作为model_name，取数据路径的倒数第二级作为data_name
-model_dir_name=$(basename "$MODEL_PATH")
-data_dir_name=$(basename "$(dirname "$DATA_PATH")")
-export WORK_DIR="work_dirs/${model_dir_name}_${data_dir_name}_${infer_backend_lower}"
-
-if [ ! -d "$WORK_DIR" ]; then
-  mkdir -p "$WORK_DIR"
-fi
-
 SCRIPT_NAME=$(basename "$0")
 cp "$0" "${WORK_DIR}/${SCRIPT_NAME}"
 cp "$CONFIG_PATH" "${WORK_DIR}/config.py"
 LOG_FILE="${WORK_DIR}/training_log_${current_time}.txt"
 
-# 4. Submit training job on Head node
+# 3. Submit training job on Head node
 if [ "$RAY_RANK" -eq 0 ]; then
   RUNTIME_ENV_JSON="{
       \"env_vars\": {