[model] feat: add qwen3-4b grpo script on ASCEND NPU A3 (verl-project#4432)

### What does this PR do?
add examples/grpo_trainer/run_qwen3-4b_npu.sh

### Test
The figure below shows the comparison curve of the critic_reward_mean
metric.
<img width="1790" height="948" alt="image"
src="https://github.com/user-attachments/assets/01df9bed-f888-470d-936c-eb335acd57e9"
/>


### API and Usage Example
```sh
# install jemalloc
sudo apt update
sudo apt install libjemalloc2
# run bash
bash examples/grpo_trainer/run_qwen3-4b_npu.sh
```

Author: 5082459

docs/ascend_tutorial/ascend_quick_start.rst

@@ -235,6 +235,8 @@ verl 中昇腾暂不支持生态库如下：
     +-----------------------+-------------------------+-------------------+-------------------+--------------------------+
     |   GRPO                | Qwen2.5-VL-32B-instruct |        FSDP       |    vllm-ascend    |    Atlas 200T A2 Box16   |
     +-----------------------+-------------------------+-------------------+-------------------+--------------------------+
+    |   GRPO                | Qwen3-4B                |        FSDP       |    vllm-ascend    |    Atlas 800T A3         |
+    +-----------------------+-------------------------+-------------------+-------------------+--------------------------+
     |   GRPO                | Qwen3-8B                |        FSDP       |    vllm-ascend    |    Atlas 200T A2 Box16   |
     +-----------------------+-------------------------+-------------------+-------------------+--------------------------+
     |   GRPO                | Qwen3-32B               |        FSDP       |    vllm-ascend    |    Atlas 200T A2 Box16   |

examples/grpo_trainer/run_qwen3_4b_grpo_vllm_1k_npu.sh

@@ -0,0 +1,82 @@
+set -xeuo pipefail
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+source /usr/local/Ascend/nnal/atb/set_env.sh
+
+# 使用v1引擎
+export VLLM_USE_V1=1
+# 指定vllm 版本
+export VLLM_VERSION=0.9.1
+
+# 开启二级流水
+export TASK_QUEUE_ENABLE=2
+# 开启细绑核
+export CPU_AFFINITY_CONF=1
+# 使用jemalloc优化内存访问（依赖安装jemalloc）
+export LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libjemalloc.so.2${LD_PRELOAD:+:$LD_PRELOAD}"
+
+# A3 机器单机8卡
+trainer_n_gpus_per_node=16
+trainer_nnodes=1
+trainer_project_name='verl_grpo_example_gsm8k'
+trainer_experiment_name="qwen3_4b_grpo_8npu}"
+
+RAY_DATA_HOME=${RAY_DATA_HOME:-"${HOME}/verl"}
+MODEL_PATH=${MODEL_PATH:-"${RAY_DATA_HOME}/models/Qwen3-4B"}
+CKPTS_DIR=${CKPTS_DIR:-"${RAY_DATA_HOME}/ckpts/${trainer_project_name}/${trainer_experiment_name}"}
+TRAIN_FILE=${TRAIN_FILE:-"${RAY_DATA_HOME}/data/gsm8k/train.parquet"}
+TEST_FILE=${TEST_FILE:-"${RAY_DATA_HOME}/data/gsm8k/test.parquet"}
+
+export TENSORBOARD_DIR="${RAY_DATA_HOME}/tensorboard_dir/${trainer_project_name}/${trainer_experiment_name}"
+mkdir -p "${RAY_DATA_HOME}/logs/${trainer_project_name}"
+LOG_PATH="${RAY_DATA_HOME}/logs/${trainer_project_name}/${trainer_experiment_name}.log"
+
+use_dynamic_bsz=True
+
+python3 -m verl.trainer.main_ppo \
+    algorithm.adv_estimator=grpo \
+    data.train_files=${TRAIN_FILE} \
+    data.val_files=${TEST_FILE} \
+    data.train_batch_size=512 \
+    data.max_prompt_length=1024 \
+    data.max_response_length=1024 \
+    data.filter_overlong_prompts=True \
+    data.truncation='error' \
+    actor_rollout_ref.model.path=${MODEL_PATH} \
+    actor_rollout_ref.actor.optim.lr=5e-7 \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.actor.entropy_coeff=0.001 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=256 \
+    actor_rollout_ref.actor.use_kl_loss=True \
+    actor_rollout_ref.actor.kl_loss_coef=0.001 \
+    actor_rollout_ref.actor.kl_loss_type=low_var_kl \
+    actor_rollout_ref.actor.use_torch_compile=False \
+    actor_rollout_ref.actor.use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.actor.ppo_max_token_len_per_gpu=3000 \
+    actor_rollout_ref.model.enable_gradient_checkpointing=True \
+    actor_rollout_ref.actor.fsdp_config.param_offload=True \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=True \
+    actor_rollout_ref.rollout.enforce_eager=True \
+    actor_rollout_ref.rollout.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.rollout.log_prob_max_token_len_per_gpu=4096 \
+    actor_rollout_ref.rollout.enable_chunked_prefill=False \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=2 \
+    actor_rollout_ref.rollout.name=vllm \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.9 \
+    actor_rollout_ref.rollout.n=5 \
+    actor_rollout_ref.ref.log_prob_use_dynamic_bsz=${use_dynamic_bsz} \
+    actor_rollout_ref.ref.log_prob_max_token_len_per_gpu=8192 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=True \
+    actor_rollout_ref.ref.use_torch_compile=True \
+    algorithm.kl_ctrl.kl_coef=0.001 \
+    trainer.critic_warmup=0 \
+    trainer.project_name=${trainer_project_name} \
+    trainer.experiment_name=${trainer_experiment_name} \
+    trainer.logger=['console','tensorboard'] \
+    trainer.default_local_dir=${CKPTS_DIR} \
+    trainer.n_gpus_per_node=$trainer_n_gpus_per_node \
+    trainer.nnodes=$trainer_nnodes \
+    trainer.save_freq=-1 \
+    trainer.test_freq=5 \
+    trainer.total_epochs=15 \
+    trainer.val_before_train=False \
+    trainer.device=npu 2>&1 | tee ${LOG_PATH}

verl/models/transformers/npu_patch.py

@@ -239,6 +239,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
 # Patches for Qwen3 Model
 modeling_qwen3.Qwen3RMSNorm.forward = rms_norm_forward_npu
 modeling_qwen3.Qwen3MLP.forward = silu_forward_npu
+modeling_qwen3.apply_rotary_pos_emb = apply_rotary_pos_emb_npu
 
 # Patches for Qwen3 MoE Model
 modeling_qwen3_moe.Qwen3MoeRMSNorm.forward = rms_norm_forward_npu