update document And fix unittest

Author: chenyushuo

README.md

@@ -191,10 +191,9 @@ Choose one of the following options:
 conda create -n trinity python=3.12
 conda activate trinity
 
-pip install -e ".[verl]"
-pip install -e ".[flash_attn]"
+pip install -e ".[vllm,flash_attn]"
 
-# If you have no GPU, comment out the two lines above and uncomment this instead:
+# If you have no GPU, comment out the line above and uncomment this instead:
 # pip install -e ".[tinker]"
 
 # If you encounter issues when installing flash-attn, try:
@@ -209,10 +208,9 @@ pip install -e ".[dev]"  # for development like linting and debugging
 python3.10 -m venv .venv
 source .venv/bin/activate
 
-pip install -e ".[verl]"
-pip install -e ".[flash_attn]"
+pip install -e ".[vllm,flash_attn]"
 
-# If you have no GPU, comment out the two lines above and uncomment this instead:
+# If you have no GPU, comment out the line above and uncomment this instead:
 # pip install -e ".[tinker]"
 
 # If you encounter issues when installing flash-attn, try:
@@ -226,7 +224,10 @@ pip install -e ".[dev]"  # for development like linting and debugging
 [`uv`](https://github.com/astral-sh/uv) is a modern Python package installer.
 
 ```bash
-uv sync --extra verl --extra dev --extra flash_attn
+uv sync --extra vllm --extra dev --extra flash_attn
+
+# If you have no GPU, try to use Tinker instead:
+# uv sync --extra tinker --extra dev
 ```
 
 

README_zh.md

@@ -190,10 +190,9 @@ cd Trinity-RFT
 conda create -n trinity python=3.12
 conda activate trinity
 
-pip install -e ".[verl]"
-pip install -e ".[flash_attn]"
+pip install -e ".[vllm,flash_attn]"
 
-# 如果没有GPU，可以注释上两行的命令，改为使用Tinker：
+# 如果没有GPU，可以注释上一行的命令，改为使用Tinker：
 # pip install -e ".[tinker]"
 
 # 如果安装 flash-attn 时遇到问题，可尝试：
@@ -208,10 +207,9 @@ pip install -e ".[dev]"  # 用于调试和开发
 python3.10 -m venv .venv
 source .venv/bin/activate
 
-pip install -e ".[verl]"
-pip install -e ".[flash_attn]"
+pip install -e ".[vllm,flash_attn]"
 
-# 如果没有GPU，可以注释上两行的命令，改为使用Tinker：
+# 如果没有GPU，可以注释上一行的命令，改为使用Tinker：
 # pip install -e ".[tinker]"
 
 # 如果安装 flash-attn 时遇到问题，可尝试：
@@ -225,7 +223,10 @@ pip install -e ".[dev]"  # 用于调试和开发
 [`uv`](https://github.com/astral-sh/uv) 是现代的 Python 包管理工具。
 
 ```bash
-uv sync --extra verl --extra dev --extra flash_attn
+uv sync --extra vllm --extra dev --extra flash_attn
+
+# 如果没有GPU，可以改为使用Tinker：
+# uv sync --extra tinker --extra dev
 ```
 
 ## 通过 PyPI 安装

docs/sphinx_doc/source/tutorial/trinity_installation.md

@@ -3,11 +3,18 @@
 
 For installing Trinity-RFT, you have three options: from source (recommended), via PyPI, or using Docker.
 
-Before installing, ensure your system meets the following requirements:
+**Before you begin**, check your system setup:
 
-- **Python**: Version 3.10 to 3.12 (inclusive)
-- **CUDA**: Version >= 12.8
-- **GPUs**: At least 2 GPUs
+### If you have GPUs and want to use them:
+Make sure your system meets these requirements:
+- **Python**: 3.10 – 3.12
+- **CUDA**: 12.8 or higher
+- **GPUs**: At least 2 available
+
+### If you don’t have GPUs (or prefer not to use them):
+You can use the `tinker` option instead, which only requires:
+- **Python**: 3.11 – 3.12
+- **GPUs**: Not required
 
 ---
 
@@ -32,10 +39,15 @@ Choose one of the following options:
 conda create -n trinity python=3.12
 conda activate trinity
 
-pip install -e ".[dev]"
-pip install -e ".[flash_attn]"
-# if you encounter issues when installing flash-attn, try:
+pip install -e ".[vllm,flash_attn]"
+
+# If you have no GPU, comment out the line above and uncomment this instead:
+# pip install -e ".[tinker]"
+
+# If you encounter issues when installing flash-attn, try:
 # pip install flash-attn==2.8.1 --no-build-isolation
+
+pip install -e ".[dev]"  # for development like linting and debugging
 ```
 
 #### Using venv
@@ -44,18 +56,26 @@ pip install -e ".[flash_attn]"
 python3.10 -m venv .venv
 source .venv/bin/activate
 
-pip install -e ".[dev]"
-pip install -e ".[flash_attn]"
-# if you encounter issues when installing flash-attn, try:
+pip install -e ".[vllm,flash_attn]"
+
+# If you have no GPU, comment out the line above and uncomment this instead:
+# pip install -e ".[tinker]"
+
+# If you encounter issues when installing flash-attn, try:
 # pip install flash-attn==2.8.1 --no-build-isolation
+
+pip install -e ".[dev]"  # for development like linting and debugging
 ```
 
 #### Using `uv`
 
 [`uv`](https://github.com/astral-sh/uv) is a modern Python package installer.
 
 ```bash
-uv sync --extra dev --extra flash_attn
+uv sync --extra vllm --extra dev --extra flash_attn
+
+# If you have no GPU, try to use Tinker instead:
+# uv sync --extra tinker --extra dev
 ```
 
 ---

docs/sphinx_doc/source_zh/tutorial/trinity_installation.md

@@ -3,11 +3,18 @@
 
 安装 Trinity-RFT 有三种方式：源码安装（推荐）、通过 PyPI 安装，或使用 Docker。
 
-在安装前，请确保您的系统满足以下要求：
+**开始之前**，请检查您的系统配置：
 
-- **Python**：3.10 至 3.12（包含）
-- **CUDA**：大于等于 12.8
-- **GPU**：至少 2 块 GPU
+### 如果您拥有 GPU 并希望使用它们：
+请确保您的系统满足以下要求：
+- **Python**：3.10 – 3.12
+- **CUDA**：12.8 或更高版本
+- **GPU**：至少 2 块可用
+
+### 如果您没有 GPU（或不希望使用 GPU）：
+您可以改用 `tinker` 选项，该选项仅需满足：
+- **Python**：3.11 – 3.12
+- **GPU**：无需
 
 ---
 
@@ -32,10 +39,15 @@ cd Trinity-RFT
 conda create -n trinity python=3.12
 conda activate trinity
 
-pip install -e ".[dev]"
-pip install -e ".[flash_attn]"
+pip install -e ".[vllm,flash_attn]"
+
+# 如果没有GPU，可以注释上一行的命令，改为使用Tinker：
+# pip install -e ".[tinker]"
+
 # 如果安装 flash-attn 时遇到问题，可尝试：
 # pip install flash-attn==2.8.1 --no-build-isolation
+
+pip install -e ".[dev]"  # 用于调试和开发
 ```
 
 #### 使用 venv
@@ -44,18 +56,26 @@ pip install -e ".[flash_attn]"
 python3.10 -m venv .venv
 source .venv/bin/activate
 
-pip install -e ".[dev]"
-pip install -e ".[flash_attn]"
+pip install -e ".[vllm,flash_attn]"
+
+# 如果没有GPU，可以注释上一行的命令，改为使用Tinker：
+# pip install -e ".[tinker]"
+
 # 如果安装 flash-attn 时遇到问题，可尝试：
 # pip install flash-attn==2.8.1 --no-build-isolation
+
+pip install -e ".[dev]"  # 用于调试和开发
 ```
 
 #### 使用 `uv`
 
 [`uv`](https://github.com/astral-sh/uv) 是现代的 Python 包管理工具。
 
 ```bash
-uv sync --extra dev --extra flash_attn
+uv sync --extra vllm --extra dev --extra flash_attn
+
+# 如果没有GPU，可以改为使用Tinker：
+# uv sync --extra tinker --extra dev
 ```
 
 ---

pyproject.toml

@@ -21,6 +21,7 @@ classifiers = [
 ]
 requires-python = ">=3.10,<3.13"
 dependencies = [
+    "verl==0.5.0",
     "ray[default]>=2.50.0",
     "tensordict",
     "wandb",
@@ -48,8 +49,7 @@ dependencies = [
 trinity = "trinity.cli.launcher:main"
 
 [project.optional-dependencies]
-verl = [
-    "verl==0.5.0",
+vllm = [
     "vllm>=0.10.2,<=0.11.0",
 ]
 data = [

scripts/docker/Dockerfile

@@ -27,7 +27,7 @@ RUN chmod 1777 /tmp && apt update && apt install -y \
 # copy the Trinity-RFT dir into the workspace
 COPY . .
 
-RUN pip install --upgrade pip && pip install -e .[verl,mm,dev] && pip install flash_attn==2.8.1 --no-build-isolation
+RUN pip install --upgrade pip && pip install -e .[vllm,mm,dev] && pip install flash_attn==2.8.1 --no-build-isolation
 
 # Set Env variables
 

scripts/docker/Dockerfile.megatron

@@ -28,7 +28,7 @@ COPY . .
 
 # Install Trinity-RFT with Megatron
 RUN pip install --upgrade pip \
-    && pip install -e .[verl,mm,dev] \
+    && pip install -e .[vllm,mm,dev] \
     && pip install flash_attn==2.8.1 --no-build-isolation \
     && pip install -e .[megatron] \
     && NVCC_APPEND_FLAGS="--threads 4" APEX_PARALLEL_BUILD=8 pip install -v \

scripts/docker/Dockerfile.uv

@@ -35,7 +35,7 @@ RUN pip install uv && uv venv /opt/venv --python=python3.12
 
 # Install minimal Trinity-RFT
 RUN . /opt/venv/bin/activate && \
-    uv pip install -e.[verl,mm,dev]
+    uv pip install -e.[vllm,mm,dev]
 
 # Install flash_attn and Megatron
 RUN . /opt/venv/bin/activate && \

tests/trainer/trainer_test.py

@@ -980,6 +980,7 @@ async def test_serve_with_trainer(self):  # noqa: C901
         trainer_config = deepcopy(config)
         trainer_config.mode = "train"
         trainer_config.check_and_update()
+        trainer_config.trainer.max_actor_ckpt_to_keep = 10
 
         trainer_process = multiprocessing.Process(target=run_trainer, args=(trainer_config,))
         trainer_process.start()