diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..a50ed3200 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,111 @@ +name: Docs + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + build-docs: + name: Build Documentation + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Setup conda env + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + miniconda-version: "latest" + activate-environment: test + python-version: '3.10' + auto-activate-base: false + - name: Verify conda environment + shell: bash -l {0} + run: | + conda info + which python + which conda + - name: Update pip + shell: bash -l {0} + run: python -m pip install --upgrade pip + - name: Install pytorch + shell: bash -l {0} + run: python -m pip install torch==2.9.0.dev20250826 --extra-index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install monarch + shell: bash -l {0} + run: python -m pip install monarch-no-torch==0.1.0.dev20250826 --find-links assets/ci + - name: Install torchforge + shell: bash -l {0} + env: + GH_TOKEN: ${{ github.token }} + run: ./scripts/install.sh + - name: Install docs dependencies + shell: bash -l {0} + run: python -m pip install -r docs/requirements.txt + - name: Build docs + shell: bash -l {0} + working-directory: docs + run: make html --keep-going SPHINXOPTS='-W' + - name: Upload docs artifact + uses: actions/upload-artifact@v4 + with: + name: docs + path: docs/build/html/ + + # doc-preview: + # runs-on: [ubuntu-latest] + # needs: build-docs + # if: ${{ github.event_name == 'pull_request' }} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Download artifact + # uses: actions/download-artifact@v4 + # with: + # name: docs + # path: docs + # - name: Add noindex to preview docs + # run: | + # echo "Adding noindex meta tag to prevent search engine indexing of preview docs" + # find docs -name "*.html" -print0 | xargs -0 sed -i 's//\n /' + # - name: Upload docs preview + # uses: seemethere/upload-artifact-s3@v5 + # if: ${{ github.event_name == 'pull_request' }} + # with: + # retention-days: 14 + # s3-bucket: doc-previews + # if-no-files-found: error + # path: docs + # s3-prefix: meta-pytorch/forge/${{ github.event.pull_request.number }} + + deploy-docs: + needs: build-docs + if: github.ref == 'refs/heads/main' + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: docs + path: . + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: . + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..90c91ee58 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,25 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 000000000..747ffb7b3 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..87c6f43da --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +sphinx==7.2.6 +-e git+https://github.com/pytorch/pytorch_sphinx_theme.git@pytorch_sphinx_theme2#egg=pytorch_sphinx_theme2 +docutils>=0.18.1,<0.21 +sphinx-design==0.6.1 +sphinxcontrib-mermaid==1.0.0 +myst-parser #==0.18.1 # if want to contribute in markdown +sphinx-sitemap==2.7.1 diff --git a/docs/source/api.md b/docs/source/api.md new file mode 100644 index 000000000..5ed009c4c --- /dev/null +++ b/docs/source/api.md @@ -0,0 +1,35 @@ +# API Reference + +This section provides comprehensive API documentation for TorchForge modules and classes. + +TorchForge is organized into several key modules, each providing specialized functionality for post-training generative AI models: + +## Module Overview + +**Core Components** +- [Interfaces & Types](api_core.md) - Core interfaces and type definitions +- [Actors](api_actors.md) - Model training and inference components +- [Controller](api_controller.md) - Distributed training orchestration and resource management + +**Data Management** +- [Data](api_data.md) - Data handling utilities, datasets, and data models + +**Training Components** +- [Losses](api_losses.md) - Loss functions for reinforcement learning and supervised fine-tuning +- [Environments](api_envs.md) - Training and inference environments + +**Tools & Utilities** +- [Utilities](api_util.md) - General utility functions and helpers + +```{toctree} +:maxdepth: 2 +:hidden: + +api_core +api_actors +api_data +api_losses +api_envs +api_controller +api_util +``` diff --git a/docs/source/api_actors.md b/docs/source/api_actors.md new file mode 100644 index 000000000..6ef5f1ff8 --- /dev/null +++ b/docs/source/api_actors.md @@ -0,0 +1,19 @@ +# Actors + +The actors module contains the core components for model training and inference in TorchForge. This includes policy actors, reference models, replay buffers, and trainers. + +## Policy Actor + +The policy actor is responsible for model inference and policy interactions during training. + +## Reference Model + +The reference model provides baseline comparisons for reinforcement learning algorithms. + +## Replay Buffer + +The replay buffer manages storage and sampling of training experiences. + +## Trainer + +The trainer orchestrates the training process and implements training algorithms. diff --git a/docs/source/api_controller.md b/docs/source/api_controller.md new file mode 100644 index 000000000..e9bedda74 --- /dev/null +++ b/docs/source/api_controller.md @@ -0,0 +1,3 @@ +# Controller + +Distributed training orchestration and resource management components for TorchForge. diff --git a/docs/source/api_core.md b/docs/source/api_core.md new file mode 100644 index 000000000..75b3e9ae5 --- /dev/null +++ b/docs/source/api_core.md @@ -0,0 +1,3 @@ +# Core Interfaces + +This section covers the fundamental interfaces and type definitions that form the foundation of TorchForge. diff --git a/docs/source/api_data.md b/docs/source/api_data.md new file mode 100644 index 000000000..cbc1cfc53 --- /dev/null +++ b/docs/source/api_data.md @@ -0,0 +1,16 @@ +# Data Management + +Comprehensive data handling utilities for training and +inference, including datasets, data models, and various +data processing utilities. + +## Prompt + +Data model for input prompts and contexts. + +```{eval-rst} +.. automodule:: forge.data_models.prompt + :members: + :undoc-members: + :show-inheritance: +``` diff --git a/docs/source/api_envs.md b/docs/source/api_envs.md new file mode 100644 index 000000000..88e9d1cea --- /dev/null +++ b/docs/source/api_envs.md @@ -0,0 +1,8 @@ +# Environments + +Training and inference environments for TorchForge models. + + +## Chat Environment + +Chat-based environment for conversational AI training and inference. diff --git a/docs/source/api_losses.md b/docs/source/api_losses.md new file mode 100644 index 000000000..097b83394 --- /dev/null +++ b/docs/source/api_losses.md @@ -0,0 +1,11 @@ +# Losses + +Loss functions for reinforcement learning and supervised fine-tuning in TorchForge. + +## GRPO Loss + +Generalized Reward Policy Optimization (GRPO) loss implementation for reinforcement learning. + +## Reinforce Loss + +Reinforce algorithm loss implementation for policy gradient methods. diff --git a/docs/source/api_util.md b/docs/source/api_util.md new file mode 100644 index 000000000..f15e03b76 --- /dev/null +++ b/docs/source/api_util.md @@ -0,0 +1,25 @@ +# Utilities + +General utility functions and helpers used throughout TorchForge. + +## Distributed Computing + +Utilities for distributed training and communication. + +```{eval-rst} +.. automodule:: forge.util.distributed + :members: + :undoc-members: + :show-inheritance: +``` + +## Logging + +Logging configuration and utilities. + +```{eval-rst} +.. automodule:: forge.util.logging + :members: + :undoc-members: + :show-inheritance: +``` diff --git a/docs/source/concepts.md b/docs/source/concepts.md new file mode 100644 index 000000000..075d1ef7f --- /dev/null +++ b/docs/source/concepts.md @@ -0,0 +1,4 @@ +# Concepts + +This guide covers the fundamental concepts and architecture behind TorchForge, +helping you understand how the system works and how to effectively use its components. diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 000000000..69b3c20b5 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,132 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# Configuration file for the Sphinx documentation builder. +# + +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os +import sys + +import pytorch_sphinx_theme2 + +# Add the source directory to Python path so modules can be imported +sys.path.insert(0, os.path.abspath("../../src")) + +project = "torchforge" +copyright = "2025, PyTorch Contributors" +author = "PyTorch Contributors" +release = "0.1" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx_design", + "sphinx_sitemap", + "sphinxcontrib.mermaid", + "pytorch_sphinx_theme2", + "myst_parser", + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", +] + +html_baseurl = "https://meta-pytorch.org/forge/" # needed for sphinx-sitemap +sitemap_locales = [None] +sitemap_excludes = [ + "search.html", + "genindex.html", +] +sitemap_url_scheme = "{link}" + +templates_path = [ + "_templates", + os.path.join(os.path.dirname(pytorch_sphinx_theme2.__file__), "templates"), +] +exclude_patterns = [] + +sys.path.insert(0, os.path.abspath(".")) +sys.path.insert(0, os.path.abspath("../../src")) +html_theme = "pytorch_sphinx_theme2" +html_theme_path = [pytorch_sphinx_theme2.get_html_theme_path()] + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + + +html_theme_options = { + "navigation_with_keys": False, + "show_lf_header": False, + "show_lf_footer": False, + "analytics_id": "GTM-NPLPKN5G", + "logo": { + "text": "Home", + }, + "icon_links": [ + { + "name": "X", + "url": "https://x.com/PyTorch", + "icon": "fa-brands fa-x-twitter", + }, + { + "name": "GitHub", + "url": "https://github.com/meta-pytorch/forge", + "icon": "fa-brands fa-github", + }, + { + "name": "Discourse", + "url": "https://discuss.pytorch.org/", + "icon": "fa-brands fa-discourse", + }, + { + "name": "PyPi", + "url": "https://pypi.org/project/forge/", + "icon": "fa-brands fa-python", + }, + ], + "use_edit_page_button": True, + "navbar_center": "navbar-nav", + "canonical_url": "https://meta-pytorch.org/forge/", + "header_links_before_dropdown": 7, +} + +theme_variables = pytorch_sphinx_theme2.get_theme_variables() + +html_context = { + "theme_variables": theme_variables, + "display_github": True, + "github_url": "https://github.com", + "github_user": "meta-pytorch", + "github_repo": "forge", + "feedback_url": "https://github.com/meta-pytorch/forge", + "github_version": "main", + "doc_path": "docs/source", +} + +myst_enable_extensions = [ + "colon_fence", + "deflist", + "html_image", +] + +autodoc_default_options = { + "members": True, + "member-order": "bysource", + "special-members": "__init__", + "undoc-members": True, + "exclude-members": "__weakref__", +} + +# Autosummary settings +autosummary_generate = True +autosummary_imported_members = True diff --git a/docs/source/faq.md b/docs/source/faq.md new file mode 100644 index 000000000..d3c027866 --- /dev/null +++ b/docs/source/faq.md @@ -0,0 +1,3 @@ +# FAQ + +This FAQ covers common questions and issues encountered when using TorchForge. diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md new file mode 100644 index 000000000..57e1b63c8 --- /dev/null +++ b/docs/source/getting_started.md @@ -0,0 +1,9 @@ +# Get Started + +Welcome to TorchForge! This guide will help you get up and running with TorchForge, a PyTorch-native platform specifically designed for post-training generative AI models. + +TorchForge specializes in post-training techniques for large language models, including: + +- **Supervised Fine-Tuning (SFT)**: Adapt pre-trained models to specific tasks using labeled data +- **Generalized Reward Policy Optimization (GRPO)**: Advanced reinforcement learning for model alignment +- **Multi-GPU Distributed Training**: Efficient scaling across multiple GPUs and nodes diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 000000000..c450b4ca7 --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,36 @@ +# Welcome to TorchForge documentation! + +**TorchForge** is a PyTorch-native platform specifically designed +for post-training generative AI models. + +Key Features +------------ + +* **Post-Training Focus**: Specializes in techniques + like Supervised Fine-Tuning (SFT) and Generalized Reward Policy Optimization (GRPO) +* **PyTorch Integration**: Built natively on PyTorch with + dependencies on [PyTorch nightly](https://pytorch.org/get-started/locally/), + [Monarch](https://meta-pytorch.org/monarch), [vLLM](https://docs.vllm.ai/en/latest/), + and [TorchTitan](https://github.com/pytorch/torchtitan). +* **Multi-GPU Support**: Designed for distributed training + with minimum 3 GPU requirement for GRPO training +* **Model Support**: Includes pre-configured setups for popular models + like Llama3 8B and Qwen3.1 7B + +```{toctree} +:maxdepth: 1 +:caption: Contents: + +getting_started +concepts +usage +tutorials +api +faq +``` + +## Indices and tables + +* {ref}`genindex` +* {ref}`modindex` +* {ref}`search` diff --git a/docs/source/tutorials.md b/docs/source/tutorials.md new file mode 100644 index 000000000..5cfd3dbaf --- /dev/null +++ b/docs/source/tutorials.md @@ -0,0 +1,4 @@ +# Tutorials + + This section provides step-by-step guides to help you master TorchForge's capabilities, + from basic model fine-tuning to advanced distributed training scenarios. diff --git a/docs/source/usage.md b/docs/source/usage.md new file mode 100644 index 000000000..2a61c577a --- /dev/null +++ b/docs/source/usage.md @@ -0,0 +1,4 @@ +# Usage + +This section covers practical usage patterns, +configuration management, and common scenarios for working with TorchForge effectively.