From d3f729f368d5a7337f8f4af4042da314e0cebe74 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 6 Oct 2025 09:18:48 -0700 Subject: [PATCH 01/26] Update --- docs/source/conf.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 760a8d71..2d1c95e6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -157,15 +157,6 @@ def get_version_path(): "html_image", ] -autodoc_default_options = { - "members": True, - "member-order": "bysource", - "special-members": "__init__", - "undoc-members": True, - "exclude-members": "__weakref__", -} - - # -- Sphinx Gallery configuration ------------------------------------------- sphinx_gallery_conf = { "examples_dirs": "tutorial_sources", # Path to examples directory @@ -179,3 +170,4 @@ def get_version_path(): "write_computation_times": True, "show_signature": False, } + From 9823f1f12fb09f0284e894104d03d000e0da0b68 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Mon, 6 Oct 2025 09:19:54 -0700 Subject: [PATCH 02/26] Update --- docs/source/api.md | 53 +- docs/source/api_actors.md | 55 +- docs/source/api_controller.md | 3 - docs/source/api_core.md | 69 +- docs/source/api_data.md | 67 +- docs/source/api_data_models.md | 51 ++ docs/source/api_envs.md | 8 - docs/source/api_losses.md | 26 +- docs/source/api_types.md | 41 + docs/source/api_util.md | 86 +- ...r_rl_fundamentals_forge_tutorial_thumb.png | Bin 0 -> 28056 bytes .../sphx_glr_template_tutorial_thumb.png | Bin 0 -> 28056 bytes docs/source/tutorials/index.rst | 63 ++ ...l_fundamentals_forge_tutorial.codeobj.json | 215 +++++ .../rl_fundamentals_forge_tutorial.ipynb | 158 ++++ .../rl_fundamentals_forge_tutorial.py | 558 +++++++++++++ .../rl_fundamentals_forge_tutorial.py.md5 | 1 + .../rl_fundamentals_forge_tutorial.rst | 788 ++++++++++++++++++ .../rl_fundamentals_forge_tutorial.zip | Bin 0 -> 40960 bytes docs/source/tutorials/sg_execution_times.rst | 40 + .../tutorials/template_tutorial.codeobj.json | 27 + docs/source/tutorials/template_tutorial.ipynb | 75 ++ docs/source/tutorials/template_tutorial.py | 91 ++ .../source/tutorials/template_tutorial.py.md5 | 1 + docs/source/tutorials/template_tutorial.rst | 152 ++++ docs/source/tutorials/template_tutorial.zip | Bin 0 -> 5757 bytes 26 files changed, 2552 insertions(+), 76 deletions(-) delete mode 100644 docs/source/api_controller.md create mode 100644 docs/source/api_data_models.md delete mode 100644 docs/source/api_envs.md create mode 100644 docs/source/api_types.md create mode 100644 docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png create mode 100644 docs/source/tutorials/images/thumb/sphx_glr_template_tutorial_thumb.png create mode 100644 docs/source/tutorials/index.rst create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.py create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.rst create mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.zip create mode 100644 docs/source/tutorials/sg_execution_times.rst create mode 100644 docs/source/tutorials/template_tutorial.codeobj.json create mode 100644 docs/source/tutorials/template_tutorial.ipynb create mode 100644 docs/source/tutorials/template_tutorial.py create mode 100644 docs/source/tutorials/template_tutorial.py.md5 create mode 100644 docs/source/tutorials/template_tutorial.rst create mode 100644 docs/source/tutorials/template_tutorial.zip diff --git a/docs/source/api.md b/docs/source/api.md index 5ed009c4..e4c2d064 100644 --- a/docs/source/api.md +++ b/docs/source/api.md @@ -1,35 +1,34 @@ # API Reference -This section provides comprehensive API documentation for TorchForge modules and classes. - -TorchForge is organized into several key modules, each providing specialized functionality for post-training generative AI models: - -## Module Overview +This API reference is organized by priority of concepts that users should be exposed to, based on how they're used in the core Forge workflows. + +```{eval-rst} +.. toctree:: + :maxdepth: 2 + + api_core + api_actors + api_data_models + api_types + api_util + api_data + api_losses +``` -**Core Components** -- [Interfaces & Types](api_core.md) - Core interfaces and type definitions -- [Actors](api_actors.md) - Model training and inference components -- [Controller](api_controller.md) - Distributed training orchestration and resource management +## Overview -**Data Management** -- [Data](api_data.md) - Data handling utilities, datasets, and data models +The Forge API is structured around key concepts in order of priority: -**Training Components** -- [Losses](api_losses.md) - Loss functions for reinforcement learning and supervised fine-tuning -- [Environments](api_envs.md) - Training and inference environments +1. **[Core Concepts](api_core.md)** - Actor System, ForgeActor, and ForgeService fundamentals +2. **[Built-in Actors](api_actors.md)** - Policy, Trainer, ReplayBuffer, and ReferenceModel +3. **[Data Models](api_data_models.md)** - Completion, Prompt, Episode, and other data structures +4. **[Configuration and Types](api_types.md)** - Core types and configuration classes +5. **[Utilities](api_util.md)** - Distributed computing, logging, and observability tools +6. **[Data Processing](api_data.md)** - Rewards, tokenization, and data handling utilities +7. **[Loss Functions](api_losses.md)** - GRPO and REINFORCE loss implementations -**Tools & Utilities** -- [Utilities](api_util.md) - General utility functions and helpers +## Quick Start -```{toctree} -:maxdepth: 2 -:hidden: +To get started with Forge, begin with the [Core Concepts](api_core.md) to understand the actor system foundation, then explore the [Built-in Actors](api_actors.md) for common RL workflows. -api_core -api_actors -api_data -api_losses -api_envs -api_controller -api_util -``` +For a practical example, see the GRPO implementation in `apps/grpo/main.py` which demonstrates how these components work together in a complete reinforcement learning training loop. diff --git a/docs/source/api_actors.md b/docs/source/api_actors.md index 6ef5f1ff..1be09712 100644 --- a/docs/source/api_actors.md +++ b/docs/source/api_actors.md @@ -1,19 +1,54 @@ -# Actors +# Built-in Actors -The actors module contains the core components for model training and inference in TorchForge. This includes policy actors, reference models, replay buffers, and trainers. +On top of the services/actors foundation, Forge provides implementations of actors that are useful in RL workflows. -## Policy Actor +## Policy -The policy actor is responsible for model inference and policy interactions during training. +Inference and generation via vLLM. The {class}`forge.actors.policy.Policy` is a key actor for generating completions from language models. -## Reference Model +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: -The reference model provides baseline comparisons for reinforcement learning algorithms. + forge.actors.policy.Policy + forge.actors.policy.PolicyWorker + forge.actors.policy.EngineConfig + forge.actors.policy.SamplingConfig +``` -## Replay Buffer +## Trainer -The replay buffer manages storage and sampling of training experiences. +Training via torchtitan. The {class}`forge.actors.trainer.RLTrainer` handles reinforcement learning training loops. -## Trainer +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.actors.trainer.RLTrainer +``` + +## ReplayBuffer + +For storing experience and sampling to the trainer - the glue between policy and trainer. The {class}`forge.actors.replay_buffer.ReplayBuffer` manages experience data for RL training. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.actors.replay_buffer.ReplayBuffer +``` + +## ReferenceModel + +Used for RL correctness. The {class}`forge.actors.reference_model.ReferenceModel` provides reference logprobs for RL algorithms. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: -The trainer orchestrates the training process and implements training algorithms. + forge.actors.reference_model.ReferenceModel +``` diff --git a/docs/source/api_controller.md b/docs/source/api_controller.md deleted file mode 100644 index e9bedda7..00000000 --- a/docs/source/api_controller.md +++ /dev/null @@ -1,3 +0,0 @@ -# Controller - -Distributed training orchestration and resource management components for TorchForge. diff --git a/docs/source/api_core.md b/docs/source/api_core.md index 75b3e9ae..785302ad 100644 --- a/docs/source/api_core.md +++ b/docs/source/api_core.md @@ -1,3 +1,68 @@ -# Core Interfaces +# Core Concepts -This section covers the fundamental interfaces and type definitions that form the foundation of TorchForge. +## Actor System + +Forge is built on top of Monarch and makes extensive use of actors. +Actors are the foundation for building distributed, fault-tolerant systems. + +## ForgeActor + +In Forge, everything centers around the {class}`forge.controller.actor.ForgeActor`, which is a customized version of a Monarch actor tailored for Forge-specific needs. + +The {class}`forge.controller.actor.ForgeActor` differs from a standard Monarch actor by allowing you to specify resource requirements using the `options()` method. This API lets you define the resources your actor needs and create two types of constructs: + +- A regular Monarch actor using `as_actor()` +- A service using `as_service()` + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.controller.actor.ForgeActor +``` + +### Resource Configuration Example + +Options are important because they demonstrate the resource requirements and how we represent them in Forge: + +```python +from forge.controller.actor import ForgeActor + +class MyActor(ForgeActor): + pass + +# Create a service with specific resource requirements +service = MyActor.options( + hosts=1, + procs=8, + replicas=1, + with_gpus=True +).as_service() +``` + +This example creates a service that has 1 replica, where the replica consists of 1 remote host and 8 processes, using Monarch's remote allocations. + +### Key Methods + +**`options()`** +Configures resource requirements for the actor. + +**`setup()`** +Sets up an actor. All actors should implement this for any heavyweight setup (like PyTorch distributed initialization, model checkpoint loading, etc.). + +**`launch()`** +Logic to provision and deploy a new replica. This is what services use to spin up replicas. + +## ForgeService + +Services are replicated, fault-tolerant versions of actors. They provide high availability and load distribution across multiple actor instances. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.controller.service.Service + forge.controller.service.ServiceInterface +``` diff --git a/docs/source/api_data.md b/docs/source/api_data.md index cbc1cfc5..a652c6d0 100644 --- a/docs/source/api_data.md +++ b/docs/source/api_data.md @@ -1,16 +1,63 @@ -# Data Management +# Data Processing -Comprehensive data handling utilities for training and -inference, including datasets, data models, and various -data processing utilities. +Data handling utilities for datasets, rewards, and tokenization. -## Prompt +## Rewards -Data model for input prompts and contexts. +Reward functions for RL training. The {mod}`forge.data.rewards` module provides reward functions like {class}`forge.data.rewards.MathReward` and {class}`forge.data.rewards.ThinkingReward`. ```{eval-rst} -.. automodule:: forge.data_models.prompt - :members: - :undoc-members: - :show-inheritance: +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.rewards +``` + +## Tokenization + +Tokenization utilities for processing text data. The {mod}`forge.data.tokenizer` module provides tokenization functions and utilities. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.tokenizer +``` + +## Data Collation + +Data collation utilities for batching and processing. The {mod}`forge.data.collate` module provides functions for collating data into batches. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.collate +``` + +## Data Sharding + +Data sharding utilities for distributed processing. The {mod}`forge.data.sharding` module provides sharding strategies for distributed training. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.sharding +``` + +## Data Utilities + +General data processing utilities. The {mod}`forge.data.utils` module provides miscellaneous data handling functions. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data.utils ``` diff --git a/docs/source/api_data_models.md b/docs/source/api_data_models.md new file mode 100644 index 00000000..a7c02caa --- /dev/null +++ b/docs/source/api_data_models.md @@ -0,0 +1,51 @@ +# Data Models + +Base data models for common RL workflows. + +## Completion + +Outputs from vLLM. The {class}`forge.data_models.completion.Completion` represents a model-generated completion for a given prompt. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data_models.completion.Completion +``` + +## Prompt + +Input prompts for models. The {class}`forge.data_models.prompt.Prompt` encapsulates prompt data for language models. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data_models.prompt.Prompt +``` + +## Episode + +Training episodes for RL. The {class}`forge.data_models.episode.Episode` represents a complete interaction episode in reinforcement learning. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data_models.episode.Episode +``` + +## ScoredCompletion + +Completions with associated scores. The {class}`forge.data_models.scored_completion.ScoredCompletion` extends completions with scoring information. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.data_models.scored_completion.ScoredCompletion +``` diff --git a/docs/source/api_envs.md b/docs/source/api_envs.md deleted file mode 100644 index 88e9d1ce..00000000 --- a/docs/source/api_envs.md +++ /dev/null @@ -1,8 +0,0 @@ -# Environments - -Training and inference environments for TorchForge models. - - -## Chat Environment - -Chat-based environment for conversational AI training and inference. diff --git a/docs/source/api_losses.md b/docs/source/api_losses.md index 097b8339..d16f2a6f 100644 --- a/docs/source/api_losses.md +++ b/docs/source/api_losses.md @@ -1,11 +1,27 @@ -# Losses +# Loss Functions -Loss functions for reinforcement learning and supervised fine-tuning in TorchForge. +Loss functions for RL training. ## GRPO Loss -Generalized Reward Policy Optimization (GRPO) loss implementation for reinforcement learning. +Group Relative Policy Optimization loss function. The {mod}`forge.losses.grpo_loss` module provides loss functions for GRPO training. -## Reinforce Loss +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: -Reinforce algorithm loss implementation for policy gradient methods. + forge.losses.grpo_loss +``` + +## REINFORCE Loss + +REINFORCE algorithm loss function. The {mod}`forge.losses.reinforce_loss` module provides loss functions for REINFORCE training. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.losses.reinforce_loss +``` diff --git a/docs/source/api_types.md b/docs/source/api_types.md new file mode 100644 index 00000000..e7bd423a --- /dev/null +++ b/docs/source/api_types.md @@ -0,0 +1,41 @@ +# Configuration and Types + +## Core Type Definitions + +Core type definitions used throughout Forge. The {mod}`forge.types` module contains fundamental types and configurations. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.types +``` + +## Configuration Classes + +Configuration classes for actors and services. + +### EngineConfig + +Configuration for vLLM engines used in Policy actors. The {class}`forge.actors.policy.EngineConfig` extends vLLM's EngineArgs with worker-specific fields. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.actors.policy.EngineConfig +``` + +### SamplingConfig + +Configuration for sampling parameters in Policy actors. The {class}`forge.actors.policy.SamplingConfig` provides overrides for vLLM's sampling parameters. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.actors.policy.SamplingConfig +``` diff --git a/docs/source/api_util.md b/docs/source/api_util.md index f15e03b7..06438b35 100644 --- a/docs/source/api_util.md +++ b/docs/source/api_util.md @@ -1,25 +1,89 @@ # Utilities -General utility functions and helpers used throughout TorchForge. +Utility functions and classes for distributed computing, logging, and operations. ## Distributed Computing -Utilities for distributed training and communication. +Utilities for distributed computing operations. The {mod}`forge.util.distributed` module provides functions for setting up distributed environments. ```{eval-rst} -.. automodule:: forge.util.distributed - :members: - :undoc-members: - :show-inheritance: +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.util.distributed ``` ## Logging -Logging configuration and utilities. +Logging utilities for Forge applications. The {mod}`forge.util.logging` module provides logging configuration and utilities. ```{eval-rst} -.. automodule:: forge.util.logging - :members: - :undoc-members: - :show-inheritance: +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.util.logging +``` + +## Operations + +Core operations and utilities. The {mod}`forge.util.ops` module contains commonly used operations for RL workflows. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.util.ops +``` + +## Metric Logging + +Metric logging utilities. The {mod}`forge.util.metric_logging` module provides utilities for logging metrics. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.util.metric_logging +``` + +## Observability + +### Metrics + +Metrics collection and reporting. The {mod}`forge.observability.metrics` module provides the core metrics infrastructure. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.observability.metrics +``` + +### Performance Tracking + +Performance tracking utilities. The {mod}`forge.observability.perf_tracker` module provides performance measurement tools. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.observability.perf_tracker +``` + +### Metric Actors + +Actors for metric collection. The {mod}`forge.observability.metric_actors` module provides actors for distributed metric collection. + +```{eval-rst} +.. autosummary:: + :toctree: _autosummary + :recursive: + + forge.observability.metric_actors ``` diff --git a/docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png b/docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png new file mode 100644 index 0000000000000000000000000000000000000000..3b362cf91299659eeec19a203a6d16971bc138fd GIT binary patch literal 28056 zcmd3NV|OM^w03Me6XT9;+qP}nwr$(CHNnKTZF@4&o9C?a>HLECL$B3c-PPTDS8dc) z`-)VM6NiWS1p@>G1TQHeq67p4obYpCK|%bqIO!C40s-0TONt1pc;sAvhC7?SrS3xe z&Axr!_+spkE_6>!Jcwf0Cd;~6Z+2TsGS?<%Vq2z=gV_|ndMi{&(Kj-o? zPygpu1jJ7d5n>$T|Lq&`fBXLb@W368x0XpbDv%2>6i^c)GLrLs=)ta4KDdevk;s+x ztHBE77wzS@At21R{*|b=-tpF&^kr(NZ1|=cQlGgA6*^$8ml(<7y9-re3$gbdRLbBz zX(3o!E+}Ff;q`Y3QtxYL$Mlu!^;g3xogJxn%olIc@cYqun9B3-5#lBs4$wV$F3k+U zX%%sWS%MRO$uE^lU>(>ftpGJv3urKNh)}4JZ2_=CBf&;?Zc0iJR3-tgyzML!En^81 zfE|ZeaHcb&N+u)po+c0hxB~W*WCTyrBak1^9v7)TuoH43jF6tO1bo&qQ6b&0s;LwS zB<-hlH)u0Jz<@$)tne;WLI8zv2Sx-=FJe2y1AK#Za57tp+TI%I%_5K=rJYf}-NDFa zs7Vl_oqN;n`!&1SY)_ZHi4)E9;UE`Hm9asdLYbrh85#@3@-I{rMOQP9D7+$KH18Rs zb_t=V7>&)UE|>>rm@MJES?&=Z=E)P^i;dY8>Y?{bpPnO~;K0NA03L9O4yfrbkggYm zle+D&Ef~Q`;}Dx5JlP-*kO5MaQ81NID;Q#PwW8B#Fm)T|4*ydmC!`HRJ&1W8krP25 z^4SN`*>R>E?ccSpb_WB@&}4kspf}7w@$mmxMA+n52<6#n1#ZCZLhTZ`J1qbdvNWem z-(OCr!-eM?7U>L~38#l5Dv;+$&|#UD87kp|6~ZktaOG8P^-F`Wg1eyS{9%Lvt<@v9 zljP_vxGHKAmYuMADY&uB)w+6Tb7LYS?%_(WazY&G99oVir9jqs(GDZjKzevjlAFd` z5O=hRA8e#4nI&XNh>c-cCb8OEx>d!^9>gvf2wWFjV&*X!;($3Qv>_RmnkXWQRGUbP7-2yy5M}HrFc@Yme3))|ngY$WxNc9_Zu^+`+uR*8QVJpg!&eCe zG0YAv$>)K{Xl7q%r~B<8_uP8#A2>b=e-4T;2baoyd}zL9=>@{&wm=@B6N_) zBX)Cg%(JKFn+G$sZsdToKB=Jb5XUv*=hej)xDStyy5l&U z1){Nx;c_4pcGH;K**k!s)1%RKO|d*LakW;R^gMhsNOMi}GiP*}Z1*YE@_@T3k^Z2u zz=^?2nT~$qly?xZmPkoX)J?qID}yMY5)^z;NqG7f>hMz)Kt!h1jHa zU!$*;g|$Sw95E$K{eH_Dz%y_QZ#!)fWfOW5$YT<@+o_d%;cm`#;Kk%zfp_Y_tH=Z2 z{XNsg<4D3B2-615AC0yiW%rhBCa_i)gph>>^vMgkAQ;Fbe64QLacSD_okgazAj zh}mS7(;@*_anFTvEk0*jJy;za^@IOiBFiu9jXwG&>4NPKKFJAoFE8E15^l>9;x2~4 z1)4_-`6Hl0o^V{b(3ifC?jSR)Cfm5a)vG?$l=)hTleSOzEqIk6mCt5{t-9UGVR=_=_GZ>m%bG$hwa#)~lH z0gAp2p=ime?$b^OV$VtP4ff6Vxwm4B-a1C3sRAvH***f@I_P^P4_F8i%~Gpa!68_0 zXVUTMDHCsZPUwshYBfK*axOR9QC$g*soPddFSSTnhOyBz<2g_UcnNeOib{aZxtZJH zYojLCTVWu*2Csrl$+oYj#ZY>fWS z66kiXPUz7$>chxu0xP2MkfSBN*+M34fw&xG(7QZLTi@Ln72!!{0AIfulD2hvB^KQ7 zNwU-5@i#|YbmAMaohR+?hia|)Q5OTv+=PLeg9g%za{usQP(D%&m<2iae1@?^- z6ffX1T8+)T-q^Pc{pb2y0iLiXLW_m~6NkvB`0D(@nxr&8kB-?)j!ZuvfWp!^R^9Z6_@04F6vPEHA%M z`k>EyqUTsW6Nkpq)LWZBcrvU-K8irZ1z8Sqq`i*YdE!J=zy5gfe z>Jz{&r@guwMQOLvt0_USM3#YgRUCPs&%`mhJDPG_`4xWSbdDkF;YuUUs(cvE_N>B5 zfOV?Ev9uo@wlKoSnD+P4n!d-_+d+fe(91;!(tGnZsIPOa=MOQ+wS zNVo3LE3imMv67n+ZwS?2y+WYo%cFkm35?(0Yn`)8$MJ{j%65=kzxn^3mNtKQxqvqF zRSZQCwj^L&E~Ll|Tjv!8zS6?Cu_-a&vYB(pgfd6Fsn@yQGgU$SX7V}i|8VqzNvtLD zA`Bkdd0+?6uyr|pZnnPew3fxjrhdE+Irk*~WqyiMt=;4>OQ%QRwY6Gr1P*eQL)l;E zxq@frgxYCgSx3vx^!q~r#cUK+M=GN_&lGJBtveVf5rRcu%tAMPB{r>2r#6u~U!kf8 zX0XMXFOZpNtAUy~j$riPu7q0Y$W_8P#TvH-p-cL%sPhfniz>f>@2);bq-4IvqRG<2 z21F4g69>)NU42^jKH^O6$+`|VKemd#Q=H!fBLOWb**_y!?;Dh7X}?=*t7bv;>LRb$ zOgoj_Zmv`(Z~y6hEXgzDfjVe&R-~raQs>KeiybS=J|^KLJcT<$Bzf+l7JsLgnEQ_W zgFKD+HDpM1=9Ya|@qf7i51Q;=PWb0X;(JN{P-e<6u{MnJpxC%eGme+D^mKilK>m9Q z)BSsyG?spIoa$64(<}6in4R{HSFEzyL!_~aP{j%|Gw!C2oz<(k!n^?g-lN8i{`6|N z47iEKC4)A&2t0pGL9R2pS*gPZ+$`tP!AFNL@X@+0YF3~qkvDv1+{-7g+<&S*ak@xM z@<-r}2T3nm@b352`XkNX9S|}tkiIqEp%wxVkjGTvV&T&CuH0T&GP9IVNT?H+OoWL$ zMT%30t&Nd8%J_p8LjRC~P}?_*l%IBLygF5!eOx(QXQTEA7!EBTB>H0JK8=s60VdOn z3!1p)J&(jbmN0umU{tSS6E_bcqshcZrh7DOShDvq(hKPd{wHKjFO(kG@u;9};T8gI znRycNN?QMaeC)uISq4|RM*AmZgaKA>uJsR zYuj(f+`qcx7o3xblS-YIl&9)4v3wJO*5g6gfPigT;@Ujmqxj` zV{=Lu(RY9yKK|?X8oKS-uip54i9`c)H=FF2e=Ogz~rB^cM z?4=RtE14^hZ)l-Y?=u}<#Qk)|TrTjD(fRCnrsJIRfcr97oEwarT@|E?5(n}<+>&k| z*y&CGndM4Dg@2X}(Sy<2KOtl@24uQbM(_K}=iQnXI-idF%W~EQscHvKT!#+7SZPv7 zi)^mKYYsVnndfDh2Tyb22*=;U_&VC(w(8?gVc`VwPA9|>M$<{Y;Y-c|!Ut zBPr&p#~rTmG9?b32ha9f=RVgY?@9Ex(nmVZ?R)B!HqIV$yoWQ;{vx47#jMnRVE7BF`+ zRv3Nl7^LQZ9L3kZQ&Mwornbu;bJCkLz=<*(?wEfIyBtBV`zu*N)s=h?%uAxn+F^SgQYYg4zM%%O_@yFPR zf>Ol>*k@QDo2hj_afA5;=O`&U#(YI?jSp3|P!y(Glv7e1BEWJZQ|w|GchA6h71t>@ zO9+$C+W7JRD^j>%w0eHg3$#RCOrdcw(z7&; zi-0!8nzBo4YWSKLwsI-TNwJ=mM=0LX1~gt=3^m|`kS2EmiQ&%gCoRScI~udX_ML2@ zk78Z0V?PVP=jH@k@Kl*(3Nr3(G=&XK>zWyj6;TkU(&!FG_71eCD zh>$CsciIZZL0iz8ZQHB+yx%U4!Uz3_o|&O}B%B!OjjZJDc`iq{cy0}minmzmUFX1N zDUY>fczX!L5OiX1(HNzRv;D>gTjvubX`&qHLy6{mo5xMc%aPsLwKVzji$KrzYqPYD zjfPKNr1&K^<9&@-M!JT5dWKwPHVxO;sx5Ugf0pgbgBDR=E%Er814iPMcgyo}_v`@Y zR`{OAXCx(fw5F(N^|BFOym7U`+3Iv9B{~u@>8QBrVaDC@jrto_=&u?-G159|5%9X# zpcdwKwry&G7Wo2WP!8a{p8JvQ9mJ9T3ZRak+=iuAhuD_o1*-13%0szCryg)r ztlYPu_XQnMHqf7CQ$^GPoTUNdON-5mGyYA~UJ5Eo?Rehg<7PqiK02-5DNvAtIwTlv zTjD0s9qp9H#2JnF2s*pzaksmON}C+RYxiU*@M9Ro0ruv(T~#Rs71V`SYmJd)Vr|ks z^QCA<384KAOYqZ?OTL{q4Ceu-0kUa+nCCDKVXXaeHpd8pasvQzg{!bkN{u4^UkSBk zrvA-kICy%|Y|4UVh4ImNWSu@knb%TE+^y>-AR>5 zEjBn|b5z%8T;ud`!!kRNw5Ztq^fFQY2_Mml%A4S^p2UqMdOiN_na>EOr(?(qK4B`0sjk3IkSe1wCNg6cbAaCg2m9+jTP41o~3!0|D=aL@g3dl z<0||Ey`AvwmPGqaK&#!Xz~|T8P(ap$z%3TrF?}RYRNpVb+kUH4f}tX=@uW>MqP}mo zM6CE-BauV_*N8})^z1+H?hVb$&|2na^J;S!>UQk?-fo@GqaZoocm0BCiR^t*kN$V% z8y&r1$0ReIlufYsXr?(iA)ZtQ5<2Gu zMi~-{K&WuF29?DAHo0dmg}w3x26sqo=?`^3D1`|$gfCKe{LeuAV#e*F%PkjEVZa9J z<#D5rF%=7mpaSC~6qL%Wz>WhvRW;_RR4&8GOS80SHal8lfLFrTIbFi4P5Hw`Jxfwo z^%inBMmsnEh*Yg<|E8fYen4koUO>@&@l}BKOOQTW11lEgUNzW_JPpFjobyUV%L z#T{m3_$tGa6b+|qZT{%*RqaZqyyf$7NlI<8`wysDzn65Qsp=LFSeTPhRl zfjqAy68Q{$SPw8nGDFKCHVY+1HiO!1P-J@7cuK=UXa!cczbypVGXFD|Y9VqVyRHQ_ ze^b^xGFH6RG(p9sqo1*%nZ;YGsuRonRIz*168Rf>BD$87PLu6G2TiP^O4XA7Z#P=q zvj8o<5DI9`d#+`oNOku=7v!X}rl?JuvP}IJb441#Qyo_olBwhJH!tS$un2(pKnaH8 zAyOHW8$LqQ4Ya{PGRWxHllc=HM-&Pds;Hxgzcyldfw`lN1~3_Bs33zbT8v*sd0h6og}3Z!S~FsD7((ftexqs)?{2y9U&mCmNq*{fUS3+2x2 zru;X`QOE z&}uU_5{A=~mw+u7lfxJU0x((Kn(p}Ev71A>@*o_uwy@zdk$f*yNHbI{0tqtR()H*mJ1ynrhBa;ID0lRu~PAull9rmNQ(s_04M z-#^%zewR^FcC5OtAb?G`y~J>%>|mrQK^og-A8b-sgULVn7Vl*jk>>0~UU*=LsBwxuY-q?0$pW@Su|RjA{i}uWcTJ+>zqE9OVc@yx zbx4viwg~u)DL}4KJ@(NSLcAZh4p9@dAfoIpN2IhXwPO-}v@YP{bd*^U-=d;S>!2aR$!q<|b!Wqbt%BvBSb-BV6tvUyQG>&pw zB+2Oe-yC^mC3UHST;WBu|K?u9*dFDMdY0fIQ=_&Z7iY{y;m?LzgqWXmq-l@#ED^BZ zqru3wCD{0lNr$Y=JMPldWy1=+!m4#}a2t@f2=?0ZRkQE*)-Q(44_og!7(lQj^x)<$tGY@WxSlg)2v|NJZ)qs$ssMmBfeLY5G|0q*(`jeQlKo;> zCXT48KL;PDEANKQ7phw6Ifv2?ewK6L?b`n>)Exxcr{1plbM(8{5*^S(HcfnqdNa=z zS51J>=7qIAIwJTy1Iwo9R1?L$2#tv7Vf6)TQ6@2&MgBSK87VlupQnysiuFi~Jc4x+ z#b&1$tgD`wxXz9K&%56#(j()(_4Bj++=@~%;Qq1#pU5Y=v%+&0WJqF&(|V5o6kx1mu+wix?=ly->K84?UVr*0&fuzYW@suL2^k6bdQ*~aM~4Zaeyb_l!<#U-yd{>ZTV$x?C&9)Ow>+>1^@9+w zoEz=jj3lbL9$iUiVn$4l=F=XQl$la^t7q>Bc|&?b}9m~<@4Dkv}gPvbQ0^2%W`tJY@|7gdoU`P zHfw0w#@biZYb!ALAkfG^iFJf++Hi65nL@fYcf@ojJ56*jE^Igw@ySgUDMiReb@mg$_b!9kiHJyNRPx| zQ=Z6qgT`d@(`QdfXQAjDaiFnJt@UV`&}1l2prnVIUS}ibLR@}edq3H&tOzI*#~3po zcV5?Fxs}%o+*5C-SiAO%pO(mqx>llqj}iYKySfb+Bqy*)8vVL4CJYqveXcPth}gcc695VCqd_mj6IzXg2f5xdd}| z&4+q=)?=zbGf`79r&`p;OYHm5SeP0O{NV?p zMA{%W2M-UV9jamb zdT6#+38Bc1@l9^ZT5+;)`0^)_{G*2xpa-=0ULkj^lF0{_A5*}OPt@f56D655Anq#n zmvX6)>u6jb>dlKV3oJGQa*Ar$y@<0XK1&xc`cOnM0**^@+ffgW{WLDfhnAj}I%_?R z5$4qD2}&e_hN)^~Pi{W#r@5%qS{%q-aCR4aK6URTGJW5geSeE&D-x`W9aOb2XMo*l zjlZ3aO7+3JVoIDdLA9%tnv$#->5ruuVn5|T;ihJtJlxlQt8YzGYpFqiKV>tBwR$USyaFNzEjFmP1ig&EUDzMPq*8;+ZEwJWTZ3 z3_G|d@I8Em+d!IlV~Bl<@(07#^ecM#JQ(%yFTtGzmz=G+`&lCFJP7YeUl9lymOXA% zr{$Ik<}U z4-`pXt>Cj@&xnOgo(5cxhoWmP7;T@tFSo(lX6pxpabwd=EzHN2e!Di!L``i^SrcweJKh!s zdnxe}Pe< z1NzI?;ew1xFsYEqbZO*En20Czv*Y{y@8hY@iYbr5|FzqBy&a>Ofm^E#1sPYxTqg-b z(iM6{<5`g4c2U@I#^#Q&HTXboYzTf{g-2!B4{%BH4#ZxPPNi^cWEWZU=g zvORrQcsv)Z1)-F5w&^o&I!)y8o}*^=QUQ8}bw_pXR{%KTRQXga`HWxkC5UOZ?Jlrk z=GYP_si8KqG2Q2V<2yUp;*vem6AY9+=zguvwyFNkxAWf9El|S`um<^q8Yl;u`#k++ z_)_Do)#qtm@HMa7)U5E_eHai|HO&dAihR}5_!9wfJgV$6pU{a#B+w`{=SkkglU6p} zbUFpboOcGEDLk_rce+p&go&l0AQTx<)hiV4f#-*dcocqRSU>3#N~lh1?H?)8A78jw zTH<%cA4u&_#zW5{l=f_$9RG#wau!=R`3`x0r zA9vT*&86-)>1R;9$9w;&2D-qRYijEFupOn-NG&K4H`tvj094}CH(NMLrfF9Q1Tq-eH>1X}6~`2N`CVQcd# zuk{pvvSMD90y53DGbcbJnNlH>&ezcG!W{7tf8v31m%`6)JWs&! zQM}Zu5AX${`-ZeQ#cgpB+FR8a?rSeEm)g0cy3GwX77)?w1 z*J!vDj}NA*2mW|3NZ@@m|KGnbUEqGrd|ACcq8@*IF3Z>8_1wWv_Q%I26hJQ=~wk zo{xBxtZ!Z&)FSp~s#(q^5B%bRZVLNdP0hK3J?#?ryP4pN{f=oYU;Oe#yW)mlRHYAH zXic69-6>B7$ss=Xp&Q*kG^@^8bQ?sZJ7A_V%stm-2W<6JaXnws(+Q^U0h^E$<^(mE z?`7=p$1)bB2M1uMJctbSD?QE|Sce4F(u4G1HT7T^5(RLtTsDlS$ObFide?W>TSi-2 zuIST*@IkGqoI(ZJ5tjGzaq42odZ2GX$pk$+;v!eWTDxsz9WaS5&^e%rc%S*YtCv?E zJ!EZZ2t#B=Vnv|=O?9uzbb=1gO#a5R=kp^${)c`MsvLlD|B%7&nPEs!Z;2)+J9kk z?cB-|$1%$%{BAlMUCg!~iNrs04vh%s{~&8ZTxO4Lu9`j{m}>T}0q)3IDZgXr(TtTaou);*JI~GFL(s$h)pnA?0doY$YOgvlB${W!>HBw7ElII zf#UvnQpsa_TbZ|b8*aABmQP90avoInuDJQPB$zr2OL7{jsl#2VT_DYyb{&?9QbpY_HzLl7Ijm*DLFg-t4P$>H)^=c6fYcPBBU`cN`F#y`Uk~|pgLX3I5NA= z^Q*&nZLq&Hcsd#Wm5Ao=x@(7`1x^F%!!FoO7O=mqenVw8eO;A~L$uX&&Z6!gqw|$R^DkRJ zz?9TVUvm0eg$ar+N7ubUKVP*M{DLbQ@(`-!1jd`1KUdT~y`@HTygpGEyR_E?Y z&uWB+(F~7#kRcHju{-6Vs*&auUcmhuXFA(TvU(ZB) zrzY0$H7}yTJkmjANM4}Q&nYg+J;Tryp|MGf-G#5XPdiUD z|8dU(MLXWyCPQJTS&5qdn}5>i&1_K?v%dA!P=wiF71jz{$jjCsy@FEBR9=T^ciMdW zrSeOk^>V36`iFX{@sepayg4}s4@X2L&-_D9f6KQ==%+g`=1)_o$y25cx_w1BY9=5B zSWtv2Kn|jn9rDNh7(|z6b++H%*=Ka$9r{b#MEq|iGaN6K zjYn_F%k#&$ULCoi=eR6cm86rocfYOb>QNPpm!Sbm!B(jS=;&1pmYRa-R3e~>C^bAC z?4^-4v)!xAEA;3q-q!n|+#7y?g<$-Ew2yErbwge-lSgi@l^8Q8QZT#7 zvnC3RYG^~^{rRL*!}=}Rj6rGy>mk!sTw<y_0u+@Am=eMHjM+i{_jdKw zuZmn`ysA6FQ-(hXhonrI3Hcx<6R7gY^Bm0xs1;U)bvu()o>ZY5it~7aFY60b6KxIX z5T)*$nWl+f@N3R2Fy6{n{@31R)HZA-ltHEdU8RwvD!RyvB=|s-M1Wy?!`8GN&CvFn zb@A7+v2f|PZRDFK88h8jMu#MZ!)cMs(?Gz%4K-3Humz;m_PN!07niL4x|$p#7o)%A)H2Cu{8P)HLwZGo4KWvk2hoX zIVb-b{fGNL7=c|gJ3&v#LO?n=+$I@E;4kRN_4azBX)93c=&>$up_%Jdc@V$JM@N%d z#n?%w7L>-Bk37??<@OSSyEW&Ibt+S<^(I4u;1!J;P*Lg$#jQ~A9jNYxCyW3!X}s6i zDB3+WO&U7LK~1Na(K-!1G!nKN;`k!M2N_h*K0;_2oz~mna>9fPcRQw%#GDVL4hLrz zk&~vUHf3*pkl#yz)dvW{o%(hE1a|lwSni{nt=sd=8Ehk=+dJ!fP{J}_Z2k)?m z?g6W$awMiI)oFz*b*hV!IX0u-JFublzT8sss{>@Iqt+!#UGzLcc}juROF=UO8UpK0 zV6)d4+%M4^mfQ^%wvf#;iNTh)HbKgX^ieIr_OIUNkN)mUJ!D1|E9Y4bC&r2i{s3WM z*G$Uczh&Ql36^ENg_hH}Jn#{oMoR&yQtL<*Dg{Z5&v24(B+dnxuy=NU;IYj1<*}b` z9$W}3$mNEoa{MhY=|n{&Vy3}o!~IjH##AloADGar1d#+6wS{adNUFRrWxX<0y7=5$ zyr{(Ym#X|(x!?&(hY8F9=SoZTETIF|4A$Be7$5Qu1&EnAkyq-65yc;4o2Y-{Hxi@m zu86;-{(vTVpxT28UC9Y2a|AX5t5rdT>H2%! zD9wpX6Mrv>#*r?ZP=AK2S~B5kgh-u04rNm>J~(i(FXI7#5)l8t^g$jmyS*kN?(x*x@FB4UB zCg)L?!(>lvNP0)`ymY|wlVwdibfb{q`G`0XG7?|Hq{3O}>CZigmk`juA3wwha@7ksMzl{&`THm{g;v}DsTUY)X zt!gW`$CNX!j%Bi`o#f`m%Z}(!e}0RKYUA#py(#>BX&uG}xf^az8ptocM;+`m!?6^0$~uo7jStBQ(PR8k zrQ*oVUs*n-y}Spl zppAOQp4eVb;B>~@@MchFCCgfPH6@9tUXofxrpoxq0^7#}xAb0>b~S>6lJef8@Kr@& zY-Z{N}p)1fkVw{u}iZ8b>=%gRj|fy&vF5|@sBVMw{oZ9dop2pazisW*kzhT(b@)8X;g--kIwyy!Xj%>o$6?k zLw9&dQ?%HOy_busXNMIfhi-wI{xW(B3Qq zEm0`zKnvAOH4RhrUuyH5?(np`Kh?IH&^q6qNXt=mm(P{%yjutxjMm?`bX)CyaYc6& zOust`rr7)be)sk{eWsU%A*v+E25cjlzA<>}n-3z1glANinMq_QfdEe0DvkQ5orokZ z2109~8G9-oeRX|t;PO!`7E!KiOpUaRsH-$r%Sfpjec3vF7CWslnpGvzGPVYF=vJ$a zy7KX!t&^8c{se}NH(I);Mmulu@Af>-xSHTZ;7c6Ai1Y!C+Y50S&oLmt4-v03kdm+1 zDK{-&7P_8-oKDj-h0(4f$( zy33HPFUw`oXks9&F^e?6FERKo2Qk=i8~xuE`e(K<6=!k!7^Kb3p|akTEopRm<$?rBXus;+iH6mXz* zVpXhWRVa$0s?;(KoJ5h402BiVPT;W+YE&iJE9PHX&}4 ztc;NmL9kLI8e0UaliHSuhAc?k=|Ik)w%7EJDUV2bHQe}(Eo1Qovq7oxP5^PIvZ?)h zd_IVQ^9+f1$PYWpRfMjv^{L+R()u=FVpNe54{{#(sEMFNxri;9nCCWwKdeMewTP*} zG(hSMo`)&FC}R9oui?8=!H~*{WJIOnfcs;p7~+;-dLHquUf%f{ad$;1cI#-D744sf zlrAL(=Q+xREm6yyo{pgP8^W4C*xpyB<$;Ng=3I17l5w9pct#^4i$W}LE#@yoo5Ob$Is!s58M7S;8Uyyk65IKykaD=vMJoexl#IzMFU z$JV`MOvz^Kaqhzs!=#9^mh{broY3}4&)!V@I~7t1ufiK66RXA@o^gkM0{3j*^~crQ zrDyN9?^L6eP|1Pd3h5GCRZA3?$W+Xcm@sL3>O0?4X@Ua3gGGt3@At2X1Bd@B3iuW$wXO`RFluZ(;r!>kUru z<}LT7qyrs^!+C=bs;5?g~`tv*Exkxs+VsHe7-5OlOHX_0{?3e#>WgZ+v&% zAH-PZT`2jNkM80=?;-6~V*?DD$B~{PkU))-kU2`R-LMycNwqC>Eiz^otAnNF#TR$M z1EpCqojEn|j!Mv;l|p9j?O3S08n<0vCgLhy80zRE^gJ_ME>VGy$<4?=kCgYX8Slq~ zX#ZEa_we1*^ROA$ZxijmQ!~uJh0=W?!;qV%@Nd-zg}6sYRHucxi|6S~z=&Q)!x}Um zeP$RV$(1ul{YEps=KN($#DC(1Zmf==`Ht0X3Kh-gq z0MYDGu+%7t_c*pXI>UV#P{~eeK3c(T#{_<`GGsEU#P&f-@2VZ`1{=kJnd88}{n16GOYJ;P zH=}c_0e{LbDE)~|`3ej6Nh#5bM`>#v=tW5R|GgSQ-Z#JtCswrYl{Is|nxRQoVI?OJ zC>0=!*wdSJtX&`dzvj-lq0;t?_u1IluE{ng+um7|ZP!kdZM!DdWY;v+WKK=CZR70c zcV58x_q>5y*LAOTt+l?NwPIpO98}T*Ry71fbcef}K1lzNKvkjWOflC7Pci98U$_dKtFRM+BS?Q=T^_p8Z=(acaG+#I@qn% z^j;-~caKH>J@FT=yW(X+Xcs||~1ag19YqG1z`#4INpa!{W6Y74{ey*-H?E?6 zLxeMGCgE>#qnM61nlzNzlW6irrGR_4gpw1n!!WaVyIo74A* z3@^Ve%KdD=gH(yl_$4wf5}XJKI)meSWzn;{40~wNpE;Y)j-LeFB1@V#VriTi__i^% zUfR^qihFG*KkU*k*6*;kN}X2#@^(&UW@~=(75u$9i3nhX>F3oPMvLE%Anr<0^_32$ zH#@1#^NYN1+%oz}8XUmhns?)9ghN(Xq3Yt5V2b_Jd~)?t)<95H<0gGN`4pz!b`30Mp^Y(v7Gr zO_&_^4Pyh=-`;sdI|=$e?(LtdO3(c8#DsU((xgh@MhDEMQBBYF-|j+Wu0 zvMFW&p@vB{lVnR!7dE7HPvz?4|SPS>dPr{ zfDoZ;Gr^$W_Hb3;ShE=!rL@wAd$U%0(xhAmv=m%ElO<|;mG@?(QKXH3M19er5&>&W z^LeNziUJNcVA{}9I=T18)Xehl@x%Bs2Vm{FZ$qG4c zPfbA0B1VrtRksYN{x}ur+Sb>?zR8Q<2?hPc=kez=%eE|!pJV*A__A*7afe*$`aU54 z@ZhTZxhFXp;5tYFuH-9X^ZDr|Ho%Rt%319TW%Clwb!&tlL_|Fkcn0QX85Y$L;?>D$ zgsCZ}hyE$#2wA)@n>qtdQJA;+;T}X8**df)8NQsHpVY}}#9i-~ z2UcPCk3=FL{6`+aMM3e!Wg!F47edh#Akp%x;R48`RC*d11xo3_|3S1Y23CCRCY49F z^6U2iaHkE+*?4OMpYS^d%iovTe&pYQd9ef)cI1*DzaSAkZpz++lfO|vKLzwWC%mes z0rpfUo8rPqG@3-Gk`BIT64j6HI@fddjl`8wyJ}|HDy%!7D|h)&(XfzJmE)UAd^?Bgbc>W+w3jo4qdFNwKs3l6synUMC(;}pq5w^zn9r#HA zSa3zlZr(tjGvo2+Iw9KG=vnJ3s7z(cW8e(%%>pRV=88HEq zRa1jwl^>Y8r(Ye6`s9-HwA+CYUnTb>Te^jPa`jAHA8gOb@{NFGaAv05ORl!bx-mZ?=~mudysN zRkHx6%xdYHD+)DWMFTTZU(tnBiQ5yqO3A$XlFF$M(U_;FJRvjrOd{W3JIE_r?A9}p zU50CDmCY)k-js7&0)Z!F%et;{360OMBWsl<2`Ig%i1gK?g-DcPUF_Ht&m zl^fUf9t9^EbU9`9L<{!$SXDO#obxsz>14Ug_|I&zM{?IXe*}Wm-~&vX&$#iruh+jH zWEfWg`?xgrKVu1ge6v_IiG*CulWuK~es*f5cDL$(qZE}P?uuUE#obJ>o45R}9gRFc zc*0b_y$*84DB?s`%c=QbOAl-SACj4C*2lD%Zq>@#DONFSW@mgZ3AjcJ56SsFT5#Ub z(*+esFcd0ecG<~yQ2R;}5y?S_E#EY%K462;FpKo?bV~pBxvf^nUx3TCsD2(Q5O>(-|ZSO`UWMoZrZu=2970A zBO_B&{>Evm^FkFj`jGYfi?Uzv$GDY*sG>XWZ9HLQ5Lg@uUBv2D4AP<<{z!EA?+y^B z7RNk+1^WN8kC=MMoqrU%7tYw7VEKJ*H4!b!ir*-7<;2wv)(V%w!IVF(8Wy>Je%LTI zb{VN&3^3}>xbUErZZ)xYR0-cy{)xgr5F^&?8^GKRyH&kg`RcxW?S%HC`bg;Vhf7C* zM}a80hdEAR6U9x0ufB*Mi70AJXoV3mF8o{dHszIOn2dS@WANYDQHeM)Rc}lNwW1S+ zsK_U4k=98fdiHr!*mxs5%W4q!=JTR0sS#d(HVRP>!Z&llNYat5hSFhR@YViIc23#B zxg6uNR`G8WQnOp(d_{boMo9ZnR@^{HeGSu~(z=XK?{;p0b7Eozr5cXRa4X5ZQ8|J| zmcNs(I5_?lsjqVTr@w7GN&A^l#VGF!BI*ASBRe)C2jl4ykk6!gRv%Vb?_-aVcl^{6 z1(46s6aHSjL=wQ&a32aZb$my=MgNMgs&v(n-)o7Rtwj^2^T7$DxR*iW2Pck{rmx8{p(Nvcv>F z2swds^b%S-JtQZ;?ToYA#ydWIM7Xc%&u!yO+vBvGzCzX$ou| zZv$9d-&`sr-*%9x+Bc5&rqNzYvAu@>(@n0b;Pn$Eoe4w?YsvCerV1+*+E7*jT-1oM z@roQut(cH#T$b4A!v5GOmXQAv*V%yC`UYoC9umWY!iz{*BI#Q^;cY{&L4tmRheB+A za0sCYesM3(Syd+yZJHY=39IW*?DP!~Nii+LC^9)=>*$QsS5@N`fySYlg53c>HYyW- zRK5!(7$0vm$zLbA1jMg&EC$n&cRP1UwPnUj$Q;3N3(2$s$$%b?)zKBJ^wqw+>W@Ci zZV|Au`5f%}-uDpBbIv(G@0eDa41_?B@AU-GJJ?V9qYU~_;XN8eV@2LX$o-%Jy_xeZA{57;9Gnp*%I z+lF+_1TEmI$a3pvL2+_)ZfINwGa)irmLx5u%$HC~XQ`JkvkTj<4#CABRg5E)(D)uM z1A#PHAg&nq%6%_r@xQ0Z@=O$3lkkP12MbIc`* ziyzr#k&H_V06wMM03278#y6i&pH;WiBzPUrbf!)VgFoqw8XKC|O(NaeUSWb$8(1s! zpk{p{-iWLF8IcX?qH#@lhVL64FqHl{z@{u(AS>%=>A#x>9+YSCG6S-&WPZS{X)7}YlB4+Z^q#R(b4}MB(*yn zm%qZIlGTj9v%?AS74QDe)0GtiAq!W`BKk~)ijWi+KA~v}M~#PgO^61t+hRHhhv0@M zwmmrM>za9=D=WL`Ync5VD%`s+58l_|P|F-QQ%Z1hhP9`;F=CSMT1*^lzDV`BZDS#_ zczcb1yptP>tBVaeg}thSDWUTSb|Nt?Nn7|I$JafNrDVro#Rn4aJ(-4l@f7GW39FHm3%q{#1-?q>7%)qjW3oUIsTjrRz4`X zTf7stQ?XhL2I*^mAdWx*(j_}_>$1adRMnjEng2{71}3KI*uf>BEs4xQo0b=zbr$QE zhppLyO7@rd0f`*m9FeL(X9b8qLG+K=>`@(?O8;HykV*W(?`Voy&g+<4{gDIX@c;*` z|7P=j>3XLrh~_ghnTutvhYWB>lf=mV-XPockeRuta9m76Kt4p%V9O+(98}&}%*ac) z45|1jD-qMt&2Ag>c>xrJAxpApX+!@^>sns#P$*;#kQwt1B7C}U&$#> zgO;lrNIhJj7zGekX7hhVjn?F04@3D-#>QA5_moZPgS?xbFR{HjrxZ2~z$@X()Q_RK z-vYypmmOXx>!%d~w%4)n>Nq?GiBQ#CGln`L;Ixxk+I>m^n!6stIPhA=K~#;wEa5}AAYn+!i&iKv*>g*ZI)WCY7+rc8 z(10#WtC*TW*3#BkL{TCaG6?I}%Z-)&&rG_McaqI$HCPQlQJj>31`u9-_Mk+|hp#Cu ziFE%cMz&K24r^+qX@bK|E5^ml<3erN-5ZX+Mxvx{T(O*PT645VncJh3$QXw(lC_=Q zHT>{3IgwrNY1URa9baFQZ?Jy|Gu(X2UxB=nK@pt1<$(Ncc^O&=MHctR64U8{S!RCb zJXX*^r5FMnwE%V^!v(scEYysU%`eHD=ojh|`&X~&$ZI?bu z$%)FS%C6Su?E$YlyoyYvSJn+s_vETk-0hH`6hMU!i2WaxOSq6qM~438@9zo#mt(Le zCRlyW6k-EuV*!&(4Bb$V020cznf7iuv7s>4!zvbp$%Cu0VXjjX3^h~L87QSfQG_k_ ztw~-r*fDF}55pACq?W;wG`O%alA$b^_iPeQ-Rk_>xor#lxZ_XfWh~1wQs`mt;e5ptLZM1pF<=UiSQ(4RJX7 z4R?yn!jcmuK?2G7M0O1Ew3yD7#$kAAVX8P2i2k>qi!v-aT9v(@xse|1d+sw$%!UXj z?alPqyI3yN;m4qDsiI>kkSBbhw}y8Mm)c{518{2(Q&~T(T*i- zI&qC6g{&}G**X8Y8$K*erWh6v2_4lmZaM)>U+xt(pv%l1v6z}isV&D+!WlR&Be=GA zqTdCj&^nwk_B#uK7venzlQA#lowhOAjHF*QRy(q?4UtH+hbtB7*~Wm3H*+ z@+^$Y4U&Fe^55W!B<~OnqtNv%kp1qg3BqW#-*3l+;OYMJeY8>#4=7QVbofupU48i_ za0h1-@lHYq%v?hL$6bE0GyVMOQ#lkzOe=ksc%y71exo3Js!%cYCN$Cmb&BWh4!w1NUv`!Xvat)d*7B7;aPjA^-;1oN$c zROQD>z8hseYH@Xsz{L{}5jJIdK(j9M-wWO-X`ZA^=w(+rXzvz0!C6H^dp05j^OjsB za7RRH16AW`Hvb!-8X*zf(O$$SEQglIg;RcAhhT;#(ZB5NXoyOx*ao+n*5sj}MRJB# zDMVC0CeDhom7HgsW#a$LGhOIaBmcCd_A}?v?s7@yBo8;o2#44dMI|3~O94FCMeptt z2*C`~GKdQ2wu=CfE)T@2re-;>192Nj9pjZsygXSyoa6_~@7Zn_#5gdc>`wRg5j8 z6ST2we^>h_3}R3yR8Z0tchni@c+fbaUT<2)rwK#^ofh_FC9UyM>6zFxPGIP0k<2es zB8hFW(MIpDh0TOrBym#Z#;9ygZ{dh*(e3iuYBQ+OF@(?@lZ(e?+*ty^I_+Y7B!Sk| z9|=H?s_+!uCTxDkn^^~uB{-tlNXE?GaMPUdu;+c!@VQ(5cSB$MiJ^lOZ?H?!t+!z zRpy8V7kDvIOdYrwx5+2~@gy`-3mJddIn+$i={yfrdEcnYSof2M^>%WLcIZpJR& zd9WOS{&MAsr5U>rWlutRYmzI|+Dcu0m}9+ONq3d&Hmc^0K56E5W~rWO{)PNHQJEJw zx~X~fs`*vbRYJxpOdjj(iLHNuSr{9YlGknM%z$=HxGjCM-jZWGI3aE9I5m1;&Mawi zs9ahfg?+p+%`v|ZsPHyv6^;dJIFlj14TShpg0U7iGTLk zyU}s|n5gDNrtCx*;-Adk6n$L8OVZp7&8(StNWq*;KKbY9anm=UwMkiaD%Nt^9pUD} z+3n*t@bXpUmwau_VMM*a49%k1UuSb<@W0YUZUj{Z%sa?{=tV6o@-fzDUrCrOaTb}G zDFn)1v2^UIb6igdz3Q>o`vW5O5bgfh9h;R+@Dg|%Yfe|4_~#$a-Y+pN#(a?^$Ly@q zNiO^+^ptGX_^1!QIQZBfy0W#0NeJAKzF$Ii4m1MGPg3)mqBI_DawYTEZX3E~e%_|y zLITMrS?&5)xvt*=|HcPE$r?cpT6)N^^DEuet36=X4YqQQ}! zpwp>wL&(pOh|2E9d@_vFt{W0;%SV z9saE7-ul=;N%NK?<%HzRocTv+j-HOy()Ogv4Fb1?{jgb*yX|vr@TLfeE!KP`j(^(? z(COidwtvPf{7v52s19@=k;dDL0?V=&Q)Vbh(wwy<9+)4FePV?~zK{J%x!MOp{)P9i zNI!DUcd-??E_m9M|Gqi(OMB)~ zgrS5Dj-a66r)3Zs#1>|<69q~GmA@*yScoE9A z)UGe)5H>y{xFhQx=|s3vyx~dcCZyfcb)(M$Ze}3%pqgJD8b15ua_fo%Z>SCSeHwBg z+r#rhtxPyvyaOBe^535{_qVJ^S>d);eP-G;ooQhdAQ&?Z&$K}(75LcFi2zr`My9F^ zLc@2l<=HgMdF-H2!G7rV7O=rNKa$_N^Y^-b-7M?A4}c(xJnE)@>~~4!^??~GKB)~2 zcj)!Wk*!N|?;{mZ=&L_|@*xy{j;Z5%C;&HqJTsU}{=-UI-35A5tkrbde=11>d}szW z73*;9%P?M->Fm5N#f4JNQ4d+Db-!PjJ$BIytVZoFgOieuwBS?)EqsP_^5MsIAj!(V za8#b;x6u40qg=liw&=X_IBHai6o7W2YdpoKhVpdBSW+vI0LNJG3(``-MGVb_rlP{W z^Ul*Z4K2`K2gRYZ`b5YTv|50^rjzqV-0owR~PIoCv`$T5!6n4>I-uJQJK8TlqQwrj9Ya{x9xyJ z(iwa*JMX+-0No`gWWna#xc9WDIa;4fsL=a1rQRwz6JSZBm2~Ly2nd!p*DW}TS}VyA zYO@I~ukl}|H_Vr{`B0tg(e!-suoUJxl2}8QWFHE~oRZD&iJz*~)fL{9UgSeDejzAG zjmHguT*SAM`1`m;!;lT_@T=LpgCLQq>E$CQs5`>{92xDo3;+EHQWL!!j-?0TU|M9K zs2E*6g9rW&V=iqg4IKAPT)-f@9mf zE&RKHXgyi&%iEEvfUcMS0K4$CsK8OlIz?2ZD||35Tdne?pzsPT-w_c$duoehH`7n} zAFb|7E(nSEk6QnkaHS16)5{<0=vL4uBDII!@Yt${Z5ZCgPwVQ{uYGMA;$5X^4UdQn z=qMfm|6cFhK!VO>U_s^UNfDOwC@aJh&KU~}=fewQL2~mrP1pH zkdgKC>tzJmCvVUvD+G#swmIX^-tV6h>+`Fu@#&w_pJ(DvCst^4;#Dhm6WTrx$!ybX z?g+(|7ezg-@@%yeiot@qGykgT4Mo&%$qK5Trqz@5W9@mtPa`MW8+Nf{EPHyu?2aqL zWp_b)7*F>`aoA2a-vTG#bD}i*HKF8_^|i^B2)ABAv>iHy2C~jo-AP{H!e782tM`RP z9%~W*sSnzOKVf}>lKiv4a%xj=qA4`5aC*}QPNV?VTE%YeFCD(D1TrWX9HlOB`FV;n$G$B+liIu^Jr1eh)7AHKvB-v6OEO>7<$oLM|CeQ z|Ks(yNzS_k!%)vpc)P5y$3@roo!%-tA(7lb$B8Z=@pa<2GwPP$uTo(B7L(AD3hoV2 zS^(C_O4CS&Bjq%BY=bI-?+m1tmQplB*$3q{7ZYxeZ;;=0dU15c55i2wzX$F*?Y8;z zDai zB3Oy22$O95-QBUo?X}wUW1W-XnO^x+1bJr{DYi@9_Qb_YA>cBc)zj7WP59mb6?Zxv zmtMEx08{)nM}CxqFyMnb^@}jv&G>{Ps`IXim2H)u@Gx_~2ads%79AFe@wof88RO?z zi?mgX;+@{fvjvorn8(QT;C**D<=Z@K|58U$tw%y?6tYh;IoKGL#|a<7z5LfAGti8V zUPMC$gVnri?kJ@-WTC(uz2GZ}J%@nr)~T)3uczj!%sOR*qnk$Fsc)%I>YSxd3v3xYU% zm5Y*=Bj14=sjsCmO@~i&dOp6=`fYrP(5o^9*uaqbpz9t&A5ZlC8o%tt>r{}~VB22&-MtY8FzbS>H{iN9Tq1QWv?4Se#eRE;Izn<(&vlfp z55M+$j*O1D6>ORBQ8|}snBbv~?Eaj?>0X$)K?V{%|LYZL^Ib?we%C5K?B0DZ&7>%P z&p(1liuWzRjKh)DwCWN*=pCduT~y4jlCWvJ8}2h|M9#R$D@m z$XF^_;(n&@H>a5 zI0Y9pl)tGiJ6xNRO0#3A`lZ~-a?*7oF{Om-;$&%=XF#?M+=oW+lZYTHYyS#_Ox*}} zYZFUDWYd?}s$}MO>nf;=d~=}<2J69;KJ ze9iJmDm~j&J^YNPN`9wh{f}vIo6{-dZq`~ssOR}HB3K@3zX|ILjcRfZmwD~*DZhy}0&C5yPV zi9C54(UVaf8pCcA*!5El35)&_C#@2_S|dw)5q)|sdJQkQ>iO7f+j;%VDf+7>+!l>; z+`;_sEsr*$0CE{JIO^Q+h7K&%qB}?rHx5!IP<*H8wJ$iAD)F2hmg|43#yK!%TJC6W zWQMCm@Cxfqqv#Ga?G3P)cs|UXKO}G$FN%PSm8i7EabifWfN^5b(583DXp`;4D={!$lzf^n$l2garA{l6coQGGoxVSTD_pzH z+|g@+2Ipeu$AuCRaMGR2 zU`nHGDM&zV(*ZpswT}<1NQ^C?GbrC_esMy1)OKS=qhVyL+J-dSXmKhEppPCV{UzAX zde8lfzsCw=JV{px%IAOH7b|H0I@}RH(x*Q*V4r?kr09O4EaN!Yc2fp{;%5?_byB4V zjWb*-KWHJUP@0XIa;{^)$5CB5euuf?6)-D&nxIrMj{9!LS%lkbej;v46|oAjQnjL8 zwouq!N!^6C&y}TU4Vvs~N&bQS`-~wCrVny!9$LUSiI$u&@kiIJJtAOr(kIe;FAfyH z@ASc{T&A^i0mq?lWbhaLqXID%L zl@79%ygAAmCmM$?{g;l}t70d%WCquGx+bn8b=)nl%bU~Jh%wGjRD&CvpiU@tODk04 zpLec)#|RBIP}TPGA!93Myl*KWP}4np)4348+QT1>h@W8|saY}<_!i|D;JFW`l!WTm zsPgn93?J=QLYg(e9|-1SzBq*g-7zQ(%yhL3?n-jNm_LJ_1vPuY=`>;AEy~<(tMl|h zr+Zf_=k=wZIMu{fJMn2oYr9+nMAE za>t95f92>2*E?5m{*4c8G;rnrScBlvl`@?7E=dW5dlL_- zgCAfgBbvy6Ls<R6LzojULQh_9r0>FQ0>0e zFHHWD{V(-E%=CMvB=)Pd>iTn8Ca);PpE*ljoLl(o z0#)WaJT}4ziHme%t4~Y3PYtNx>{)vD2RQToe`kJ~nN$3SS^a-C&HLECL$B3c-PPTDS8dc) z`-)VM6NiWS1p@>G1TQHeq67p4obYpCK|%bqIO!C40s-0TONt1pc;sAvhC7?SrS3xe z&Axr!_+spkE_6>!Jcwf0Cd;~6Z+2TsGS?<%Vq2z=gV_|ndMi{&(Kj-o? zPygpu1jJ7d5n>$T|Lq&`fBXLb@W368x0XpbDv%2>6i^c)GLrLs=)ta4KDdevk;s+x ztHBE77wzS@At21R{*|b=-tpF&^kr(NZ1|=cQlGgA6*^$8ml(<7y9-re3$gbdRLbBz zX(3o!E+}Ff;q`Y3QtxYL$Mlu!^;g3xogJxn%olIc@cYqun9B3-5#lBs4$wV$F3k+U zX%%sWS%MRO$uE^lU>(>ftpGJv3urKNh)}4JZ2_=CBf&;?Zc0iJR3-tgyzML!En^81 zfE|ZeaHcb&N+u)po+c0hxB~W*WCTyrBak1^9v7)TuoH43jF6tO1bo&qQ6b&0s;LwS zB<-hlH)u0Jz<@$)tne;WLI8zv2Sx-=FJe2y1AK#Za57tp+TI%I%_5K=rJYf}-NDFa zs7Vl_oqN;n`!&1SY)_ZHi4)E9;UE`Hm9asdLYbrh85#@3@-I{rMOQP9D7+$KH18Rs zb_t=V7>&)UE|>>rm@MJES?&=Z=E)P^i;dY8>Y?{bpPnO~;K0NA03L9O4yfrbkggYm zle+D&Ef~Q`;}Dx5JlP-*kO5MaQ81NID;Q#PwW8B#Fm)T|4*ydmC!`HRJ&1W8krP25 z^4SN`*>R>E?ccSpb_WB@&}4kspf}7w@$mmxMA+n52<6#n1#ZCZLhTZ`J1qbdvNWem z-(OCr!-eM?7U>L~38#l5Dv;+$&|#UD87kp|6~ZktaOG8P^-F`Wg1eyS{9%Lvt<@v9 zljP_vxGHKAmYuMADY&uB)w+6Tb7LYS?%_(WazY&G99oVir9jqs(GDZjKzevjlAFd` z5O=hRA8e#4nI&XNh>c-cCb8OEx>d!^9>gvf2wWFjV&*X!;($3Qv>_RmnkXWQRGUbP7-2yy5M}HrFc@Yme3))|ngY$WxNc9_Zu^+`+uR*8QVJpg!&eCe zG0YAv$>)K{Xl7q%r~B<8_uP8#A2>b=e-4T;2baoyd}zL9=>@{&wm=@B6N_) zBX)Cg%(JKFn+G$sZsdToKB=Jb5XUv*=hej)xDStyy5l&U z1){Nx;c_4pcGH;K**k!s)1%RKO|d*LakW;R^gMhsNOMi}GiP*}Z1*YE@_@T3k^Z2u zz=^?2nT~$qly?xZmPkoX)J?qID}yMY5)^z;NqG7f>hMz)Kt!h1jHa zU!$*;g|$Sw95E$K{eH_Dz%y_QZ#!)fWfOW5$YT<@+o_d%;cm`#;Kk%zfp_Y_tH=Z2 z{XNsg<4D3B2-615AC0yiW%rhBCa_i)gph>>^vMgkAQ;Fbe64QLacSD_okgazAj zh}mS7(;@*_anFTvEk0*jJy;za^@IOiBFiu9jXwG&>4NPKKFJAoFE8E15^l>9;x2~4 z1)4_-`6Hl0o^V{b(3ifC?jSR)Cfm5a)vG?$l=)hTleSOzEqIk6mCt5{t-9UGVR=_=_GZ>m%bG$hwa#)~lH z0gAp2p=ime?$b^OV$VtP4ff6Vxwm4B-a1C3sRAvH***f@I_P^P4_F8i%~Gpa!68_0 zXVUTMDHCsZPUwshYBfK*axOR9QC$g*soPddFSSTnhOyBz<2g_UcnNeOib{aZxtZJH zYojLCTVWu*2Csrl$+oYj#ZY>fWS z66kiXPUz7$>chxu0xP2MkfSBN*+M34fw&xG(7QZLTi@Ln72!!{0AIfulD2hvB^KQ7 zNwU-5@i#|YbmAMaohR+?hia|)Q5OTv+=PLeg9g%za{usQP(D%&m<2iae1@?^- z6ffX1T8+)T-q^Pc{pb2y0iLiXLW_m~6NkvB`0D(@nxr&8kB-?)j!ZuvfWp!^R^9Z6_@04F6vPEHA%M z`k>EyqUTsW6Nkpq)LWZBcrvU-K8irZ1z8Sqq`i*YdE!J=zy5gfe z>Jz{&r@guwMQOLvt0_USM3#YgRUCPs&%`mhJDPG_`4xWSbdDkF;YuUUs(cvE_N>B5 zfOV?Ev9uo@wlKoSnD+P4n!d-_+d+fe(91;!(tGnZsIPOa=MOQ+wS zNVo3LE3imMv67n+ZwS?2y+WYo%cFkm35?(0Yn`)8$MJ{j%65=kzxn^3mNtKQxqvqF zRSZQCwj^L&E~Ll|Tjv!8zS6?Cu_-a&vYB(pgfd6Fsn@yQGgU$SX7V}i|8VqzNvtLD zA`Bkdd0+?6uyr|pZnnPew3fxjrhdE+Irk*~WqyiMt=;4>OQ%QRwY6Gr1P*eQL)l;E zxq@frgxYCgSx3vx^!q~r#cUK+M=GN_&lGJBtveVf5rRcu%tAMPB{r>2r#6u~U!kf8 zX0XMXFOZpNtAUy~j$riPu7q0Y$W_8P#TvH-p-cL%sPhfniz>f>@2);bq-4IvqRG<2 z21F4g69>)NU42^jKH^O6$+`|VKemd#Q=H!fBLOWb**_y!?;Dh7X}?=*t7bv;>LRb$ zOgoj_Zmv`(Z~y6hEXgzDfjVe&R-~raQs>KeiybS=J|^KLJcT<$Bzf+l7JsLgnEQ_W zgFKD+HDpM1=9Ya|@qf7i51Q;=PWb0X;(JN{P-e<6u{MnJpxC%eGme+D^mKilK>m9Q z)BSsyG?spIoa$64(<}6in4R{HSFEzyL!_~aP{j%|Gw!C2oz<(k!n^?g-lN8i{`6|N z47iEKC4)A&2t0pGL9R2pS*gPZ+$`tP!AFNL@X@+0YF3~qkvDv1+{-7g+<&S*ak@xM z@<-r}2T3nm@b352`XkNX9S|}tkiIqEp%wxVkjGTvV&T&CuH0T&GP9IVNT?H+OoWL$ zMT%30t&Nd8%J_p8LjRC~P}?_*l%IBLygF5!eOx(QXQTEA7!EBTB>H0JK8=s60VdOn z3!1p)J&(jbmN0umU{tSS6E_bcqshcZrh7DOShDvq(hKPd{wHKjFO(kG@u;9};T8gI znRycNN?QMaeC)uISq4|RM*AmZgaKA>uJsR zYuj(f+`qcx7o3xblS-YIl&9)4v3wJO*5g6gfPigT;@Ujmqxj` zV{=Lu(RY9yKK|?X8oKS-uip54i9`c)H=FF2e=Ogz~rB^cM z?4=RtE14^hZ)l-Y?=u}<#Qk)|TrTjD(fRCnrsJIRfcr97oEwarT@|E?5(n}<+>&k| z*y&CGndM4Dg@2X}(Sy<2KOtl@24uQbM(_K}=iQnXI-idF%W~EQscHvKT!#+7SZPv7 zi)^mKYYsVnndfDh2Tyb22*=;U_&VC(w(8?gVc`VwPA9|>M$<{Y;Y-c|!Ut zBPr&p#~rTmG9?b32ha9f=RVgY?@9Ex(nmVZ?R)B!HqIV$yoWQ;{vx47#jMnRVE7BF`+ zRv3Nl7^LQZ9L3kZQ&Mwornbu;bJCkLz=<*(?wEfIyBtBV`zu*N)s=h?%uAxn+F^SgQYYg4zM%%O_@yFPR zf>Ol>*k@QDo2hj_afA5;=O`&U#(YI?jSp3|P!y(Glv7e1BEWJZQ|w|GchA6h71t>@ zO9+$C+W7JRD^j>%w0eHg3$#RCOrdcw(z7&; zi-0!8nzBo4YWSKLwsI-TNwJ=mM=0LX1~gt=3^m|`kS2EmiQ&%gCoRScI~udX_ML2@ zk78Z0V?PVP=jH@k@Kl*(3Nr3(G=&XK>zWyj6;TkU(&!FG_71eCD zh>$CsciIZZL0iz8ZQHB+yx%U4!Uz3_o|&O}B%B!OjjZJDc`iq{cy0}minmzmUFX1N zDUY>fczX!L5OiX1(HNzRv;D>gTjvubX`&qHLy6{mo5xMc%aPsLwKVzji$KrzYqPYD zjfPKNr1&K^<9&@-M!JT5dWKwPHVxO;sx5Ugf0pgbgBDR=E%Er814iPMcgyo}_v`@Y zR`{OAXCx(fw5F(N^|BFOym7U`+3Iv9B{~u@>8QBrVaDC@jrto_=&u?-G159|5%9X# zpcdwKwry&G7Wo2WP!8a{p8JvQ9mJ9T3ZRak+=iuAhuD_o1*-13%0szCryg)r ztlYPu_XQnMHqf7CQ$^GPoTUNdON-5mGyYA~UJ5Eo?Rehg<7PqiK02-5DNvAtIwTlv zTjD0s9qp9H#2JnF2s*pzaksmON}C+RYxiU*@M9Ro0ruv(T~#Rs71V`SYmJd)Vr|ks z^QCA<384KAOYqZ?OTL{q4Ceu-0kUa+nCCDKVXXaeHpd8pasvQzg{!bkN{u4^UkSBk zrvA-kICy%|Y|4UVh4ImNWSu@knb%TE+^y>-AR>5 zEjBn|b5z%8T;ud`!!kRNw5Ztq^fFQY2_Mml%A4S^p2UqMdOiN_na>EOr(?(qK4B`0sjk3IkSe1wCNg6cbAaCg2m9+jTP41o~3!0|D=aL@g3dl z<0||Ey`AvwmPGqaK&#!Xz~|T8P(ap$z%3TrF?}RYRNpVb+kUH4f}tX=@uW>MqP}mo zM6CE-BauV_*N8})^z1+H?hVb$&|2na^J;S!>UQk?-fo@GqaZoocm0BCiR^t*kN$V% z8y&r1$0ReIlufYsXr?(iA)ZtQ5<2Gu zMi~-{K&WuF29?DAHo0dmg}w3x26sqo=?`^3D1`|$gfCKe{LeuAV#e*F%PkjEVZa9J z<#D5rF%=7mpaSC~6qL%Wz>WhvRW;_RR4&8GOS80SHal8lfLFrTIbFi4P5Hw`Jxfwo z^%inBMmsnEh*Yg<|E8fYen4koUO>@&@l}BKOOQTW11lEgUNzW_JPpFjobyUV%L z#T{m3_$tGa6b+|qZT{%*RqaZqyyf$7NlI<8`wysDzn65Qsp=LFSeTPhRl zfjqAy68Q{$SPw8nGDFKCHVY+1HiO!1P-J@7cuK=UXa!cczbypVGXFD|Y9VqVyRHQ_ ze^b^xGFH6RG(p9sqo1*%nZ;YGsuRonRIz*168Rf>BD$87PLu6G2TiP^O4XA7Z#P=q zvj8o<5DI9`d#+`oNOku=7v!X}rl?JuvP}IJb441#Qyo_olBwhJH!tS$un2(pKnaH8 zAyOHW8$LqQ4Ya{PGRWxHllc=HM-&Pds;Hxgzcyldfw`lN1~3_Bs33zbT8v*sd0h6og}3Z!S~FsD7((ftexqs)?{2y9U&mCmNq*{fUS3+2x2 zru;X`QOE z&}uU_5{A=~mw+u7lfxJU0x((Kn(p}Ev71A>@*o_uwy@zdk$f*yNHbI{0tqtR()H*mJ1ynrhBa;ID0lRu~PAull9rmNQ(s_04M z-#^%zewR^FcC5OtAb?G`y~J>%>|mrQK^og-A8b-sgULVn7Vl*jk>>0~UU*=LsBwxuY-q?0$pW@Su|RjA{i}uWcTJ+>zqE9OVc@yx zbx4viwg~u)DL}4KJ@(NSLcAZh4p9@dAfoIpN2IhXwPO-}v@YP{bd*^U-=d;S>!2aR$!q<|b!Wqbt%BvBSb-BV6tvUyQG>&pw zB+2Oe-yC^mC3UHST;WBu|K?u9*dFDMdY0fIQ=_&Z7iY{y;m?LzgqWXmq-l@#ED^BZ zqru3wCD{0lNr$Y=JMPldWy1=+!m4#}a2t@f2=?0ZRkQE*)-Q(44_og!7(lQj^x)<$tGY@WxSlg)2v|NJZ)qs$ssMmBfeLY5G|0q*(`jeQlKo;> zCXT48KL;PDEANKQ7phw6Ifv2?ewK6L?b`n>)Exxcr{1plbM(8{5*^S(HcfnqdNa=z zS51J>=7qIAIwJTy1Iwo9R1?L$2#tv7Vf6)TQ6@2&MgBSK87VlupQnysiuFi~Jc4x+ z#b&1$tgD`wxXz9K&%56#(j()(_4Bj++=@~%;Qq1#pU5Y=v%+&0WJqF&(|V5o6kx1mu+wix?=ly->K84?UVr*0&fuzYW@suL2^k6bdQ*~aM~4Zaeyb_l!<#U-yd{>ZTV$x?C&9)Ow>+>1^@9+w zoEz=jj3lbL9$iUiVn$4l=F=XQl$la^t7q>Bc|&?b}9m~<@4Dkv}gPvbQ0^2%W`tJY@|7gdoU`P zHfw0w#@biZYb!ALAkfG^iFJf++Hi65nL@fYcf@ojJ56*jE^Igw@ySgUDMiReb@mg$_b!9kiHJyNRPx| zQ=Z6qgT`d@(`QdfXQAjDaiFnJt@UV`&}1l2prnVIUS}ibLR@}edq3H&tOzI*#~3po zcV5?Fxs}%o+*5C-SiAO%pO(mqx>llqj}iYKySfb+Bqy*)8vVL4CJYqveXcPth}gcc695VCqd_mj6IzXg2f5xdd}| z&4+q=)?=zbGf`79r&`p;OYHm5SeP0O{NV?p zMA{%W2M-UV9jamb zdT6#+38Bc1@l9^ZT5+;)`0^)_{G*2xpa-=0ULkj^lF0{_A5*}OPt@f56D655Anq#n zmvX6)>u6jb>dlKV3oJGQa*Ar$y@<0XK1&xc`cOnM0**^@+ffgW{WLDfhnAj}I%_?R z5$4qD2}&e_hN)^~Pi{W#r@5%qS{%q-aCR4aK6URTGJW5geSeE&D-x`W9aOb2XMo*l zjlZ3aO7+3JVoIDdLA9%tnv$#->5ruuVn5|T;ihJtJlxlQt8YzGYpFqiKV>tBwR$USyaFNzEjFmP1ig&EUDzMPq*8;+ZEwJWTZ3 z3_G|d@I8Em+d!IlV~Bl<@(07#^ecM#JQ(%yFTtGzmz=G+`&lCFJP7YeUl9lymOXA% zr{$Ik<}U z4-`pXt>Cj@&xnOgo(5cxhoWmP7;T@tFSo(lX6pxpabwd=EzHN2e!Di!L``i^SrcweJKh!s zdnxe}Pe< z1NzI?;ew1xFsYEqbZO*En20Czv*Y{y@8hY@iYbr5|FzqBy&a>Ofm^E#1sPYxTqg-b z(iM6{<5`g4c2U@I#^#Q&HTXboYzTf{g-2!B4{%BH4#ZxPPNi^cWEWZU=g zvORrQcsv)Z1)-F5w&^o&I!)y8o}*^=QUQ8}bw_pXR{%KTRQXga`HWxkC5UOZ?Jlrk z=GYP_si8KqG2Q2V<2yUp;*vem6AY9+=zguvwyFNkxAWf9El|S`um<^q8Yl;u`#k++ z_)_Do)#qtm@HMa7)U5E_eHai|HO&dAihR}5_!9wfJgV$6pU{a#B+w`{=SkkglU6p} zbUFpboOcGEDLk_rce+p&go&l0AQTx<)hiV4f#-*dcocqRSU>3#N~lh1?H?)8A78jw zTH<%cA4u&_#zW5{l=f_$9RG#wau!=R`3`x0r zA9vT*&86-)>1R;9$9w;&2D-qRYijEFupOn-NG&K4H`tvj094}CH(NMLrfF9Q1Tq-eH>1X}6~`2N`CVQcd# zuk{pvvSMD90y53DGbcbJnNlH>&ezcG!W{7tf8v31m%`6)JWs&! zQM}Zu5AX${`-ZeQ#cgpB+FR8a?rSeEm)g0cy3GwX77)?w1 z*J!vDj}NA*2mW|3NZ@@m|KGnbUEqGrd|ACcq8@*IF3Z>8_1wWv_Q%I26hJQ=~wk zo{xBxtZ!Z&)FSp~s#(q^5B%bRZVLNdP0hK3J?#?ryP4pN{f=oYU;Oe#yW)mlRHYAH zXic69-6>B7$ss=Xp&Q*kG^@^8bQ?sZJ7A_V%stm-2W<6JaXnws(+Q^U0h^E$<^(mE z?`7=p$1)bB2M1uMJctbSD?QE|Sce4F(u4G1HT7T^5(RLtTsDlS$ObFide?W>TSi-2 zuIST*@IkGqoI(ZJ5tjGzaq42odZ2GX$pk$+;v!eWTDxsz9WaS5&^e%rc%S*YtCv?E zJ!EZZ2t#B=Vnv|=O?9uzbb=1gO#a5R=kp^${)c`MsvLlD|B%7&nPEs!Z;2)+J9kk z?cB-|$1%$%{BAlMUCg!~iNrs04vh%s{~&8ZTxO4Lu9`j{m}>T}0q)3IDZgXr(TtTaou);*JI~GFL(s$h)pnA?0doY$YOgvlB${W!>HBw7ElII zf#UvnQpsa_TbZ|b8*aABmQP90avoInuDJQPB$zr2OL7{jsl#2VT_DYyb{&?9QbpY_HzLl7Ijm*DLFg-t4P$>H)^=c6fYcPBBU`cN`F#y`Uk~|pgLX3I5NA= z^Q*&nZLq&Hcsd#Wm5Ao=x@(7`1x^F%!!FoO7O=mqenVw8eO;A~L$uX&&Z6!gqw|$R^DkRJ zz?9TVUvm0eg$ar+N7ubUKVP*M{DLbQ@(`-!1jd`1KUdT~y`@HTygpGEyR_E?Y z&uWB+(F~7#kRcHju{-6Vs*&auUcmhuXFA(TvU(ZB) zrzY0$H7}yTJkmjANM4}Q&nYg+J;Tryp|MGf-G#5XPdiUD z|8dU(MLXWyCPQJTS&5qdn}5>i&1_K?v%dA!P=wiF71jz{$jjCsy@FEBR9=T^ciMdW zrSeOk^>V36`iFX{@sepayg4}s4@X2L&-_D9f6KQ==%+g`=1)_o$y25cx_w1BY9=5B zSWtv2Kn|jn9rDNh7(|z6b++H%*=Ka$9r{b#MEq|iGaN6K zjYn_F%k#&$ULCoi=eR6cm86rocfYOb>QNPpm!Sbm!B(jS=;&1pmYRa-R3e~>C^bAC z?4^-4v)!xAEA;3q-q!n|+#7y?g<$-Ew2yErbwge-lSgi@l^8Q8QZT#7 zvnC3RYG^~^{rRL*!}=}Rj6rGy>mk!sTw<y_0u+@Am=eMHjM+i{_jdKw zuZmn`ysA6FQ-(hXhonrI3Hcx<6R7gY^Bm0xs1;U)bvu()o>ZY5it~7aFY60b6KxIX z5T)*$nWl+f@N3R2Fy6{n{@31R)HZA-ltHEdU8RwvD!RyvB=|s-M1Wy?!`8GN&CvFn zb@A7+v2f|PZRDFK88h8jMu#MZ!)cMs(?Gz%4K-3Humz;m_PN!07niL4x|$p#7o)%A)H2Cu{8P)HLwZGo4KWvk2hoX zIVb-b{fGNL7=c|gJ3&v#LO?n=+$I@E;4kRN_4azBX)93c=&>$up_%Jdc@V$JM@N%d z#n?%w7L>-Bk37??<@OSSyEW&Ibt+S<^(I4u;1!J;P*Lg$#jQ~A9jNYxCyW3!X}s6i zDB3+WO&U7LK~1Na(K-!1G!nKN;`k!M2N_h*K0;_2oz~mna>9fPcRQw%#GDVL4hLrz zk&~vUHf3*pkl#yz)dvW{o%(hE1a|lwSni{nt=sd=8Ehk=+dJ!fP{J}_Z2k)?m z?g6W$awMiI)oFz*b*hV!IX0u-JFublzT8sss{>@Iqt+!#UGzLcc}juROF=UO8UpK0 zV6)d4+%M4^mfQ^%wvf#;iNTh)HbKgX^ieIr_OIUNkN)mUJ!D1|E9Y4bC&r2i{s3WM z*G$Uczh&Ql36^ENg_hH}Jn#{oMoR&yQtL<*Dg{Z5&v24(B+dnxuy=NU;IYj1<*}b` z9$W}3$mNEoa{MhY=|n{&Vy3}o!~IjH##AloADGar1d#+6wS{adNUFRrWxX<0y7=5$ zyr{(Ym#X|(x!?&(hY8F9=SoZTETIF|4A$Be7$5Qu1&EnAkyq-65yc;4o2Y-{Hxi@m zu86;-{(vTVpxT28UC9Y2a|AX5t5rdT>H2%! zD9wpX6Mrv>#*r?ZP=AK2S~B5kgh-u04rNm>J~(i(FXI7#5)l8t^g$jmyS*kN?(x*x@FB4UB zCg)L?!(>lvNP0)`ymY|wlVwdibfb{q`G`0XG7?|Hq{3O}>CZigmk`juA3wwha@7ksMzl{&`THm{g;v}DsTUY)X zt!gW`$CNX!j%Bi`o#f`m%Z}(!e}0RKYUA#py(#>BX&uG}xf^az8ptocM;+`m!?6^0$~uo7jStBQ(PR8k zrQ*oVUs*n-y}Spl zppAOQp4eVb;B>~@@MchFCCgfPH6@9tUXofxrpoxq0^7#}xAb0>b~S>6lJef8@Kr@& zY-Z{N}p)1fkVw{u}iZ8b>=%gRj|fy&vF5|@sBVMw{oZ9dop2pazisW*kzhT(b@)8X;g--kIwyy!Xj%>o$6?k zLw9&dQ?%HOy_busXNMIfhi-wI{xW(B3Qq zEm0`zKnvAOH4RhrUuyH5?(np`Kh?IH&^q6qNXt=mm(P{%yjutxjMm?`bX)CyaYc6& zOust`rr7)be)sk{eWsU%A*v+E25cjlzA<>}n-3z1glANinMq_QfdEe0DvkQ5orokZ z2109~8G9-oeRX|t;PO!`7E!KiOpUaRsH-$r%Sfpjec3vF7CWslnpGvzGPVYF=vJ$a zy7KX!t&^8c{se}NH(I);Mmulu@Af>-xSHTZ;7c6Ai1Y!C+Y50S&oLmt4-v03kdm+1 zDK{-&7P_8-oKDj-h0(4f$( zy33HPFUw`oXks9&F^e?6FERKo2Qk=i8~xuE`e(K<6=!k!7^Kb3p|akTEopRm<$?rBXus;+iH6mXz* zVpXhWRVa$0s?;(KoJ5h402BiVPT;W+YE&iJE9PHX&}4 ztc;NmL9kLI8e0UaliHSuhAc?k=|Ik)w%7EJDUV2bHQe}(Eo1Qovq7oxP5^PIvZ?)h zd_IVQ^9+f1$PYWpRfMjv^{L+R()u=FVpNe54{{#(sEMFNxri;9nCCWwKdeMewTP*} zG(hSMo`)&FC}R9oui?8=!H~*{WJIOnfcs;p7~+;-dLHquUf%f{ad$;1cI#-D744sf zlrAL(=Q+xREm6yyo{pgP8^W4C*xpyB<$;Ng=3I17l5w9pct#^4i$W}LE#@yoo5Ob$Is!s58M7S;8Uyyk65IKykaD=vMJoexl#IzMFU z$JV`MOvz^Kaqhzs!=#9^mh{broY3}4&)!V@I~7t1ufiK66RXA@o^gkM0{3j*^~crQ zrDyN9?^L6eP|1Pd3h5GCRZA3?$W+Xcm@sL3>O0?4X@Ua3gGGt3@At2X1Bd@B3iuW$wXO`RFluZ(;r!>kUru z<}LT7qyrs^!+C=bs;5?g~`tv*Exkxs+VsHe7-5OlOHX_0{?3e#>WgZ+v&% zAH-PZT`2jNkM80=?;-6~V*?DD$B~{PkU))-kU2`R-LMycNwqC>Eiz^otAnNF#TR$M z1EpCqojEn|j!Mv;l|p9j?O3S08n<0vCgLhy80zRE^gJ_ME>VGy$<4?=kCgYX8Slq~ zX#ZEa_we1*^ROA$ZxijmQ!~uJh0=W?!;qV%@Nd-zg}6sYRHucxi|6S~z=&Q)!x}Um zeP$RV$(1ul{YEps=KN($#DC(1Zmf==`Ht0X3Kh-gq z0MYDGu+%7t_c*pXI>UV#P{~eeK3c(T#{_<`GGsEU#P&f-@2VZ`1{=kJnd88}{n16GOYJ;P zH=}c_0e{LbDE)~|`3ej6Nh#5bM`>#v=tW5R|GgSQ-Z#JtCswrYl{Is|nxRQoVI?OJ zC>0=!*wdSJtX&`dzvj-lq0;t?_u1IluE{ng+um7|ZP!kdZM!DdWY;v+WKK=CZR70c zcV58x_q>5y*LAOTt+l?NwPIpO98}T*Ry71fbcef}K1lzNKvkjWOflC7Pci98U$_dKtFRM+BS?Q=T^_p8Z=(acaG+#I@qn% z^j;-~caKH>J@FT=yW(X+Xcs||~1ag19YqG1z`#4INpa!{W6Y74{ey*-H?E?6 zLxeMGCgE>#qnM61nlzNzlW6irrGR_4gpw1n!!WaVyIo74A* z3@^Ve%KdD=gH(yl_$4wf5}XJKI)meSWzn;{40~wNpE;Y)j-LeFB1@V#VriTi__i^% zUfR^qihFG*KkU*k*6*;kN}X2#@^(&UW@~=(75u$9i3nhX>F3oPMvLE%Anr<0^_32$ zH#@1#^NYN1+%oz}8XUmhns?)9ghN(Xq3Yt5V2b_Jd~)?t)<95H<0gGN`4pz!b`30Mp^Y(v7Gr zO_&_^4Pyh=-`;sdI|=$e?(LtdO3(c8#DsU((xgh@MhDEMQBBYF-|j+Wu0 zvMFW&p@vB{lVnR!7dE7HPvz?4|SPS>dPr{ zfDoZ;Gr^$W_Hb3;ShE=!rL@wAd$U%0(xhAmv=m%ElO<|;mG@?(QKXH3M19er5&>&W z^LeNziUJNcVA{}9I=T18)Xehl@x%Bs2Vm{FZ$qG4c zPfbA0B1VrtRksYN{x}ur+Sb>?zR8Q<2?hPc=kez=%eE|!pJV*A__A*7afe*$`aU54 z@ZhTZxhFXp;5tYFuH-9X^ZDr|Ho%Rt%319TW%Clwb!&tlL_|Fkcn0QX85Y$L;?>D$ zgsCZ}hyE$#2wA)@n>qtdQJA;+;T}X8**df)8NQsHpVY}}#9i-~ z2UcPCk3=FL{6`+aMM3e!Wg!F47edh#Akp%x;R48`RC*d11xo3_|3S1Y23CCRCY49F z^6U2iaHkE+*?4OMpYS^d%iovTe&pYQd9ef)cI1*DzaSAkZpz++lfO|vKLzwWC%mes z0rpfUo8rPqG@3-Gk`BIT64j6HI@fddjl`8wyJ}|HDy%!7D|h)&(XfzJmE)UAd^?Bgbc>W+w3jo4qdFNwKs3l6synUMC(;}pq5w^zn9r#HA zSa3zlZr(tjGvo2+Iw9KG=vnJ3s7z(cW8e(%%>pRV=88HEq zRa1jwl^>Y8r(Ye6`s9-HwA+CYUnTb>Te^jPa`jAHA8gOb@{NFGaAv05ORl!bx-mZ?=~mudysN zRkHx6%xdYHD+)DWMFTTZU(tnBiQ5yqO3A$XlFF$M(U_;FJRvjrOd{W3JIE_r?A9}p zU50CDmCY)k-js7&0)Z!F%et;{360OMBWsl<2`Ig%i1gK?g-DcPUF_Ht&m zl^fUf9t9^EbU9`9L<{!$SXDO#obxsz>14Ug_|I&zM{?IXe*}Wm-~&vX&$#iruh+jH zWEfWg`?xgrKVu1ge6v_IiG*CulWuK~es*f5cDL$(qZE}P?uuUE#obJ>o45R}9gRFc zc*0b_y$*84DB?s`%c=QbOAl-SACj4C*2lD%Zq>@#DONFSW@mgZ3AjcJ56SsFT5#Ub z(*+esFcd0ecG<~yQ2R;}5y?S_E#EY%K462;FpKo?bV~pBxvf^nUx3TCsD2(Q5O>(-|ZSO`UWMoZrZu=2970A zBO_B&{>Evm^FkFj`jGYfi?Uzv$GDY*sG>XWZ9HLQ5Lg@uUBv2D4AP<<{z!EA?+y^B z7RNk+1^WN8kC=MMoqrU%7tYw7VEKJ*H4!b!ir*-7<;2wv)(V%w!IVF(8Wy>Je%LTI zb{VN&3^3}>xbUErZZ)xYR0-cy{)xgr5F^&?8^GKRyH&kg`RcxW?S%HC`bg;Vhf7C* zM}a80hdEAR6U9x0ufB*Mi70AJXoV3mF8o{dHszIOn2dS@WANYDQHeM)Rc}lNwW1S+ zsK_U4k=98fdiHr!*mxs5%W4q!=JTR0sS#d(HVRP>!Z&llNYat5hSFhR@YViIc23#B zxg6uNR`G8WQnOp(d_{boMo9ZnR@^{HeGSu~(z=XK?{;p0b7Eozr5cXRa4X5ZQ8|J| zmcNs(I5_?lsjqVTr@w7GN&A^l#VGF!BI*ASBRe)C2jl4ykk6!gRv%Vb?_-aVcl^{6 z1(46s6aHSjL=wQ&a32aZb$my=MgNMgs&v(n-)o7Rtwj^2^T7$DxR*iW2Pck{rmx8{p(Nvcv>F z2swds^b%S-JtQZ;?ToYA#ydWIM7Xc%&u!yO+vBvGzCzX$ou| zZv$9d-&`sr-*%9x+Bc5&rqNzYvAu@>(@n0b;Pn$Eoe4w?YsvCerV1+*+E7*jT-1oM z@roQut(cH#T$b4A!v5GOmXQAv*V%yC`UYoC9umWY!iz{*BI#Q^;cY{&L4tmRheB+A za0sCYesM3(Syd+yZJHY=39IW*?DP!~Nii+LC^9)=>*$QsS5@N`fySYlg53c>HYyW- zRK5!(7$0vm$zLbA1jMg&EC$n&cRP1UwPnUj$Q;3N3(2$s$$%b?)zKBJ^wqw+>W@Ci zZV|Au`5f%}-uDpBbIv(G@0eDa41_?B@AU-GJJ?V9qYU~_;XN8eV@2LX$o-%Jy_xeZA{57;9Gnp*%I z+lF+_1TEmI$a3pvL2+_)ZfINwGa)irmLx5u%$HC~XQ`JkvkTj<4#CABRg5E)(D)uM z1A#PHAg&nq%6%_r@xQ0Z@=O$3lkkP12MbIc`* ziyzr#k&H_V06wMM03278#y6i&pH;WiBzPUrbf!)VgFoqw8XKC|O(NaeUSWb$8(1s! zpk{p{-iWLF8IcX?qH#@lhVL64FqHl{z@{u(AS>%=>A#x>9+YSCG6S-&WPZS{X)7}YlB4+Z^q#R(b4}MB(*yn zm%qZIlGTj9v%?AS74QDe)0GtiAq!W`BKk~)ijWi+KA~v}M~#PgO^61t+hRHhhv0@M zwmmrM>za9=D=WL`Ync5VD%`s+58l_|P|F-QQ%Z1hhP9`;F=CSMT1*^lzDV`BZDS#_ zczcb1yptP>tBVaeg}thSDWUTSb|Nt?Nn7|I$JafNrDVro#Rn4aJ(-4l@f7GW39FHm3%q{#1-?q>7%)qjW3oUIsTjrRz4`X zTf7stQ?XhL2I*^mAdWx*(j_}_>$1adRMnjEng2{71}3KI*uf>BEs4xQo0b=zbr$QE zhppLyO7@rd0f`*m9FeL(X9b8qLG+K=>`@(?O8;HykV*W(?`Voy&g+<4{gDIX@c;*` z|7P=j>3XLrh~_ghnTutvhYWB>lf=mV-XPockeRuta9m76Kt4p%V9O+(98}&}%*ac) z45|1jD-qMt&2Ag>c>xrJAxpApX+!@^>sns#P$*;#kQwt1B7C}U&$#> zgO;lrNIhJj7zGekX7hhVjn?F04@3D-#>QA5_moZPgS?xbFR{HjrxZ2~z$@X()Q_RK z-vYypmmOXx>!%d~w%4)n>Nq?GiBQ#CGln`L;Ixxk+I>m^n!6stIPhA=K~#;wEa5}AAYn+!i&iKv*>g*ZI)WCY7+rc8 z(10#WtC*TW*3#BkL{TCaG6?I}%Z-)&&rG_McaqI$HCPQlQJj>31`u9-_Mk+|hp#Cu ziFE%cMz&K24r^+qX@bK|E5^ml<3erN-5ZX+Mxvx{T(O*PT645VncJh3$QXw(lC_=Q zHT>{3IgwrNY1URa9baFQZ?Jy|Gu(X2UxB=nK@pt1<$(Ncc^O&=MHctR64U8{S!RCb zJXX*^r5FMnwE%V^!v(scEYysU%`eHD=ojh|`&X~&$ZI?bu z$%)FS%C6Su?E$YlyoyYvSJn+s_vETk-0hH`6hMU!i2WaxOSq6qM~438@9zo#mt(Le zCRlyW6k-EuV*!&(4Bb$V020cznf7iuv7s>4!zvbp$%Cu0VXjjX3^h~L87QSfQG_k_ ztw~-r*fDF}55pACq?W;wG`O%alA$b^_iPeQ-Rk_>xor#lxZ_XfWh~1wQs`mt;e5ptLZM1pF<=UiSQ(4RJX7 z4R?yn!jcmuK?2G7M0O1Ew3yD7#$kAAVX8P2i2k>qi!v-aT9v(@xse|1d+sw$%!UXj z?alPqyI3yN;m4qDsiI>kkSBbhw}y8Mm)c{518{2(Q&~T(T*i- zI&qC6g{&}G**X8Y8$K*erWh6v2_4lmZaM)>U+xt(pv%l1v6z}isV&D+!WlR&Be=GA zqTdCj&^nwk_B#uK7venzlQA#lowhOAjHF*QRy(q?4UtH+hbtB7*~Wm3H*+ z@+^$Y4U&Fe^55W!B<~OnqtNv%kp1qg3BqW#-*3l+;OYMJeY8>#4=7QVbofupU48i_ za0h1-@lHYq%v?hL$6bE0GyVMOQ#lkzOe=ksc%y71exo3Js!%cYCN$Cmb&BWh4!w1NUv`!Xvat)d*7B7;aPjA^-;1oN$c zROQD>z8hseYH@Xsz{L{}5jJIdK(j9M-wWO-X`ZA^=w(+rXzvz0!C6H^dp05j^OjsB za7RRH16AW`Hvb!-8X*zf(O$$SEQglIg;RcAhhT;#(ZB5NXoyOx*ao+n*5sj}MRJB# zDMVC0CeDhom7HgsW#a$LGhOIaBmcCd_A}?v?s7@yBo8;o2#44dMI|3~O94FCMeptt z2*C`~GKdQ2wu=CfE)T@2re-;>192Nj9pjZsygXSyoa6_~@7Zn_#5gdc>`wRg5j8 z6ST2we^>h_3}R3yR8Z0tchni@c+fbaUT<2)rwK#^ofh_FC9UyM>6zFxPGIP0k<2es zB8hFW(MIpDh0TOrBym#Z#;9ygZ{dh*(e3iuYBQ+OF@(?@lZ(e?+*ty^I_+Y7B!Sk| z9|=H?s_+!uCTxDkn^^~uB{-tlNXE?GaMPUdu;+c!@VQ(5cSB$MiJ^lOZ?H?!t+!z zRpy8V7kDvIOdYrwx5+2~@gy`-3mJddIn+$i={yfrdEcnYSof2M^>%WLcIZpJR& zd9WOS{&MAsr5U>rWlutRYmzI|+Dcu0m}9+ONq3d&Hmc^0K56E5W~rWO{)PNHQJEJw zx~X~fs`*vbRYJxpOdjj(iLHNuSr{9YlGknM%z$=HxGjCM-jZWGI3aE9I5m1;&Mawi zs9ahfg?+p+%`v|ZsPHyv6^;dJIFlj14TShpg0U7iGTLk zyU}s|n5gDNrtCx*;-Adk6n$L8OVZp7&8(StNWq*;KKbY9anm=UwMkiaD%Nt^9pUD} z+3n*t@bXpUmwau_VMM*a49%k1UuSb<@W0YUZUj{Z%sa?{=tV6o@-fzDUrCrOaTb}G zDFn)1v2^UIb6igdz3Q>o`vW5O5bgfh9h;R+@Dg|%Yfe|4_~#$a-Y+pN#(a?^$Ly@q zNiO^+^ptGX_^1!QIQZBfy0W#0NeJAKzF$Ii4m1MGPg3)mqBI_DawYTEZX3E~e%_|y zLITMrS?&5)xvt*=|HcPE$r?cpT6)N^^DEuet36=X4YqQQ}! zpwp>wL&(pOh|2E9d@_vFt{W0;%SV z9saE7-ul=;N%NK?<%HzRocTv+j-HOy()Ogv4Fb1?{jgb*yX|vr@TLfeE!KP`j(^(? z(COidwtvPf{7v52s19@=k;dDL0?V=&Q)Vbh(wwy<9+)4FePV?~zK{J%x!MOp{)P9i zNI!DUcd-??E_m9M|Gqi(OMB)~ zgrS5Dj-a66r)3Zs#1>|<69q~GmA@*yScoE9A z)UGe)5H>y{xFhQx=|s3vyx~dcCZyfcb)(M$Ze}3%pqgJD8b15ua_fo%Z>SCSeHwBg z+r#rhtxPyvyaOBe^535{_qVJ^S>d);eP-G;ooQhdAQ&?Z&$K}(75LcFi2zr`My9F^ zLc@2l<=HgMdF-H2!G7rV7O=rNKa$_N^Y^-b-7M?A4}c(xJnE)@>~~4!^??~GKB)~2 zcj)!Wk*!N|?;{mZ=&L_|@*xy{j;Z5%C;&HqJTsU}{=-UI-35A5tkrbde=11>d}szW z73*;9%P?M->Fm5N#f4JNQ4d+Db-!PjJ$BIytVZoFgOieuwBS?)EqsP_^5MsIAj!(V za8#b;x6u40qg=liw&=X_IBHai6o7W2YdpoKhVpdBSW+vI0LNJG3(``-MGVb_rlP{W z^Ul*Z4K2`K2gRYZ`b5YTv|50^rjzqV-0owR~PIoCv`$T5!6n4>I-uJQJK8TlqQwrj9Ya{x9xyJ z(iwa*JMX+-0No`gWWna#xc9WDIa;4fsL=a1rQRwz6JSZBm2~Ly2nd!p*DW}TS}VyA zYO@I~ukl}|H_Vr{`B0tg(e!-suoUJxl2}8QWFHE~oRZD&iJz*~)fL{9UgSeDejzAG zjmHguT*SAM`1`m;!;lT_@T=LpgCLQq>E$CQs5`>{92xDo3;+EHQWL!!j-?0TU|M9K zs2E*6g9rW&V=iqg4IKAPT)-f@9mf zE&RKHXgyi&%iEEvfUcMS0K4$CsK8OlIz?2ZD||35Tdne?pzsPT-w_c$duoehH`7n} zAFb|7E(nSEk6QnkaHS16)5{<0=vL4uBDII!@Yt${Z5ZCgPwVQ{uYGMA;$5X^4UdQn z=qMfm|6cFhK!VO>U_s^UNfDOwC@aJh&KU~}=fewQL2~mrP1pH zkdgKC>tzJmCvVUvD+G#swmIX^-tV6h>+`Fu@#&w_pJ(DvCst^4;#Dhm6WTrx$!ybX z?g+(|7ezg-@@%yeiot@qGykgT4Mo&%$qK5Trqz@5W9@mtPa`MW8+Nf{EPHyu?2aqL zWp_b)7*F>`aoA2a-vTG#bD}i*HKF8_^|i^B2)ABAv>iHy2C~jo-AP{H!e782tM`RP z9%~W*sSnzOKVf}>lKiv4a%xj=qA4`5aC*}QPNV?VTE%YeFCD(D1TrWX9HlOB`FV;n$G$B+liIu^Jr1eh)7AHKvB-v6OEO>7<$oLM|CeQ z|Ks(yNzS_k!%)vpc)P5y$3@roo!%-tA(7lb$B8Z=@pa<2GwPP$uTo(B7L(AD3hoV2 zS^(C_O4CS&Bjq%BY=bI-?+m1tmQplB*$3q{7ZYxeZ;;=0dU15c55i2wzX$F*?Y8;z zDai zB3Oy22$O95-QBUo?X}wUW1W-XnO^x+1bJr{DYi@9_Qb_YA>cBc)zj7WP59mb6?Zxv zmtMEx08{)nM}CxqFyMnb^@}jv&G>{Ps`IXim2H)u@Gx_~2ads%79AFe@wof88RO?z zi?mgX;+@{fvjvorn8(QT;C**D<=Z@K|58U$tw%y?6tYh;IoKGL#|a<7z5LfAGti8V zUPMC$gVnri?kJ@-WTC(uz2GZ}J%@nr)~T)3uczj!%sOR*qnk$Fsc)%I>YSxd3v3xYU% zm5Y*=Bj14=sjsCmO@~i&dOp6=`fYrP(5o^9*uaqbpz9t&A5ZlC8o%tt>r{}~VB22&-MtY8FzbS>H{iN9Tq1QWv?4Se#eRE;Izn<(&vlfp z55M+$j*O1D6>ORBQ8|}snBbv~?Eaj?>0X$)K?V{%|LYZL^Ib?we%C5K?B0DZ&7>%P z&p(1liuWzRjKh)DwCWN*=pCduT~y4jlCWvJ8}2h|M9#R$D@m z$XF^_;(n&@H>a5 zI0Y9pl)tGiJ6xNRO0#3A`lZ~-a?*7oF{Om-;$&%=XF#?M+=oW+lZYTHYyS#_Ox*}} zYZFUDWYd?}s$}MO>nf;=d~=}<2J69;KJ ze9iJmDm~j&J^YNPN`9wh{f}vIo6{-dZq`~ssOR}HB3K@3zX|ILjcRfZmwD~*DZhy}0&C5yPV zi9C54(UVaf8pCcA*!5El35)&_C#@2_S|dw)5q)|sdJQkQ>iO7f+j;%VDf+7>+!l>; z+`;_sEsr*$0CE{JIO^Q+h7K&%qB}?rHx5!IP<*H8wJ$iAD)F2hmg|43#yK!%TJC6W zWQMCm@Cxfqqv#Ga?G3P)cs|UXKO}G$FN%PSm8i7EabifWfN^5b(583DXp`;4D={!$lzf^n$l2garA{l6coQGGoxVSTD_pzH z+|g@+2Ipeu$AuCRaMGR2 zU`nHGDM&zV(*ZpswT}<1NQ^C?GbrC_esMy1)OKS=qhVyL+J-dSXmKhEppPCV{UzAX zde8lfzsCw=JV{px%IAOH7b|H0I@}RH(x*Q*V4r?kr09O4EaN!Yc2fp{;%5?_byB4V zjWb*-KWHJUP@0XIa;{^)$5CB5euuf?6)-D&nxIrMj{9!LS%lkbej;v46|oAjQnjL8 zwouq!N!^6C&y}TU4Vvs~N&bQS`-~wCrVny!9$LUSiI$u&@kiIJJtAOr(kIe;FAfyH z@ASc{T&A^i0mq?lWbhaLqXID%L zl@79%ygAAmCmM$?{g;l}t70d%WCquGx+bn8b=)nl%bU~Jh%wGjRD&CvpiU@tODk04 zpLec)#|RBIP}TPGA!93Myl*KWP}4np)4348+QT1>h@W8|saY}<_!i|D;JFW`l!WTm zsPgn93?J=QLYg(e9|-1SzBq*g-7zQ(%yhL3?n-jNm_LJ_1vPuY=`>;AEy~<(tMl|h zr+Zf_=k=wZIMu{fJMn2oYr9+nMAE za>t95f92>2*E?5m{*4c8G;rnrScBlvl`@?7E=dW5dlL_- zgCAfgBbvy6Ls<R6LzojULQh_9r0>FQ0>0e zFHHWD{V(-E%=CMvB=)Pd>iTn8Ca);PpE*ljoLl(o z0#)WaJT}4ziHme%t4~Y3PYtNx>{)vD2RQToe`kJ~nN$3SS^a-C& + +.. thumbnail-parent-div-open + +.. raw:: html + +
+ +.. only:: html + + .. image:: /tutorials/images/thumb/sphx_glr_template_tutorial_thumb.png + :alt: + + :ref:`sphx_glr_tutorials_template_tutorial.py` + +.. raw:: html + +
Template Tutorial
+
+ + +.. raw:: html + +
+ +.. only:: html + + .. image:: /tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png + :alt: + + :ref:`sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py` + +.. raw:: html + +
RL Fundamentals Using Forge Terminology
+
+ + +.. thumbnail-parent-div-close + +.. raw:: html + + + + +.. toctree:: + :hidden: + + /tutorials/template_tutorial + /tutorials/rl_fundamentals_forge_tutorial + diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json b/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json new file mode 100644 index 00000000..c1c95a0b --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json @@ -0,0 +1,215 @@ +{ + "Any": [ + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "_SpecialForm" + }, + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "Any" + } + ], + "Dict": [ + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "_SpecialGenericAlias" + }, + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "Dict" + } + ], + "MockResponse": [ + { + "is_class": true, + "is_explicit": false, + "module": "__main__", + "module_short": "__main__", + "name": "MockResponse" + } + ], + "Optional": [ + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "_SpecialForm" + }, + { + "is_class": false, + "is_explicit": false, + "module": "typing", + "module_short": "typing", + "name": "Optional" + } + ], + "asyncio.gather": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + }, + { + "is_class": false, + "is_explicit": false, + "module": "asyncio", + "module_short": "asyncio", + "name": "gather" + } + ], + "conceptual_rl_step": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_advantages_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_dataset_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_episode_from_response": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_policy_service": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_reference_model_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_replay_buffer_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_reward_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "create_trainer_actor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "demonstrate_production_scaling": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "example_experience": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "dict" + } + ], + "show_infrastructure_challenges": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "function" + } + ], + "torch.cat": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "builtin_function_or_method" + }, + { + "is_class": false, + "is_explicit": false, + "module": "torch", + "module_short": "torch", + "name": "cat" + } + ], + "torch.tensor": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "builtin_function_or_method" + }, + { + "is_class": false, + "is_explicit": false, + "module": "torch", + "module_short": "torch", + "name": "tensor" + } + ] +} \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb b/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb new file mode 100644 index 00000000..3981a531 --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb @@ -0,0 +1,158 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n# RL Fundamentals Using Forge Terminology\n\n**Author:** [Your Name](https://github.com/yourusername)\n\n.. grid:: 2\n\n .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn\n :class-card: card-prerequisites\n\n * Core RL components in Forge (Dataset, Policy, Reward Model, etc.)\n * How RL concepts map to distributed Forge services\n * Building scalable RL training loops with fault tolerance\n * Resource management and independent scaling patterns\n\n .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites\n :class-card: card-prerequisites\n\n * PyTorch v2.0.0+\n * GPU access recommended\n * Basic understanding of reinforcement learning\n * Familiarity with async/await in Python\n\nThis tutorial teaches RL fundamentals using Forge's exact terminology and architecture.\nWe'll start with a simple math tutoring example to understand how traditional RL concepts\nmap to Forge's distributed service model.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Core RL Components in Forge\n\nLet's start with a simple math tutoring example to understand RL concepts\nwith the exact names Forge uses. Think of it as teaching an AI student:\n\n- **Dataset**: Provides questions (like \"What is 2+2?\")\n- **Policy**: The AI student being trained (generates answers like \"The answer is 4\")\n- **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95)\n- **Reference Model**: Copy of original student (prevents drift from baseline)\n- **Replay Buffer**: Notebook that stores experiences (question + answer + score)\n- **Trainer**: The tutor that improves the student based on experiences\n\nHere's how these components interact in a conceptual RL step:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import asyncio\nfrom typing import Any, Dict, Optional\n\nimport torch\n\n\ndef conceptual_rl_step():\n \"\"\"\n Conceptual example showing the RL learning flow.\n See apps/grpo/main.py for actual GRPO implementation.\n \"\"\"\n # 1. Get a math problem\n question = \"What is 2+2?\" # dataset.sample()\n\n # 2. Student generates answer\n answer = \"The answer is 4\" # policy.generate(question)\n\n # 3. Teacher grades it\n score = 0.95 # reward_model.evaluate(question, answer)\n\n # 4. Compare to original student\n baseline = 0.85 # reference_model.compute_logprobs(question, answer)\n\n # 5. Store the experience\n experience = {\n \"question\": question,\n \"answer\": answer,\n \"score\": score,\n \"baseline\": baseline,\n }\n # replay_buffer.add(experience)\n\n # 6. When enough experiences collected, improve student\n # trainer.train_step(batch) # Student gets better!\n\n return experience\n\n\nexample_experience = conceptual_rl_step()\nprint(\"Example RL experience:\", example_experience)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## From Concepts to Forge Services\n\nHere's the key insight: **Each RL component becomes a Forge service**.\nThe toy example above maps directly to Forge's distributed architecture:\n\n* Dataset \u2192 DatasetActor\n* Policy \u2192 Policy\n* Reward Model \u2192 RewardActor\n* Reference Model \u2192 ReferenceModel\n* Replay Buffer \u2192 ReplayBuffer\n* Trainer \u2192 RLTrainer\n\nLet's see how the conceptual example translates to actual Forge service calls:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]:\n \"\"\"\n RL step using actual Forge service APIs.\n This shows the same logic as conceptual_rl_step but with real service calls.\n \"\"\"\n # 1. Get a math problem - Using actual DatasetActor API\n sample = await services[\"dataloader\"].sample.call_one()\n prompt, target = sample[\"request\"], sample[\"target\"]\n\n # 2. Student generates answer - Using actual Policy API\n responses = await services[\"policy\"].generate.route(prompt=prompt)\n answer = responses[0].text\n\n # 3. Teacher grades it - Using actual RewardActor API\n score = await services[\"reward_actor\"].evaluate_response.route(\n prompt=prompt, response=answer, target=target\n )\n\n # 4. Compare to baseline - Using actual ReferenceModel API\n # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs\n input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids])\n ref_logprobs = await services[\"ref_model\"].forward.route(\n input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True\n )\n\n # 5. Store experience - Using actual Episode structure from apps/grpo/main.py\n episode = create_episode_from_response(responses[0], score, ref_logprobs, step)\n await services[\"replay_buffer\"].add.call_one(episode)\n\n # 6. Improve student - Using actual trainer pattern\n batch = await services[\"replay_buffer\"].sample.call_one(curr_policy_version=step)\n if batch is not None:\n inputs, targets = batch # GRPO returns (inputs, targets) tuple\n loss = await services[\"trainer\"].train_step.call(inputs, targets)\n\n # 7. Policy synchronization - Using actual weight update pattern\n await services[\"trainer\"].push_weights.call(step + 1)\n await services[\"policy\"].update_weights.fanout(step + 1)\n\n return loss\n\n return None\n\n\ndef create_episode_from_response(response, score, ref_logprobs, step):\n \"\"\"Helper function to create episode from response data\"\"\"\n return {\n \"response\": response,\n \"score\": score,\n \"ref_logprobs\": ref_logprobs,\n \"step\": step,\n }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setting Up Forge Services\n\nHere's how to initialize the complete RL system with proper resource allocation.\nEach service can scale independently based on its computational needs:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "async def setup_forge_rl_system():\n \"\"\"\n Complete setup of Forge RL services with proper resource allocation.\n This example uses Qwen 3.1-1.7B model for demonstration.\n \"\"\"\n # Note: In actual Forge environment, imports would be:\n # from forge.actors.policy import Policy\n # from forge.actors.replay_buffer import ReplayBuffer\n # from forge.actors.reference_model import ReferenceModel\n # from forge.actors.trainer import RLTrainer\n # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages\n # from forge.data.rewards import MathReward, ThinkingReward\n\n model = \"Qwen/Qwen3-1.7B\"\n group_size = 1\n\n # Initialize all services with appropriate resource allocation\n services = await asyncio.gather(\n # Dataset actor (CPU intensive for I/O)\n create_dataset_actor(model),\n # Policy service (GPU for inference)\n create_policy_service(model, group_size),\n # Trainer actor (GPU for training)\n create_trainer_actor(model),\n # Replay buffer (CPU for memory management)\n create_replay_buffer_actor(),\n # Advantage computation (CPU)\n create_advantages_actor(),\n # Reference model (GPU for baseline)\n create_reference_model_actor(model),\n # Reward actor (CPU/small GPU for evaluation)\n create_reward_actor(),\n )\n\n service_names = [\n \"dataloader\",\n \"policy\",\n \"trainer\",\n \"replay_buffer\",\n \"compute_advantages\",\n \"ref_model\",\n \"reward_actor\",\n ]\n\n return dict(zip(service_names, services))\n\n\n# Service creation functions (would use actual Forge APIs)\nasync def create_dataset_actor(model):\n \"\"\"DatasetActor for loading training data\"\"\"\n return {\n \"name\": \"DatasetActor\",\n \"config\": {\n \"path\": \"openai/gsm8k\",\n \"revision\": \"main\",\n \"data_split\": \"train\",\n \"streaming\": True,\n \"model\": model,\n },\n \"resources\": \"CPU\",\n \"sample\": lambda: {\n \"call_one\": lambda: {\"request\": \"What is 2+2?\", \"target\": \"4\"}\n },\n }\n\n\nasync def create_policy_service(model, group_size):\n \"\"\"Policy service for text generation\"\"\"\n return {\n \"name\": \"Policy\",\n \"config\": {\n \"engine_config\": {\n \"model\": model,\n \"tensor_parallel_size\": 1,\n \"pipeline_parallel_size\": 1,\n \"enforce_eager\": False,\n },\n \"sampling_config\": {\n \"n\": group_size,\n \"max_tokens\": 16,\n \"temperature\": 1.0,\n \"top_p\": 1.0,\n },\n },\n \"resources\": \"GPU\",\n \"generate\": lambda: {\"route\": lambda prompt: [MockResponse()]},\n }\n\n\nasync def create_trainer_actor(model):\n \"\"\"RLTrainer for policy optimization\"\"\"\n return {\n \"name\": \"RLTrainer\",\n \"config\": {\n \"model\": {\n \"name\": \"qwen3\",\n \"flavor\": \"1.7B\",\n \"hf_assets_path\": f\"hf://{model}\",\n },\n \"optimizer\": {\"name\": \"AdamW\", \"lr\": 1e-5},\n \"training\": {\"local_batch_size\": 2, \"seq_len\": 2048},\n },\n \"resources\": \"GPU\",\n \"train_step\": lambda: {\"call\": lambda inputs, targets: 0.5},\n }\n\n\nasync def create_replay_buffer_actor():\n \"\"\"ReplayBuffer for experience storage\"\"\"\n return {\n \"name\": \"ReplayBuffer\",\n \"config\": {\"batch_size\": 2, \"max_policy_age\": 1, \"dp_size\": 1},\n \"resources\": \"CPU\",\n \"add\": lambda: {\"call_one\": lambda episode: None},\n \"sample\": lambda: {\"call_one\": lambda curr_policy_version: ([], [])},\n }\n\n\nasync def create_advantages_actor():\n \"\"\"ComputeAdvantages for advantage estimation\"\"\"\n return {\"name\": \"ComputeAdvantages\", \"resources\": \"CPU\"}\n\n\nasync def create_reference_model_actor(model):\n \"\"\"ReferenceModel for baseline computation\"\"\"\n return {\n \"name\": \"ReferenceModel\",\n \"config\": {\n \"model\": {\n \"name\": \"qwen3\",\n \"flavor\": \"1.7B\",\n \"hf_assets_path\": f\"hf://{model}\",\n },\n \"training\": {\"dtype\": \"bfloat16\"},\n },\n \"resources\": \"GPU\",\n \"forward\": lambda: {\n \"route\": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor(\n [0.1, 0.2]\n )\n },\n }\n\n\nasync def create_reward_actor():\n \"\"\"RewardActor for response evaluation\"\"\"\n return {\n \"name\": \"RewardActor\",\n \"config\": {\"reward_functions\": [\"MathReward\", \"ThinkingReward\"]},\n \"resources\": \"CPU\",\n \"evaluate_response\": lambda: {\"route\": lambda prompt, response, target: 0.95},\n }\n\n\nclass MockResponse:\n \"\"\"Mock response object for demonstration\"\"\"\n\n def __init__(self):\n self.text = \"The answer is 4\"\n self.prompt_ids = torch.tensor([1, 2, 3])\n self.token_ids = torch.tensor([4, 5, 6])\n\n\n# Demonstrate the setup\nprint(\"Setting up Forge RL system...\")\n# services = await setup_forge_rl_system() # Would work in async context\nprint(\"Forge services configured with independent scaling capabilities\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Why Forge Architecture Matters\n\nTraditional ML infrastructure fails for RL because each component has\ndifferent resource needs, scaling patterns, and failure modes:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def show_infrastructure_challenges():\n \"\"\"\n Demonstrate why traditional monolithic RL fails and how Forge solves it.\n \"\"\"\n print(\"=== Infrastructure Challenges ===\\n\")\n\n print(\"Problem 1: Different Resource Needs\")\n resource_requirements = {\n \"Policy (Student AI)\": {\n \"generates\": \"'The answer is 4'\",\n \"needs\": \"Large GPU memory\",\n \"scaling\": \"Multiple replicas for speed\",\n },\n \"Reward Model (Teacher)\": {\n \"scores\": \"answers: 0.95\",\n \"needs\": \"Moderate compute\",\n \"scaling\": \"CPU or small GPU\",\n },\n \"Trainer (Tutor)\": {\n \"improves\": \"student weights\",\n \"needs\": \"Massive GPU compute\",\n \"scaling\": \"Distributed training\",\n },\n \"Dataset (Question Bank)\": {\n \"provides\": \"'What is 2+2?'\",\n \"needs\": \"CPU intensive I/O\",\n \"scaling\": \"High memory bandwidth\",\n },\n }\n\n for component, reqs in resource_requirements.items():\n print(f\"{component}:\")\n for key, value in reqs.items():\n print(f\" {key}: {value}\")\n print()\n\n print(\"Problem 2: Coordination Complexity\")\n print(\"Unlike supervised learning with independent batches,\")\n print(\"RL requires complex coordination between components:\")\n print(\"- Policy waits idle while reward model works\")\n print(\"- Training waits for single episode (batch size = 1)\")\n print(\"- Everything stops if any component fails\")\n print()\n\n print(\"=== Forge Solutions ===\\n\")\n\n print(\"\u2705 Automatic Resource Management\")\n print(\"- Routing to least loaded replica\")\n print(\"- GPU memory management\")\n print(\"- Batch optimization\")\n print(\"- Failure recovery\")\n print(\"- Auto-scaling based on demand\")\n print()\n\n print(\"\u2705 Independent Scaling\")\n print(\"- Policy: num_replicas=8 for high inference demand\")\n print(\"- RewardActor: num_replicas=16 for parallel evaluation\")\n print(\"- Trainer: Multiple actors for distributed training\")\n print()\n\n print(\"\u2705 Fault Tolerance\")\n print(\"- Automatic routing to healthy replicas\")\n print(\"- Background replica respawn\")\n print(\"- Graceful degradation\")\n print(\"- System continues during component failures\")\n\n\nshow_infrastructure_challenges()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Production Scaling Example\n\nHere's how you would scale the system for production workloads:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def demonstrate_production_scaling():\n \"\"\"\n Show how Forge services scale independently for production.\n \"\"\"\n print(\"=== Production Scaling Configuration ===\\n\")\n\n scaling_config = {\n \"Policy Service\": {\n \"replicas\": 8,\n \"reason\": \"High inference demand from multiple training runs\",\n \"resources\": \"GPU-heavy instances\",\n },\n \"RewardActor Service\": {\n \"replicas\": 16,\n \"reason\": \"Parallel evaluation of many responses\",\n \"resources\": \"CPU/small GPU instances\",\n },\n \"Trainer Actor\": {\n \"replicas\": 4,\n \"reason\": \"Distributed training across multiple nodes\",\n \"resources\": \"Large GPU clusters\",\n },\n \"Dataset Actor\": {\n \"replicas\": 2,\n \"reason\": \"I/O intensive data loading\",\n \"resources\": \"High-bandwidth CPU instances\",\n },\n \"ReplayBuffer Actor\": {\n \"replicas\": 1,\n \"reason\": \"Centralized experience storage\",\n \"resources\": \"High-memory instances\",\n },\n }\n\n for service, config in scaling_config.items():\n print(f\"{service}:\")\n print(f\" Replicas: {config['replicas']}\")\n print(f\" Reason: {config['reason']}\")\n print(f\" Resources: {config['resources']}\")\n print()\n\n print(\"Key Benefits:\")\n print(\"- Each service scales based on its bottlenecks\")\n print(\"- Resource utilization is optimized\")\n print(\"- Costs are minimized (no idle GPUs)\")\n print(\"- System maintains performance under load\")\n\n\ndemonstrate_production_scaling()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Complete RL Training Loop\n\nHere's a complete example showing multiple RL training steps:\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "async def complete_rl_training_example(num_steps: int = 5):\n \"\"\"\n Complete RL training loop using Forge services.\n \"\"\"\n print(f\"=== Running {num_steps} RL Training Steps ===\\n\")\n\n # Setup services (mock for demonstration)\n services = {\n \"dataloader\": await create_dataset_actor(\"Qwen/Qwen3-1.7B\"),\n \"policy\": await create_policy_service(\"Qwen/Qwen3-1.7B\", 1),\n \"trainer\": await create_trainer_actor(\"Qwen/Qwen3-1.7B\"),\n \"replay_buffer\": await create_replay_buffer_actor(),\n \"ref_model\": await create_reference_model_actor(\"Qwen/Qwen3-1.7B\"),\n \"reward_actor\": await create_reward_actor(),\n }\n\n losses = []\n\n for step in range(num_steps):\n print(f\"Step {step + 1}:\")\n\n # Simulate the RL step (would use actual forge_rl_step in practice)\n sample = await services[\"dataloader\"][\"sample\"]()[\"call_one\"]()\n print(f\" Question: {sample['request']}\")\n print(f\" Target: {sample['target']}\")\n\n # Generate response\n responses = await services[\"policy\"][\"generate\"]()[\"route\"](sample[\"request\"])\n print(f\" Generated: {responses[0].text}\")\n\n # Get reward\n score = await services[\"reward_actor\"][\"evaluate_response\"]()[\"route\"](\n sample[\"request\"], responses[0].text, sample[\"target\"]\n )\n print(f\" Reward: {score}\")\n\n # Simulate training (every few steps when buffer has enough data)\n if step >= 2: # Start training after accumulating some experience\n loss = await services[\"trainer\"][\"train_step\"]()[\"call\"]([], [])\n losses.append(loss)\n print(f\" Training Loss: {loss:.4f}\")\n\n print()\n\n print(f\"Training completed! Average loss: {sum(losses)/len(losses):.4f}\")\n return losses\n\n\n# Run the example (would work in async context)\nprint(\"Complete RL training example:\")\nprint(\"(In real usage, run: await complete_rl_training_example(5))\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n\nThis tutorial demonstrated how RL fundamentals map to Forge's distributed\nservice architecture. Key takeaways:\n\n1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.)\n becomes an independent, scalable Forge service\n\n2. **Resource Optimization**: Services scale independently based on their\n computational needs (GPU for inference/training, CPU for data/rewards)\n\n3. **Fault Tolerance**: Individual service failures don't stop the entire\n training pipeline - Forge handles routing and recovery automatically\n\n4. **Simple Interface**: Complex distributed systems are hidden behind\n simple async function calls\n\nThe same RL logic that works conceptually scales to production workloads\nwithout infrastructure code - Forge handles distribution, scaling, and\nfault tolerance automatically.\n\n## Further Reading\n\n* [Forge Architecture Documentation](#)\n* [GRPO Implementation (apps/grpo/main.py)](#)\n* [Forge Service APIs](#)\n* [Production RL Scaling Guide](#)\n\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py new file mode 100644 index 00000000..3e77639a --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py @@ -0,0 +1,558 @@ +""" +RL Fundamentals Using Forge Terminology +======================================== + +**Author:** `Your Name `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + :class-card: card-prerequisites + + * Core RL components in Forge (Dataset, Policy, Reward Model, etc.) + * How RL concepts map to distributed Forge services + * Building scalable RL training loops with fault tolerance + * Resource management and independent scaling patterns + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + :class-card: card-prerequisites + + * PyTorch v2.0.0+ + * GPU access recommended + * Basic understanding of reinforcement learning + * Familiarity with async/await in Python + +This tutorial teaches RL fundamentals using Forge's exact terminology and architecture. +We'll start with a simple math tutoring example to understand how traditional RL concepts +map to Forge's distributed service model. + +""" + +######################################################################## +# Core RL Components in Forge +# ---------------------------- +# +# Let's start with a simple math tutoring example to understand RL concepts +# with the exact names Forge uses. Think of it as teaching an AI student: +# +# - **Dataset**: Provides questions (like "What is 2+2?") +# - **Policy**: The AI student being trained (generates answers like "The answer is 4") +# - **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95) +# - **Reference Model**: Copy of original student (prevents drift from baseline) +# - **Replay Buffer**: Notebook that stores experiences (question + answer + score) +# - **Trainer**: The tutor that improves the student based on experiences +# +# Here's how these components interact in a conceptual RL step: + +import asyncio +from typing import Any, Dict, Optional + +import torch + + +def conceptual_rl_step(): + """ + Conceptual example showing the RL learning flow. + See apps/grpo/main.py for actual GRPO implementation. + """ + # 1. Get a math problem + question = "What is 2+2?" # dataset.sample() + + # 2. Student generates answer + answer = "The answer is 4" # policy.generate(question) + + # 3. Teacher grades it + score = 0.95 # reward_model.evaluate(question, answer) + + # 4. Compare to original student + baseline = 0.85 # reference_model.compute_logprobs(question, answer) + + # 5. Store the experience + experience = { + "question": question, + "answer": answer, + "score": score, + "baseline": baseline, + } + # replay_buffer.add(experience) + + # 6. When enough experiences collected, improve student + # trainer.train_step(batch) # Student gets better! + + return experience + + +example_experience = conceptual_rl_step() +print("Example RL experience:", example_experience) + +######################################################################## +# From Concepts to Forge Services +# -------------------------------- +# +# Here's the key insight: **Each RL component becomes a Forge service**. +# The toy example above maps directly to Forge's distributed architecture: +# +# * Dataset → DatasetActor +# * Policy → Policy +# * Reward Model → RewardActor +# * Reference Model → ReferenceModel +# * Replay Buffer → ReplayBuffer +# * Trainer → RLTrainer +# +# Let's see how the conceptual example translates to actual Forge service calls: + + +async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]: + """ + RL step using actual Forge service APIs. + This shows the same logic as conceptual_rl_step but with real service calls. + """ + # 1. Get a math problem - Using actual DatasetActor API + sample = await services["dataloader"].sample.call_one() + prompt, target = sample["request"], sample["target"] + + # 2. Student generates answer - Using actual Policy API + responses = await services["policy"].generate.route(prompt=prompt) + answer = responses[0].text + + # 3. Teacher grades it - Using actual RewardActor API + score = await services["reward_actor"].evaluate_response.route( + prompt=prompt, response=answer, target=target + ) + + # 4. Compare to baseline - Using actual ReferenceModel API + # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs + input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids]) + ref_logprobs = await services["ref_model"].forward.route( + input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True + ) + + # 5. Store experience - Using actual Episode structure from apps/grpo/main.py + episode = create_episode_from_response(responses[0], score, ref_logprobs, step) + await services["replay_buffer"].add.call_one(episode) + + # 6. Improve student - Using actual trainer pattern + batch = await services["replay_buffer"].sample.call_one(curr_policy_version=step) + if batch is not None: + inputs, targets = batch # GRPO returns (inputs, targets) tuple + loss = await services["trainer"].train_step.call(inputs, targets) + + # 7. Policy synchronization - Using actual weight update pattern + await services["trainer"].push_weights.call(step + 1) + await services["policy"].update_weights.fanout(step + 1) + + return loss + + return None + + +def create_episode_from_response(response, score, ref_logprobs, step): + """Helper function to create episode from response data""" + return { + "response": response, + "score": score, + "ref_logprobs": ref_logprobs, + "step": step, + } + + +######################################################################## +# Setting Up Forge Services +# -------------------------- +# +# Here's how to initialize the complete RL system with proper resource allocation. +# Each service can scale independently based on its computational needs: + + +async def setup_forge_rl_system(): + """ + Complete setup of Forge RL services with proper resource allocation. + This example uses Qwen 3.1-1.7B model for demonstration. + """ + # Note: In actual Forge environment, imports would be: + # from forge.actors.policy import Policy + # from forge.actors.replay_buffer import ReplayBuffer + # from forge.actors.reference_model import ReferenceModel + # from forge.actors.trainer import RLTrainer + # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages + # from forge.data.rewards import MathReward, ThinkingReward + + model = "Qwen/Qwen3-1.7B" + group_size = 1 + + # Initialize all services with appropriate resource allocation + services = await asyncio.gather( + # Dataset actor (CPU intensive for I/O) + create_dataset_actor(model), + # Policy service (GPU for inference) + create_policy_service(model, group_size), + # Trainer actor (GPU for training) + create_trainer_actor(model), + # Replay buffer (CPU for memory management) + create_replay_buffer_actor(), + # Advantage computation (CPU) + create_advantages_actor(), + # Reference model (GPU for baseline) + create_reference_model_actor(model), + # Reward actor (CPU/small GPU for evaluation) + create_reward_actor(), + ) + + service_names = [ + "dataloader", + "policy", + "trainer", + "replay_buffer", + "compute_advantages", + "ref_model", + "reward_actor", + ] + + return dict(zip(service_names, services)) + + +# Service creation functions (would use actual Forge APIs) +async def create_dataset_actor(model): + """DatasetActor for loading training data""" + return { + "name": "DatasetActor", + "config": { + "path": "openai/gsm8k", + "revision": "main", + "data_split": "train", + "streaming": True, + "model": model, + }, + "resources": "CPU", + "sample": lambda: { + "call_one": lambda: {"request": "What is 2+2?", "target": "4"} + }, + } + + +async def create_policy_service(model, group_size): + """Policy service for text generation""" + return { + "name": "Policy", + "config": { + "engine_config": { + "model": model, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "enforce_eager": False, + }, + "sampling_config": { + "n": group_size, + "max_tokens": 16, + "temperature": 1.0, + "top_p": 1.0, + }, + }, + "resources": "GPU", + "generate": lambda: {"route": lambda prompt: [MockResponse()]}, + } + + +async def create_trainer_actor(model): + """RLTrainer for policy optimization""" + return { + "name": "RLTrainer", + "config": { + "model": { + "name": "qwen3", + "flavor": "1.7B", + "hf_assets_path": f"hf://{model}", + }, + "optimizer": {"name": "AdamW", "lr": 1e-5}, + "training": {"local_batch_size": 2, "seq_len": 2048}, + }, + "resources": "GPU", + "train_step": lambda: {"call": lambda inputs, targets: 0.5}, + } + + +async def create_replay_buffer_actor(): + """ReplayBuffer for experience storage""" + return { + "name": "ReplayBuffer", + "config": {"batch_size": 2, "max_policy_age": 1, "dp_size": 1}, + "resources": "CPU", + "add": lambda: {"call_one": lambda episode: None}, + "sample": lambda: {"call_one": lambda curr_policy_version: ([], [])}, + } + + +async def create_advantages_actor(): + """ComputeAdvantages for advantage estimation""" + return {"name": "ComputeAdvantages", "resources": "CPU"} + + +async def create_reference_model_actor(model): + """ReferenceModel for baseline computation""" + return { + "name": "ReferenceModel", + "config": { + "model": { + "name": "qwen3", + "flavor": "1.7B", + "hf_assets_path": f"hf://{model}", + }, + "training": {"dtype": "bfloat16"}, + }, + "resources": "GPU", + "forward": lambda: { + "route": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor( + [0.1, 0.2] + ) + }, + } + + +async def create_reward_actor(): + """RewardActor for response evaluation""" + return { + "name": "RewardActor", + "config": {"reward_functions": ["MathReward", "ThinkingReward"]}, + "resources": "CPU", + "evaluate_response": lambda: {"route": lambda prompt, response, target: 0.95}, + } + + +class MockResponse: + """Mock response object for demonstration""" + + def __init__(self): + self.text = "The answer is 4" + self.prompt_ids = torch.tensor([1, 2, 3]) + self.token_ids = torch.tensor([4, 5, 6]) + + +# Demonstrate the setup +print("Setting up Forge RL system...") +# services = await setup_forge_rl_system() # Would work in async context +print("Forge services configured with independent scaling capabilities") + +######################################################################## +# Why Forge Architecture Matters +# ------------------------------- +# +# Traditional ML infrastructure fails for RL because each component has +# different resource needs, scaling patterns, and failure modes: + + +def show_infrastructure_challenges(): + """ + Demonstrate why traditional monolithic RL fails and how Forge solves it. + """ + print("=== Infrastructure Challenges ===\n") + + print("Problem 1: Different Resource Needs") + resource_requirements = { + "Policy (Student AI)": { + "generates": "'The answer is 4'", + "needs": "Large GPU memory", + "scaling": "Multiple replicas for speed", + }, + "Reward Model (Teacher)": { + "scores": "answers: 0.95", + "needs": "Moderate compute", + "scaling": "CPU or small GPU", + }, + "Trainer (Tutor)": { + "improves": "student weights", + "needs": "Massive GPU compute", + "scaling": "Distributed training", + }, + "Dataset (Question Bank)": { + "provides": "'What is 2+2?'", + "needs": "CPU intensive I/O", + "scaling": "High memory bandwidth", + }, + } + + for component, reqs in resource_requirements.items(): + print(f"{component}:") + for key, value in reqs.items(): + print(f" {key}: {value}") + print() + + print("Problem 2: Coordination Complexity") + print("Unlike supervised learning with independent batches,") + print("RL requires complex coordination between components:") + print("- Policy waits idle while reward model works") + print("- Training waits for single episode (batch size = 1)") + print("- Everything stops if any component fails") + print() + + print("=== Forge Solutions ===\n") + + print("✅ Automatic Resource Management") + print("- Routing to least loaded replica") + print("- GPU memory management") + print("- Batch optimization") + print("- Failure recovery") + print("- Auto-scaling based on demand") + print() + + print("✅ Independent Scaling") + print("- Policy: num_replicas=8 for high inference demand") + print("- RewardActor: num_replicas=16 for parallel evaluation") + print("- Trainer: Multiple actors for distributed training") + print() + + print("✅ Fault Tolerance") + print("- Automatic routing to healthy replicas") + print("- Background replica respawn") + print("- Graceful degradation") + print("- System continues during component failures") + + +show_infrastructure_challenges() + +######################################################################## +# Production Scaling Example +# --------------------------- +# +# Here's how you would scale the system for production workloads: + + +def demonstrate_production_scaling(): + """ + Show how Forge services scale independently for production. + """ + print("=== Production Scaling Configuration ===\n") + + scaling_config = { + "Policy Service": { + "replicas": 8, + "reason": "High inference demand from multiple training runs", + "resources": "GPU-heavy instances", + }, + "RewardActor Service": { + "replicas": 16, + "reason": "Parallel evaluation of many responses", + "resources": "CPU/small GPU instances", + }, + "Trainer Actor": { + "replicas": 4, + "reason": "Distributed training across multiple nodes", + "resources": "Large GPU clusters", + }, + "Dataset Actor": { + "replicas": 2, + "reason": "I/O intensive data loading", + "resources": "High-bandwidth CPU instances", + }, + "ReplayBuffer Actor": { + "replicas": 1, + "reason": "Centralized experience storage", + "resources": "High-memory instances", + }, + } + + for service, config in scaling_config.items(): + print(f"{service}:") + print(f" Replicas: {config['replicas']}") + print(f" Reason: {config['reason']}") + print(f" Resources: {config['resources']}") + print() + + print("Key Benefits:") + print("- Each service scales based on its bottlenecks") + print("- Resource utilization is optimized") + print("- Costs are minimized (no idle GPUs)") + print("- System maintains performance under load") + + +demonstrate_production_scaling() + +######################################################################## +# Complete RL Training Loop +# -------------------------- +# +# Here's a complete example showing multiple RL training steps: + + +async def complete_rl_training_example(num_steps: int = 5): + """ + Complete RL training loop using Forge services. + """ + print(f"=== Running {num_steps} RL Training Steps ===\n") + + # Setup services (mock for demonstration) + services = { + "dataloader": await create_dataset_actor("Qwen/Qwen3-1.7B"), + "policy": await create_policy_service("Qwen/Qwen3-1.7B", 1), + "trainer": await create_trainer_actor("Qwen/Qwen3-1.7B"), + "replay_buffer": await create_replay_buffer_actor(), + "ref_model": await create_reference_model_actor("Qwen/Qwen3-1.7B"), + "reward_actor": await create_reward_actor(), + } + + losses = [] + + for step in range(num_steps): + print(f"Step {step + 1}:") + + # Simulate the RL step (would use actual forge_rl_step in practice) + sample = await services["dataloader"]["sample"]()["call_one"]() + print(f" Question: {sample['request']}") + print(f" Target: {sample['target']}") + + # Generate response + responses = await services["policy"]["generate"]()["route"](sample["request"]) + print(f" Generated: {responses[0].text}") + + # Get reward + score = await services["reward_actor"]["evaluate_response"]()["route"]( + sample["request"], responses[0].text, sample["target"] + ) + print(f" Reward: {score}") + + # Simulate training (every few steps when buffer has enough data) + if step >= 2: # Start training after accumulating some experience + loss = await services["trainer"]["train_step"]()["call"]([], []) + losses.append(loss) + print(f" Training Loss: {loss:.4f}") + + print() + + print(f"Training completed! Average loss: {sum(losses)/len(losses):.4f}") + return losses + + +# Run the example (would work in async context) +print("Complete RL training example:") +print("(In real usage, run: await complete_rl_training_example(5))") + +######################################################################## +# Conclusion +# ---------- +# +# This tutorial demonstrated how RL fundamentals map to Forge's distributed +# service architecture. Key takeaways: +# +# 1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.) +# becomes an independent, scalable Forge service +# +# 2. **Resource Optimization**: Services scale independently based on their +# computational needs (GPU for inference/training, CPU for data/rewards) +# +# 3. **Fault Tolerance**: Individual service failures don't stop the entire +# training pipeline - Forge handles routing and recovery automatically +# +# 4. **Simple Interface**: Complex distributed systems are hidden behind +# simple async function calls +# +# The same RL logic that works conceptually scales to production workloads +# without infrastructure code - Forge handles distribution, scaling, and +# fault tolerance automatically. +# +# Further Reading +# --------------- +# +# * `Forge Architecture Documentation <#>`_ +# * `GRPO Implementation (apps/grpo/main.py) <#>`_ +# * `Forge Service APIs <#>`_ +# * `Production RL Scaling Guide <#>`_ + diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 new file mode 100644 index 00000000..d93f4a3a --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 @@ -0,0 +1 @@ +d1dbe1df9283c920a3039b06b0d5ce4d \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst b/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst new file mode 100644 index 00000000..95ad7c0b --- /dev/null +++ b/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst @@ -0,0 +1,788 @@ + +.. DO NOT EDIT. +.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. +.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: +.. "tutorials/rl_fundamentals_forge_tutorial.py" +.. LINE NUMBERS ARE GIVEN BELOW. + +.. only:: html + + .. note:: + :class: sphx-glr-download-link-note + + :ref:`Go to the end ` + to download the full example code. + +.. rst-class:: sphx-glr-example-title + +.. _sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py: + + +RL Fundamentals Using Forge Terminology +======================================== + +**Author:** `Your Name `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + :class-card: card-prerequisites + + * Core RL components in Forge (Dataset, Policy, Reward Model, etc.) + * How RL concepts map to distributed Forge services + * Building scalable RL training loops with fault tolerance + * Resource management and independent scaling patterns + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + :class-card: card-prerequisites + + * PyTorch v2.0.0+ + * GPU access recommended + * Basic understanding of reinforcement learning + * Familiarity with async/await in Python + +This tutorial teaches RL fundamentals using Forge's exact terminology and architecture. +We'll start with a simple math tutoring example to understand how traditional RL concepts +map to Forge's distributed service model. + +.. GENERATED FROM PYTHON SOURCE LINES 32-46 + +Core RL Components in Forge +---------------------------- + +Let's start with a simple math tutoring example to understand RL concepts +with the exact names Forge uses. Think of it as teaching an AI student: + +- **Dataset**: Provides questions (like "What is 2+2?") +- **Policy**: The AI student being trained (generates answers like "The answer is 4") +- **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95) +- **Reference Model**: Copy of original student (prevents drift from baseline) +- **Replay Buffer**: Notebook that stores experiences (question + answer + score) +- **Trainer**: The tutor that improves the student based on experiences + +Here's how these components interact in a conceptual RL step: + +.. GENERATED FROM PYTHON SOURCE LINES 46-88 + +.. code-block:: Python + + + import asyncio + from typing import Any, Dict, Optional + + import torch + + + def conceptual_rl_step(): + """ + Conceptual example showing the RL learning flow. + See apps/grpo/main.py for actual GRPO implementation. + """ + # 1. Get a math problem + question = "What is 2+2?" # dataset.sample() + + # 2. Student generates answer + answer = "The answer is 4" # policy.generate(question) + + # 3. Teacher grades it + score = 0.95 # reward_model.evaluate(question, answer) + + # 4. Compare to original student + baseline = 0.85 # reference_model.compute_logprobs(question, answer) + + # 5. Store the experience + experience = { + "question": question, + "answer": answer, + "score": score, + "baseline": baseline, + } + # replay_buffer.add(experience) + + # 6. When enough experiences collected, improve student + # trainer.train_step(batch) # Student gets better! + + return experience + + + example_experience = conceptual_rl_step() + print("Example RL experience:", example_experience) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + Example RL experience: {'question': 'What is 2+2?', 'answer': 'The answer is 4', 'score': 0.95, 'baseline': 0.85} + + + + +.. GENERATED FROM PYTHON SOURCE LINES 89-103 + +From Concepts to Forge Services +-------------------------------- + +Here's the key insight: **Each RL component becomes a Forge service**. +The toy example above maps directly to Forge's distributed architecture: + +* Dataset → DatasetActor +* Policy → Policy +* Reward Model → RewardActor +* Reference Model → ReferenceModel +* Replay Buffer → ReplayBuffer +* Trainer → RLTrainer + +Let's see how the conceptual example translates to actual Forge service calls: + +.. GENERATED FROM PYTHON SOURCE LINES 103-159 + +.. code-block:: Python + + + + async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]: + """ + RL step using actual Forge service APIs. + This shows the same logic as conceptual_rl_step but with real service calls. + """ + # 1. Get a math problem - Using actual DatasetActor API + sample = await services["dataloader"].sample.call_one() + prompt, target = sample["request"], sample["target"] + + # 2. Student generates answer - Using actual Policy API + responses = await services["policy"].generate.route(prompt=prompt) + answer = responses[0].text + + # 3. Teacher grades it - Using actual RewardActor API + score = await services["reward_actor"].evaluate_response.route( + prompt=prompt, response=answer, target=target + ) + + # 4. Compare to baseline - Using actual ReferenceModel API + # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs + input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids]) + ref_logprobs = await services["ref_model"].forward.route( + input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True + ) + + # 5. Store experience - Using actual Episode structure from apps/grpo/main.py + episode = create_episode_from_response(responses[0], score, ref_logprobs, step) + await services["replay_buffer"].add.call_one(episode) + + # 6. Improve student - Using actual trainer pattern + batch = await services["replay_buffer"].sample.call_one(curr_policy_version=step) + if batch is not None: + inputs, targets = batch # GRPO returns (inputs, targets) tuple + loss = await services["trainer"].train_step.call(inputs, targets) + + # 7. Policy synchronization - Using actual weight update pattern + await services["trainer"].push_weights.call(step + 1) + await services["policy"].update_weights.fanout(step + 1) + + return loss + + return None + + + def create_episode_from_response(response, score, ref_logprobs, step): + """Helper function to create episode from response data""" + return { + "response": response, + "score": score, + "ref_logprobs": ref_logprobs, + "step": step, + } + + + + + + + + + +.. GENERATED FROM PYTHON SOURCE LINES 160-165 + +Setting Up Forge Services +-------------------------- + +Here's how to initialize the complete RL system with proper resource allocation. +Each service can scale independently based on its computational needs: + +.. GENERATED FROM PYTHON SOURCE LINES 165-335 + +.. code-block:: Python + + + + async def setup_forge_rl_system(): + """ + Complete setup of Forge RL services with proper resource allocation. + This example uses Qwen 3.1-1.7B model for demonstration. + """ + # Note: In actual Forge environment, imports would be: + # from forge.actors.policy import Policy + # from forge.actors.replay_buffer import ReplayBuffer + # from forge.actors.reference_model import ReferenceModel + # from forge.actors.trainer import RLTrainer + # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages + # from forge.data.rewards import MathReward, ThinkingReward + + model = "Qwen/Qwen3-1.7B" + group_size = 1 + + # Initialize all services with appropriate resource allocation + services = await asyncio.gather( + # Dataset actor (CPU intensive for I/O) + create_dataset_actor(model), + # Policy service (GPU for inference) + create_policy_service(model, group_size), + # Trainer actor (GPU for training) + create_trainer_actor(model), + # Replay buffer (CPU for memory management) + create_replay_buffer_actor(), + # Advantage computation (CPU) + create_advantages_actor(), + # Reference model (GPU for baseline) + create_reference_model_actor(model), + # Reward actor (CPU/small GPU for evaluation) + create_reward_actor(), + ) + + service_names = [ + "dataloader", + "policy", + "trainer", + "replay_buffer", + "compute_advantages", + "ref_model", + "reward_actor", + ] + + return dict(zip(service_names, services)) + + + # Service creation functions (would use actual Forge APIs) + async def create_dataset_actor(model): + """DatasetActor for loading training data""" + return { + "name": "DatasetActor", + "config": { + "path": "openai/gsm8k", + "revision": "main", + "data_split": "train", + "streaming": True, + "model": model, + }, + "resources": "CPU", + "sample": lambda: { + "call_one": lambda: {"request": "What is 2+2?", "target": "4"} + }, + } + + + async def create_policy_service(model, group_size): + """Policy service for text generation""" + return { + "name": "Policy", + "config": { + "engine_config": { + "model": model, + "tensor_parallel_size": 1, + "pipeline_parallel_size": 1, + "enforce_eager": False, + }, + "sampling_config": { + "n": group_size, + "max_tokens": 16, + "temperature": 1.0, + "top_p": 1.0, + }, + }, + "resources": "GPU", + "generate": lambda: {"route": lambda prompt: [MockResponse()]}, + } + + + async def create_trainer_actor(model): + """RLTrainer for policy optimization""" + return { + "name": "RLTrainer", + "config": { + "model": { + "name": "qwen3", + "flavor": "1.7B", + "hf_assets_path": f"hf://{model}", + }, + "optimizer": {"name": "AdamW", "lr": 1e-5}, + "training": {"local_batch_size": 2, "seq_len": 2048}, + }, + "resources": "GPU", + "train_step": lambda: {"call": lambda inputs, targets: 0.5}, + } + + + async def create_replay_buffer_actor(): + """ReplayBuffer for experience storage""" + return { + "name": "ReplayBuffer", + "config": {"batch_size": 2, "max_policy_age": 1, "dp_size": 1}, + "resources": "CPU", + "add": lambda: {"call_one": lambda episode: None}, + "sample": lambda: {"call_one": lambda curr_policy_version: ([], [])}, + } + + + async def create_advantages_actor(): + """ComputeAdvantages for advantage estimation""" + return {"name": "ComputeAdvantages", "resources": "CPU"} + + + async def create_reference_model_actor(model): + """ReferenceModel for baseline computation""" + return { + "name": "ReferenceModel", + "config": { + "model": { + "name": "qwen3", + "flavor": "1.7B", + "hf_assets_path": f"hf://{model}", + }, + "training": {"dtype": "bfloat16"}, + }, + "resources": "GPU", + "forward": lambda: { + "route": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor( + [0.1, 0.2] + ) + }, + } + + + async def create_reward_actor(): + """RewardActor for response evaluation""" + return { + "name": "RewardActor", + "config": {"reward_functions": ["MathReward", "ThinkingReward"]}, + "resources": "CPU", + "evaluate_response": lambda: {"route": lambda prompt, response, target: 0.95}, + } + + + class MockResponse: + """Mock response object for demonstration""" + + def __init__(self): + self.text = "The answer is 4" + self.prompt_ids = torch.tensor([1, 2, 3]) + self.token_ids = torch.tensor([4, 5, 6]) + + + # Demonstrate the setup + print("Setting up Forge RL system...") + # services = await setup_forge_rl_system() # Would work in async context + print("Forge services configured with independent scaling capabilities") + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + Setting up Forge RL system... + Forge services configured with independent scaling capabilities + + + + +.. GENERATED FROM PYTHON SOURCE LINES 336-341 + +Why Forge Architecture Matters +------------------------------- + +Traditional ML infrastructure fails for RL because each component has +different resource needs, scaling patterns, and failure modes: + +.. GENERATED FROM PYTHON SOURCE LINES 341-412 + +.. code-block:: Python + + + + def show_infrastructure_challenges(): + """ + Demonstrate why traditional monolithic RL fails and how Forge solves it. + """ + print("=== Infrastructure Challenges ===\n") + + print("Problem 1: Different Resource Needs") + resource_requirements = { + "Policy (Student AI)": { + "generates": "'The answer is 4'", + "needs": "Large GPU memory", + "scaling": "Multiple replicas for speed", + }, + "Reward Model (Teacher)": { + "scores": "answers: 0.95", + "needs": "Moderate compute", + "scaling": "CPU or small GPU", + }, + "Trainer (Tutor)": { + "improves": "student weights", + "needs": "Massive GPU compute", + "scaling": "Distributed training", + }, + "Dataset (Question Bank)": { + "provides": "'What is 2+2?'", + "needs": "CPU intensive I/O", + "scaling": "High memory bandwidth", + }, + } + + for component, reqs in resource_requirements.items(): + print(f"{component}:") + for key, value in reqs.items(): + print(f" {key}: {value}") + print() + + print("Problem 2: Coordination Complexity") + print("Unlike supervised learning with independent batches,") + print("RL requires complex coordination between components:") + print("- Policy waits idle while reward model works") + print("- Training waits for single episode (batch size = 1)") + print("- Everything stops if any component fails") + print() + + print("=== Forge Solutions ===\n") + + print("✅ Automatic Resource Management") + print("- Routing to least loaded replica") + print("- GPU memory management") + print("- Batch optimization") + print("- Failure recovery") + print("- Auto-scaling based on demand") + print() + + print("✅ Independent Scaling") + print("- Policy: num_replicas=8 for high inference demand") + print("- RewardActor: num_replicas=16 for parallel evaluation") + print("- Trainer: Multiple actors for distributed training") + print() + + print("✅ Fault Tolerance") + print("- Automatic routing to healthy replicas") + print("- Background replica respawn") + print("- Graceful degradation") + print("- System continues during component failures") + + + show_infrastructure_challenges() + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + === Infrastructure Challenges === + + Problem 1: Different Resource Needs + Policy (Student AI): + generates: 'The answer is 4' + needs: Large GPU memory + scaling: Multiple replicas for speed + + Reward Model (Teacher): + scores: answers: 0.95 + needs: Moderate compute + scaling: CPU or small GPU + + Trainer (Tutor): + improves: student weights + needs: Massive GPU compute + scaling: Distributed training + + Dataset (Question Bank): + provides: 'What is 2+2?' + needs: CPU intensive I/O + scaling: High memory bandwidth + + Problem 2: Coordination Complexity + Unlike supervised learning with independent batches, + RL requires complex coordination between components: + - Policy waits idle while reward model works + - Training waits for single episode (batch size = 1) + - Everything stops if any component fails + + === Forge Solutions === + + ✅ Automatic Resource Management + - Routing to least loaded replica + - GPU memory management + - Batch optimization + - Failure recovery + - Auto-scaling based on demand + + ✅ Independent Scaling + - Policy: num_replicas=8 for high inference demand + - RewardActor: num_replicas=16 for parallel evaluation + - Trainer: Multiple actors for distributed training + + ✅ Fault Tolerance + - Automatic routing to healthy replicas + - Background replica respawn + - Graceful degradation + - System continues during component failures + + + + +.. GENERATED FROM PYTHON SOURCE LINES 413-417 + +Production Scaling Example +--------------------------- + +Here's how you would scale the system for production workloads: + +.. GENERATED FROM PYTHON SOURCE LINES 417-469 + +.. code-block:: Python + + + + def demonstrate_production_scaling(): + """ + Show how Forge services scale independently for production. + """ + print("=== Production Scaling Configuration ===\n") + + scaling_config = { + "Policy Service": { + "replicas": 8, + "reason": "High inference demand from multiple training runs", + "resources": "GPU-heavy instances", + }, + "RewardActor Service": { + "replicas": 16, + "reason": "Parallel evaluation of many responses", + "resources": "CPU/small GPU instances", + }, + "Trainer Actor": { + "replicas": 4, + "reason": "Distributed training across multiple nodes", + "resources": "Large GPU clusters", + }, + "Dataset Actor": { + "replicas": 2, + "reason": "I/O intensive data loading", + "resources": "High-bandwidth CPU instances", + }, + "ReplayBuffer Actor": { + "replicas": 1, + "reason": "Centralized experience storage", + "resources": "High-memory instances", + }, + } + + for service, config in scaling_config.items(): + print(f"{service}:") + print(f" Replicas: {config['replicas']}") + print(f" Reason: {config['reason']}") + print(f" Resources: {config['resources']}") + print() + + print("Key Benefits:") + print("- Each service scales based on its bottlenecks") + print("- Resource utilization is optimized") + print("- Costs are minimized (no idle GPUs)") + print("- System maintains performance under load") + + + demonstrate_production_scaling() + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + === Production Scaling Configuration === + + Policy Service: + Replicas: 8 + Reason: High inference demand from multiple training runs + Resources: GPU-heavy instances + + RewardActor Service: + Replicas: 16 + Reason: Parallel evaluation of many responses + Resources: CPU/small GPU instances + + Trainer Actor: + Replicas: 4 + Reason: Distributed training across multiple nodes + Resources: Large GPU clusters + + Dataset Actor: + Replicas: 2 + Reason: I/O intensive data loading + Resources: High-bandwidth CPU instances + + ReplayBuffer Actor: + Replicas: 1 + Reason: Centralized experience storage + Resources: High-memory instances + + Key Benefits: + - Each service scales based on its bottlenecks + - Resource utilization is optimized + - Costs are minimized (no idle GPUs) + - System maintains performance under load + + + + +.. GENERATED FROM PYTHON SOURCE LINES 470-474 + +Complete RL Training Loop +-------------------------- + +Here's a complete example showing multiple RL training steps: + +.. GENERATED FROM PYTHON SOURCE LINES 474-528 + +.. code-block:: Python + + + + async def complete_rl_training_example(num_steps: int = 5): + """ + Complete RL training loop using Forge services. + """ + print(f"=== Running {num_steps} RL Training Steps ===\n") + + # Setup services (mock for demonstration) + services = { + "dataloader": await create_dataset_actor("Qwen/Qwen3-1.7B"), + "policy": await create_policy_service("Qwen/Qwen3-1.7B", 1), + "trainer": await create_trainer_actor("Qwen/Qwen3-1.7B"), + "replay_buffer": await create_replay_buffer_actor(), + "ref_model": await create_reference_model_actor("Qwen/Qwen3-1.7B"), + "reward_actor": await create_reward_actor(), + } + + losses = [] + + for step in range(num_steps): + print(f"Step {step + 1}:") + + # Simulate the RL step (would use actual forge_rl_step in practice) + sample = await services["dataloader"]["sample"]()["call_one"]() + print(f" Question: {sample['request']}") + print(f" Target: {sample['target']}") + + # Generate response + responses = await services["policy"]["generate"]()["route"](sample["request"]) + print(f" Generated: {responses[0].text}") + + # Get reward + score = await services["reward_actor"]["evaluate_response"]()["route"]( + sample["request"], responses[0].text, sample["target"] + ) + print(f" Reward: {score}") + + # Simulate training (every few steps when buffer has enough data) + if step >= 2: # Start training after accumulating some experience + loss = await services["trainer"]["train_step"]()["call"]([], []) + losses.append(loss) + print(f" Training Loss: {loss:.4f}") + + print() + + print(f"Training completed! Average loss: {sum(losses)/len(losses):.4f}") + return losses + + + # Run the example (would work in async context) + print("Complete RL training example:") + print("(In real usage, run: await complete_rl_training_example(5))") + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + Complete RL training example: + (In real usage, run: await complete_rl_training_example(5)) + + + + +.. GENERATED FROM PYTHON SOURCE LINES 529-558 + +Conclusion +---------- + +This tutorial demonstrated how RL fundamentals map to Forge's distributed +service architecture. Key takeaways: + +1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.) + becomes an independent, scalable Forge service + +2. **Resource Optimization**: Services scale independently based on their + computational needs (GPU for inference/training, CPU for data/rewards) + +3. **Fault Tolerance**: Individual service failures don't stop the entire + training pipeline - Forge handles routing and recovery automatically + +4. **Simple Interface**: Complex distributed systems are hidden behind + simple async function calls + +The same RL logic that works conceptually scales to production workloads +without infrastructure code - Forge handles distribution, scaling, and +fault tolerance automatically. + +Further Reading +--------------- + +* `Forge Architecture Documentation <#>`_ +* `GRPO Implementation (apps/grpo/main.py) <#>`_ +* `Forge Service APIs <#>`_ +* `Production RL Scaling Guide <#>`_ + + +.. rst-class:: sphx-glr-timing + + **Total running time of the script:** (0 minutes 0.006 seconds) + + +.. _sphx_glr_download_tutorials_rl_fundamentals_forge_tutorial.py: + +.. only:: html + + .. container:: sphx-glr-footer sphx-glr-footer-example + + .. container:: sphx-glr-download sphx-glr-download-jupyter + + :download:`Download Jupyter notebook: rl_fundamentals_forge_tutorial.ipynb ` + + .. container:: sphx-glr-download sphx-glr-download-python + + :download:`Download Python source code: rl_fundamentals_forge_tutorial.py ` + + .. container:: sphx-glr-download sphx-glr-download-zip + + :download:`Download zipped: rl_fundamentals_forge_tutorial.zip ` diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.zip b/docs/source/tutorials/rl_fundamentals_forge_tutorial.zip new file mode 100644 index 0000000000000000000000000000000000000000..c77b9084e4c0471638195cc647797a00896fc0af GIT binary patch literal 40960 zcmdU2&2Ah?a+ZG&ycyUBs4d!R<;m;HKC zPv@gz+?`$JdA@P@b@nW1lzmf;Cg<5RHqBm^^UKj>I-Z_iZ9HD{&&I~~_LD_@F`e&k zZ)e~A<#aL6ehn6~pI_AVtlGbS|9n(mEKa+F>E-<^JY7`fe1ZnQ_^!XP(d}mE^U-jB zKYO^bk>QW|V0ToPm%D>v&PV&xK|LBwC;Q)BPUm$o-#wk;!JqGymw*0U_U%PcXP}$C z8I8x;xGd%qAp~0cgK<$+N+lz$-PydHm)|c&706fmLbTh?o=)dw2FZZO*>nQ&RoQ4_ zgwpw`$$qvgmAeOzD4$JXQR@Q^=mVM=R_RHxTr8F6oGtjv#W?4PW zhNG&Uk4_hLIW%-3hS#G3*x<17XFL0RF&Ym!8`Yo~7pG%ksh$_32|pZ9r?U#9tS_>& zVll2kbzII1yv9>FEGtNOP~xqV;=H6HWW{8N!3@h;iT~ku5+}V`QP-FWBZk#-AA|ed z#bi1kLVi~A2azy!5I2zd!PU#@d~lJye%O7`eee@Mh_4R5$%+9aQ)TlKGlEeJ%b|b% zc~Ol98FaXuS2ae*!A#H4b~J&;4|LR`2zcVVJu5Cp<54jm)mJLTqPm(4?iX*0QO(&t zxPlT-Ha1>fjH=8onyfC1!9`hdO2f64E&Q_isLIN>#Q+oHmybxj!0@1GgL*M9yBpt@ zA3;IE@w_(g&#KYo46}S$U{3Xo7z${~r%)O{#_R%W1yvZ1>d|ykj6)UMFe)Z%7|evK zW|S1HfRAhHt*=q#->|1PumLzPj^@R+1Nybuge+?-&Q8FV0D>+maL5WqBN61 z6H}R`29r~DGfeE{6*UF(U0{kiv(%kpl0A72CKg;C`@+|5w!LkZ{r2{Lb}*m59t}b6 z`$buC0;;St9=$5FT&xfD<>5~r{zbmUA)}|-33hx5c0BFu6e@boX&g;p20G{E1Pd6J z2rN`@pv{>QWiS0C1L;SOZfJKbb=A*#R?`fYuZ!^_AP9c3)}im6^U-U(rW(M`nYTRX z{_GRW@L_p|bqz_XO(X56)7cee1=n#tqQV*ZWF6Sh*J2@u^U+zIoz170*(nSpj7sSV z&c?+R4B8n8liaVTb$L3SzEVCbcqH&qzMYlx5#I}WTG3`dv19*<@@r|n6tM|43P{$A zlEW&(1c3-=-||V8hS_uyzD4Html!IPP8Jkab{QLetYI!?*n@{hGn|bW}onX0z)4`Fu9Le+gZK=g(jnVN`35 z<*UPk-;md_SjvJNYES#x&1|omeN|$RYMLNjIHOCU?xyK+L;168GaIU>>{epII$P>t zH?xP`>`0ea*3@|+YLp!$o2pN$GjY3J+tX=^BmXhhwwUQTf{B9bbW{sn(f90eMqMYR zxj62=8biBS9K9W*c#iI)E@t3zRvbB|d4pAadC6zyC1x?1_b{e_+wUW)q9Ci~ z@B4&eVkAO6_KQPa~1Nk@j+ztqj?2r2N zy-=kLxwQMIvb4L!aM5?MJbe&Q!dUOfjnWZ5qZKvF9d>ML9M_SJ5m)#&|)*>PvZDJKen2Ez+PW%c;VaoGqt!l*~R z>2_v(Y4-1b`*-%|Cj+QD-n4DPYI&xfQ$7uyojlVIe7AW zNfYB)^x@a$w=h0|MK$gs&>klnDC(pdi`&J-86&G3DjR+w{e^QV$FTQBeG~r|8$C(m4&bi}5*p^e6;+T$e zZIhp@;h>s)tBN{~5yxQ1i1;$Wv^p*@Y+YA(K7}Vh1Vv6C>p#l8^MH=t@q?3YUB0a? zizz49qnONwct_xZTZH!n(;&9bX?0*^!x3IxMgf4uO&UlJ+NZT%&VS;J-Nx~&z zyhTs#2uW>bKkZtl%U#UHd^#EZPWAy!DZVKgKV*v;>~9&#l7yNj#WOQoR2O~iTq!%0 z7q)hLDzj8?XD#(@uKQVmAhPaw0!}6-ndG!9`={z7`s&_fsl;E zwi2#h7ei)OhinDy8j!t^-B1}ugRtE;Lzr!U41(}w!Fjlrq-U_SoK$j3ijh73v7!5& z#;QjMySWAZW_FV(HH=Rsn3`g3Ae)5*#qV^-!;R9ogq|^uxdQu_8gRf2Q}U3L<~HH> zr!dyCZQaZyu=U}{1a_xDPA$lIB2=Z%kz~LK+Xmf`2?0bhld>E(LNg?L7c-MRXMiTW zwD;zQTWQTyiH3Y+-bC%!v(I%PftcOiGgFoQ_6@d9A9we5_qsp*TvK7LP6+4l`y{nGaubK`os$US%+{aKfZkS-G;wrLo9 z+H}4IqC2wTM~wDl_`1L$%DG9;N6a%8?P~ahutU>9FN*p?qpuxK=;Rd^i2kmYN!&l| zch0%J&;NZav#WWYbHs(Se#M3GINP%k-*dk_pz;l^!oaXBaIS;(l2{ocG^dBwZkeg7*~=zNa|SeMC4y;LeTPTsf$`b{4L@}UzW0~DvON{la?w2 za(jtYH@`wqX-`yIgrfyu$VZ$yZ3=CVyrD&_aEg~AbZCQMV-=i8ZBX5!2bZC#U5!9j z3|_)%KmYfuOKP2!s@WgF=q6slt}(kNQZp0T*JS+T?AWPXxTEtrlbawQ_{7Xw9()wZ zJA7bMZC;4sQ+iJg!y{UrdDGbJ3XRhqB9Ytq-DuW{#)6Q+82K#>6w3z^qu6F9dL z$8-~Nz-^76D0zd<-o7fofA-%0h|Ru62^HS~GZwrd*SH=J#>M67uxJg^CKSRJE_(oS z(P7UHh9H>+zUWb|=}~fNxOpFCd(7NkYv0^NtIxGsCR5F9fX$y%%;u1w9bDQ;RvMuM zmQ8Ot!68rCU-EdB?D>p;aypACwn@btW}`&Z1aAh>?D@IThE%4}Oiqi|YFz4}Zoh;x zo`c{sR5mp9BA9<+Mdlewv~oODQ}0+3Tr!KG83YdF(?knIs0d689`CYA_d%-7bk=7y z-g+7^9sC&UFN|+Edp3;(a!!MDNh!HcOcrcEJAN@8ygD>Vn$FhADy>X8MyH5}HmnjI zR0YnaIM=!~d&cYa(2)oSvqaZ&yT)S!B=c(Z`g?{?0WGIkXXD~EvRPP=jGR*~F3$QW z2`TFex+fOxEWbEIjmkTD?fXWTU}A&;Gi=erce!Qf2`X;B#Th|9=BxM0-A|hRM8Im* z-sRl9jQf%Wu!{sqoxDQAYFtt)A3k{WSwJE9aiw<1OC$l{v;h__r%`hsAoN0(%%>yv zaWrPRO&YAo)mj(tbt4G!MLmLi6VG_DWY^G+fJh+!+$%`l5*F8{v88+mqz(xVHH7AG zOJ&yZ2uLzH*;rvQrR+Ymn|q#MPs)&N+nI={4m=+50pPv@7^Ao|lG4!2X2ct5erjNYn5RF3sG`EVr`C`z9-{ zZryJct*t$N(8Zd2(0zCk$=`1dR%v&5_7LP%bi*tganTuQq@1nTx|W1WyO%rN4v3`n z1p<|j-v#K9-f`|X@LUV=7CyJ$Q#7o#PQ2I8Tc;@D%%oj4C9lJ@Nn#r1pM_Rg${10a z;~lzNY5dR;$fl=%g`-)KU6d>0LK8{x;IZFl-mTxqhGu+b3QjmM{wgO($AwKiGS?=_xo)>BRZ@0ShPT+{bTXC4bkw(!1$#4{zk*lPdLEv8%9)!f2Y4n` zv5u&EBt&~d&Flp^M#(yIA2ujv#VP6x>rq*OrMs*V`1az;g@qY37gZ1ygtwyF z&Q%D!3~C5pe9ctMya>*KiqTm82}Fwmfr6V>mez>bh6hCb=taSI4M)t9@OQf&G?R*r zF(M91!q%6vtY(Ok)zKJ|ldYSiTd9gXeug%NEc6E#+&@f^c?&Fqs9&fXZy*bpQ&$d& z#3+)K^#zWv!8u>XqPn3R+nZUOj#(l$s^imYGt($`M(#9ZBzx)@$?&u{;cQZ$O^*XD ziputQWN&40XmixRW>y(J*ue^jXOasPPcRwFb>Ji1-0b!`w!G!Z^DS5o-+*Lt8~)4I z_M^rM{U|VY(pvJ`Ad-L0U8>}9G?UeAXy%JeU!bgb#EMjgeWL*i0hj=*W*AJ+D|Ak3 zZMAGL>zFF61n+X9A&je4sno~?RAZ)J;B1nmktSRyn+*e|nX4q1HeD4o4LOD4V~dNt z-b>cQCs;JqNW!EoA~Z)>0m%lN*i0}($}E{lJ~i70oPH71fV)`0dykM?lb^{tzcm%7 z+2_UNRf21nVku4K^nDa2D3&WkoZLlXcLmG8#QC4ifSzJOy%`O$ZEEq1P?3V6wonXW zDCpR}$AX)r7eJR4{%(!4w=S%+{GB85e%~x1OB_W_I~j{ydB7v|+hjsgS@sTH-eW0A zr}qIleOr1>KVK4nKmRBwG(R93^xM3R;_V(ZZ{%Af%{Z%Dux+WpLV6FF2+UlQI;a3(M$&>r?Px$N@NUUq2?+3N)NI%dKzP@`5OM6fbw zcKW;+ykhbX5rS!gf2&#XhF;zq5J==Jl!KRNi!lVo+G5>TH`*NOVKsxt(FFE78!k{z zidfK^LL2>X0d8zu7xmm}(1E*mV|_o{fl4ZLUN;guk=_m|Yv$9Q9y8 z*Yo^vDbEPbV6*KxJP**si@~_86fQMv*M)1D27Id}d>CY5*AR2#SSL`8r`X-#2nYeg zFsPwtbRg;A)=&dg7PgHKUZD{|uxYHr4oof2Eg2NX%eNgKs;5SjY5GPnrhS;f6RP(_ z=%}=hKeCJaqmy@BbVZ$j%V;wL-$EC`v3R zZcBtUP^Z(n#xA=Yw1YwyTEUxQ8Z`n&638BY#9<0tt|@wdtlKf{*8eBcV2TzV8W7jJo;_qbDUse!(hpoz_)8~+*?Z&y0Y z&{y)-m?;diHC^=gq~nYXf_`V;S*6z~KeGgQIl#8V#S^_##mG+k&@jL|ebTx!W?mn@ zg?ER;E35Wgxly`r)t@o>;bJ1!j^DXAzGrI1N#%%)$DWf_5JYBH3yYH6(i{?NRtEc2;6*`c$aO2p5F#A*EH z5=qJ~)AqlVQjnm%)`n~es)3`0+MPCAB3cwu2O z53=umQ1;bBaOqxVZ<)}eIabIf^SV?F_u+c?NRAUnAQ7vMo}anshE9t!B$S(%cG#7F zQd6fvE+>N54;sVrI%lh;I=IV>;=Z%;jnqq{HkQ{GO$i_}-S!3}&60l|k{qJezj&M> zRYYaRbmRWNrCOXJE<|iOScs0(_*b}V7sNx67_O`B$9*MI*0X)5zfb{lklZ4M+wQ0j z1#Cixq&?phnpX&C%D+YRIb7sR)>UEukfpt^{B7}khiu%fgdeq_W8rT9a&T+|Y| z+6w>7mCCxE_kF7^sQNaQ$z105)$n22P&T-Bx4qp+L6IjU<3>ol^FFjowvgtsv*jx-$-ACr|^s5E%MohDfg9!EM-p~Kz zzyIW$d-wSFPp?OP8qKaIr#wl_2PN+6A`Cqie_;<7<>S6BM&U%b)IwgqEeDd4M>1?N zVcG6vF&^6>HNPxtW<-l9A%nWD@fi2Sab&oUSwgXR@hco&t59P9$zZy=F(RHr~`SskSUfi3&8LbDD)tmGt zAp+L1L%=%eT!VnM#l8Uo7JsGrT8+iD9tM_D0?eQyhd6_9Yw1W5(>d#+_E2nalqO%=IH1uKiy^1mJn%dv42**D;@9F1)` zTyEoG1r0vG2_RO`Z#_gTN+k|U6m#efFtK`*|20so-eh@f-rZjL|*>4NFPN4oso7WYpX-AlL*3m@Q+G(9_njxZJNV(5^VMAB2TP0ykJW zKdjIKVhUiqdI(1Wz;i8*7l&+mO4%cS?_rO#1k6iqZ7-Mq#IvJm}sQ<;%a)!EQO8~%nX#}tv7{3qMj38{d z1|E!qHKv>@g_1YK?YDydVyFivdI!i~znaIg(qMm<4@gVd+Cj%FhSWJfq;?X4HV~x0tTmX zD%A^azV{MfLoh}(_sE|m(ad`GkOzH{6w{-Tp1F#Q&!}GPztS%^M_`trJw!k+dgHp3#(ZZ*mwc{y@Y$;z_YM*Bd_2p6Q&7wyxzEj+I9*?lsZP zv|A9K#4&_*q3tn*rpXWE?h($EL3h~;B9dqoNKChZI8$mokG2q|X_*^z1q%F3@1?cG z>{z-b%vnqFE|l`VzbW=F*F-u~+P3idj-bwzXv+jdhyN5<679?{ksITk>3g+!J7I!a z?^i>z2IQF%PGX+vBd(^#En}#QassQg4(ORDls*}Sxzo!Wl$`vy0{6@ck$in9@|kC> zHKPGXE5OfI5b$R88vL_xK*15J{n&iO>&pDd>`rm=c`a-o&sbnkd2honb$p1zjIDj; zTs`$|pwL1eZ&-{rH`YIJ4`P3gHYqFiFCo9$Kj7bsyeG(LvIWAB>;~rsvDgO&@%H*+ z3pb6k6lP-X0vWA0iE01`0J&dzdb$i4jlA7JL_gGU6{(9Pq+6)g`m-x1kS@2wbyG?b zoZ2G58vwhtd9o1#N*JPucs074(uttUBqB<~yDe`WISUJTb{NzPuj(4rs`<`gB(%J{ zF{-Yc-<7E~m!usJUq1}uEx`cOUF&1KagmTCIZuGM>ou$KVEv?l2b@8xJ=<*)C-@Et z5$EXke8PO&3kJ9f>P;z|A_P|Jsc&jvy=f~Io7u2Xmyk7H0`}%RB1e!Ln+xt@jd1kYei|SetaT zC|vnw$gKnD2+L6v^vyRjj5_p=DoM)cI#k14EsNc1r1*oQKVVC{7;{SV{YbU~y4@ZD{f0v@tIOBpT5WD`=cH6HQ38bI5UQ zCeU$SfAn=`vMqX?fBe3oROSkXz7LgQ(c2J2O3sB(9SE1FrLRH}k08^>jeO>Y!VT|g z;|=&kONRBek9xUDy_2Hl5FhcsmS$XQMk6&j5I9%sWYmPl&~odmh|+3E2q|`4#^FF( z$`Jw0C0blFRea1p5LN1sfbuupq-|E;HUhzXD>>WW}B|0J?MXjG^xM@gBu&!0zPtm2z!6FytD& z^h6oj{>lPCSll~5iD?JuZq#_n*)`yIt0&0EZOPf;E+5D3NP2!HSwG;R^x|j1 zrp^v%xd!@83jh_Aj;c|3a>hM@O|AwTyPN16-}6Mz)dGRX%7)NE*l}AO%I&xpVC?A{ zLxL;-%11Ka4pWshwFIMgwvP4u{D@g=&Y3+^wIXN&?nM?U0lF88bv@ucD_QNCK;P2` z9B&A}mnf8KG}NIa0rlV>6oIeN#eHcJQbc+jXfgX71fSise&G>2w2YA1Q(q)w6WK`| zKB>-hmyditYDor`hcYXW`20(sg_nlp8_KZ9SRacIY~3J@{ehYO)kuAm^2-3b?$$7T zINNc^b9S-Z1ci@`+(0Nltla)u_<_0v>?-6k3sw6=Km^;AZ3!7K&p(f^H#MLHA$QhB zD+$}};Cehk>BT|V$!;ME`X%2>VG7bdHnuB0ccr-E(_Y>J7L1|_z8#wfq8n;+4~L>j z(?n~YI~nHV~3tL1iQk?-F zTnSp3=qxJ=CS`rx!UT~up$i*5Tl26Czc6X!QRw>92cN9QF>JiBM_4|OY?p#EO!VN{ zSnCuk(XI_>m}u0*G)y#E1#8%7-jbjZ3T{-F#4O-Vy(8-~(AcjOib5044RFF02#1ZA zi!^Tma+n}v%=-E`hYcFbX4J!a-Bqa1 zHwZ~TFiuCzWl;-_##wMg@)?xm+y?!SByKR2A6|6rF`Jg)HtO08qw6DrRPb+Ow?@C+ z{7y#tVnhpO0=jKt)@{+kCT0p(sbmMJSM<@S7nxiWDg-c&-ub%V*qqKLBN-c@^5JJe z-huXrNhM;tX_RwgK*uOZvGkg!ElpI%sFRIsv~!r_S$9Z4ck}|WJoquKh42`$WNAd> zKu{Od6F5-6sAD`98!ek8Q)0ssPRyf~gGj}>s2)i`K1K{BN8&d+7WXmg6A!{h%lPR20f7^9+~+J5Mzmct>7Ypi*y~Vi3N~1 zzvRKlODjc}BSt2=A6h)_WuTFXKG)+$COWMN9VtEOfv^FkB93aGyy6q_BZ|*$x>(8P`m^aFe(k{IX5Jt!@S#>qB5h zk;mFFnEHCN=$I)Fa&vKooI@(pI(`;KA)jteMibShwiZu~sg?)PgllQ2sVCEqUGZgt zj38oq(-_>lXpj2 zm80`ByrN=2GPw>ZaYnXsi4ovAEtpI%PN!=}4WLY8yhw_F?f@&(zf`N&T$Qj@KAg@` zsT6q?Cvo`Q*0~XI5a)ZYqz%7&a&8RQfMz;N&H)9Tmg8o&gkbS2-3g}W0?tsrC@ef} zG9aM7pp;|sB}+Ogb3*74G^tv0wH_i!kJ56HM$t?#ZGkjBx00bw&44*Jl~P=tgTx7E zgKaXyoySPWUcF4;8)bj~Nsn8%7zjXwdks1KyF(SVX7@(Wnx52$V{3X^ z9M1Ju$RIAUU#8T^>pSMaLTt8aHZV5h2BC&qd3@bK*=*>Fc4@?INHx1qfq84j4>!Zw zbPF%IDpm^ii(s7H1aQ-z#U|aXh1`@k>S%f&g1QMh|Chm;#pI+BkKwL@5b~xdoJe*V zSj|-+oP6b-VQ`8f-VB11@4qtyPI*6d^}_);$=99XZwg;7CbR{?xGb!xs`cSF$<^JU zZwgB*z&H6O!g0W=e=zo@0)l-w^d^v!*kylb>#9$13XOK$|C> zO&CBlToYx}wO7JNW~~t*!aWKyhd8>jXg+nZGLMlX8qN#2fG= z-Jr(h04kMu)_x)M0lpCnH$6V~$dhA$r7r+Oh+g?jp899Dmwx`S+B^eKncI%cmruiY zGF^N*n$Pi->n!FA9s%jok_7fAIdSj8BcL=Bc^K_KHtc&O%rksz7{GGm>qhjeYPz#4 z*Ezo&0YKAyX)V7c^Zwi227UM9*5{|lTj13-v3P*HC9wJ}NzAU!<+~B_J=kxxg&9B~ zNzZ@&ad!`2Li?E!7)B!_$tS0XzX2SIAa-#$J5=YRdT^M84M?;ijDSU%PN{nKCbUwjcm2)o?J3jj)QVVWVNCb=d?tEKo) gfBm0$!8POrvGw3rfAq)f_51kmAMx+=f8cNb17h3yE&u=k literal 0 HcmV?d00001 diff --git a/docs/source/tutorials/sg_execution_times.rst b/docs/source/tutorials/sg_execution_times.rst new file mode 100644 index 00000000..0bb42ae9 --- /dev/null +++ b/docs/source/tutorials/sg_execution_times.rst @@ -0,0 +1,40 @@ + +:orphan: + +.. _sphx_glr_tutorials_sg_execution_times: + + +Computation times +================= +**00:01.107** total execution time for 2 files **from tutorials**: + +.. container:: + + .. raw:: html + + + + + + + + .. list-table:: + :header-rows: 1 + :class: table table-striped sg-datatable + + * - Example + - Time + - Mem (MB) + * - :ref:`sphx_glr_tutorials_template_tutorial.py` (``template_tutorial.py``) + - 00:01.101 + - 0.0 + * - :ref:`sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py` (``rl_fundamentals_forge_tutorial.py``) + - 00:00.006 + - 0.0 diff --git a/docs/source/tutorials/template_tutorial.codeobj.json b/docs/source/tutorials/template_tutorial.codeobj.json new file mode 100644 index 00000000..21bdbcd5 --- /dev/null +++ b/docs/source/tutorials/template_tutorial.codeobj.json @@ -0,0 +1,27 @@ +{ + "torch.rand": [ + { + "is_class": false, + "is_explicit": false, + "module": "builtins", + "module_short": "builtins", + "name": "builtin_function_or_method" + }, + { + "is_class": false, + "is_explicit": false, + "module": "torch", + "module_short": "torch", + "name": "rand" + } + ], + "x": [ + { + "is_class": false, + "is_explicit": false, + "module": "torch", + "module_short": "torch", + "name": "Tensor" + } + ] +} \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.ipynb b/docs/source/tutorials/template_tutorial.ipynb new file mode 100644 index 00000000..ae808fec --- /dev/null +++ b/docs/source/tutorials/template_tutorial.ipynb @@ -0,0 +1,75 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n# Template Tutorial\n\n**Author:** [FirstName LastName](https://github.com/username)\n\n.. grid:: 2\n\n .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn\n :class-card: card-prerequisites\n\n * Item 1\n * Item 2\n * Item 3\n\n .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites\n :class-card: card-prerequisites\n\n * PyTorch v2.0.0\n * GPU ???\n * Other items 3\n\n\nTo test your tutorial locally, you can do one of the following:\n\n* You can control specific files that generate the results by using\n ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable\n respects regular expressions.\n For example to run only ``neural_style_transfer_tutorial.py``,\n use the following command:\n\n```sh\nGALLERY_PATTERN=\"neural_style_transfer_tutorial.py\" make html\n```\n or\n\n```sh\nGALLERY_PATTERN=\"neural_style_transfer_tutorial.py\" sphinx-build . _build\n```\n* Make a copy of this repository and add only your\n tutorial to the `beginner_source` directory removing all other tutorials.\n Then run ``make html``.\n\nVerify that all outputs were generated correctly in the created HTML.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Overview\n\nDescribe Why is this topic important? Add Links to relevant research papers.\n\nThis tutorial walks you through the process of....\n\n## Steps\n\nExample code (the output below is generated automatically):\n\n\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "import torch\n\nx = torch.rand(5, 3)\nprint(x)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## (Optional) Additional Exercises\n\nAdd additional practice exercises for users to test their knowledge.\nExample: [NLP from Scratch](https://pytorch.org/tutorials/intermediate/char_rnn_generation_tutorial.html#exercises)_.\n\n\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Conclusion\n\nSummarize the steps and concepts covered. Highlight key takeaways.\n\n## Further Reading\n\n* Link1\n* Link2\n\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.18" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.py b/docs/source/tutorials/template_tutorial.py new file mode 100644 index 00000000..4018aa1b --- /dev/null +++ b/docs/source/tutorials/template_tutorial.py @@ -0,0 +1,91 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +""" +Template Tutorial +================= + +**Author:** `FirstName LastName `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + :class-card: card-prerequisites + + * Item 1 + * Item 2 + * Item 3 + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + :class-card: card-prerequisites + + * PyTorch v2.0.0 + * GPU ??? + * Other items 3 + + +To test your tutorial locally, you can do one of the following: + +* You can control specific files that generate the results by using + ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable + respects regular expressions. + For example to run only ``neural_style_transfer_tutorial.py``, + use the following command: + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html + + or + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build + +* Make a copy of this repository and add only your + tutorial to the `beginner_source` directory removing all other tutorials. + Then run ``make html``. + +Verify that all outputs were generated correctly in the created HTML. +""" + +######################################################################### +# Overview +# -------- +# +# Describe Why is this topic important? Add Links to relevant research papers. +# +# This tutorial walks you through the process of.... +# +# Steps +# ----- +# +# Example code (the output below is generated automatically): +# +import torch + +x = torch.rand(5, 3) +print(x) + +###################################################################### +# (Optional) Additional Exercises +# ------------------------------- +# +# Add additional practice exercises for users to test their knowledge. +# Example: `NLP from Scratch `__. +# + +###################################################################### +# Conclusion +# ---------- +# +# Summarize the steps and concepts covered. Highlight key takeaways. +# +# Further Reading +# --------------- +# +# * Link1 +# * Link2 diff --git a/docs/source/tutorials/template_tutorial.py.md5 b/docs/source/tutorials/template_tutorial.py.md5 new file mode 100644 index 00000000..60b15697 --- /dev/null +++ b/docs/source/tutorials/template_tutorial.py.md5 @@ -0,0 +1 @@ +1d0dbb374fded9aaf6cf60085291fe43 \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.rst b/docs/source/tutorials/template_tutorial.rst new file mode 100644 index 00000000..e4768c07 --- /dev/null +++ b/docs/source/tutorials/template_tutorial.rst @@ -0,0 +1,152 @@ + +.. DO NOT EDIT. +.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. +.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: +.. "tutorials/template_tutorial.py" +.. LINE NUMBERS ARE GIVEN BELOW. + +.. only:: html + + .. note:: + :class: sphx-glr-download-link-note + + :ref:`Go to the end ` + to download the full example code. + +.. rst-class:: sphx-glr-example-title + +.. _sphx_glr_tutorials_template_tutorial.py: + + +Template Tutorial +================= + +**Author:** `FirstName LastName `_ + +.. grid:: 2 + + .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn + :class-card: card-prerequisites + + * Item 1 + * Item 2 + * Item 3 + + .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites + :class-card: card-prerequisites + + * PyTorch v2.0.0 + * GPU ??? + * Other items 3 + + +To test your tutorial locally, you can do one of the following: + +* You can control specific files that generate the results by using + ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable + respects regular expressions. + For example to run only ``neural_style_transfer_tutorial.py``, + use the following command: + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html + + or + + .. code-block:: sh + + GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build + +* Make a copy of this repository and add only your + tutorial to the `beginner_source` directory removing all other tutorials. + Then run ``make html``. + +Verify that all outputs were generated correctly in the created HTML. + +.. GENERATED FROM PYTHON SOURCE LINES 56-68 + +Overview +-------- + +Describe Why is this topic important? Add Links to relevant research papers. + +This tutorial walks you through the process of.... + +Steps +----- + +Example code (the output below is generated automatically): + + +.. GENERATED FROM PYTHON SOURCE LINES 68-73 + +.. code-block:: Python + + import torch + + x = torch.rand(5, 3) + print(x) + + + + + +.. rst-class:: sphx-glr-script-out + + .. code-block:: none + + tensor([[0.0394, 0.0994, 0.5448], + [0.9781, 0.6839, 0.4422], + [0.9133, 0.0060, 0.0412], + [0.7065, 0.5727, 0.8577], + [0.4415, 0.9282, 0.3205]]) + + + + +.. GENERATED FROM PYTHON SOURCE LINES 74-80 + +(Optional) Additional Exercises +------------------------------- + +Add additional practice exercises for users to test their knowledge. +Example: `NLP from Scratch `__. + + +.. GENERATED FROM PYTHON SOURCE LINES 82-92 + +Conclusion +---------- + +Summarize the steps and concepts covered. Highlight key takeaways. + +Further Reading +--------------- + +* Link1 +* Link2 + + +.. rst-class:: sphx-glr-timing + + **Total running time of the script:** (0 minutes 1.101 seconds) + + +.. _sphx_glr_download_tutorials_template_tutorial.py: + +.. only:: html + + .. container:: sphx-glr-footer sphx-glr-footer-example + + .. container:: sphx-glr-download sphx-glr-download-jupyter + + :download:`Download Jupyter notebook: template_tutorial.ipynb ` + + .. container:: sphx-glr-download sphx-glr-download-python + + :download:`Download Python source code: template_tutorial.py ` + + .. container:: sphx-glr-download sphx-glr-download-zip + + :download:`Download zipped: template_tutorial.zip ` diff --git a/docs/source/tutorials/template_tutorial.zip b/docs/source/tutorials/template_tutorial.zip new file mode 100644 index 0000000000000000000000000000000000000000..321b881c49affa7b268465506f2f059b02457417 GIT binary patch literal 5757 zcmd5=-EJGl6;_kBKo`AC(c6O+3P^%TSyBtMOdLmb9?n!cPc;CvYn)n#Vjeq)8KRR^zreF(_?~A@Ca0L)_Tg7lNr>Xk{w~~LYj9{cB3ec z_igtF6YCo~Ax3fd(6Xh!ZLKKotMDy^;#QU=9?d^?~pVY%{W{bTV zrW&mJG)Rso zo~E{@O9g*wDVzz6z^j*OQB^Z8#JoDClTQArRu$M+Gl5vQLr)=%_S=2U?}L@!hwq}H zR+aD6#yUhvwmSB&iDfe}-d(%?^vt?!LKg?!2i*q?t-k30o*q4Vv>?1h+#Rv&DuTAr znI+g6+=G;9;z-(NLhI>n@LDDe<(90Gvs{o!BCYMEGUFcNOY{|7%ZuQD@2sY(l$jc- zEM!%M?1B5o(nyE24rz&8uC=ddIHkG*8&)q(zc@NMIezt3(my&oJAU;dO-Y&y<*X^B z@pK_jN<%HX$bCsGYv;4f3EZIo*gH9{wQwY_5Y(zt)}W}c;E83LK$SurORhFBPEVnN zk+l;#2_=(wCrmYxZn=)rbeB0GL4zT4N5J-?K>flYm_~+jQFey#<2kaknzV_uqW8o2 zPKhxU;#|_i7dkMp?tdVnDksWZb%wRlIdv%s4;odqe10c|LO= zD$K$9c@n|3+=J4g94mw9C$oN)F7*~C3wwbCAY>cakecoG5GsV)g!rWC+zV-n_VWkn z)Mz?Ot+2QDWsQ(uqNL`zmP2pHZ49w(dRyisk}#()&z_%jLlcWO*EP^iUtXX`D|v~> zPBTKg{#;g>Q$vYX1XZ{s_`{Z{5ml7jv5bF2NATo{GUq&nG)V^ z`+w^yx)d6lxWoC0v-Nlq9A7${A?;DLU5v2p)XTDJl?#;PrVhd(vc>VT!IduNh%lcS;i*2~hP zn1$YEm#Nt^9bZY8sR~Vd;ruoTE8oF#_O@7TFP-2cM4~e^e9-0`SGo)F2wf8XQI5`y zz0@)v;~d zmsun|a0z`0bfW{pIhQC+bPjPTrcJ^=sa+`VS5oBM@o&T^2-^wyzmM73(?QgqL#=-U zYW?#cr~b2_Zfx-X&u@iVs+^kPU(kl)Oln=lJ$lU@nV7^QkP|;GC8lD|`Z#oSo`Q$7 znzJd%fRa2Gm|E-kbOFqk3ok%#xiLSSg`x!P0(m5KB^yCs&-cv=DyZ^iPMq~Z?3t+- zRQm;rtJKzOHAKJC%E-83_9lEcEL-8%O+k>(b9-qP<}B;G(B?M50|P|43g!l8V3xt# z>zm;1&DLAM+cq;;0&o12&0Gh3n{fiOI}lPU6DwN;H{aSqxK+YoyXozU-i_ksjNeDe z>p*VKVuudQ?E$D4vzNhZE zxxjU-{5r85^9!%?bI-pEl;d4cDu6kDz6XZm?eFEt|F08s8Go`Nw;|AhX(8zMmoIl9 zR91Bz@N*t|z=&1!xj38Gi$1rL?t3TAeXt1DW)JtIFw^FM_H> z^P3Q_Xbf2Ly~`1TwA#9i8r``%>^x4pRVJ|5qGbB Date: Tue, 7 Oct 2025 11:46:42 -0700 Subject: [PATCH 03/26] Add API docs for actor/service --- docs/source/api.md | 48 +++++++------ docs/source/api_actors.md | 58 ++++------------ docs/source/api_core.md | 68 ------------------- docs/source/api_data.md | 63 ----------------- docs/source/api_data_models.md | 51 -------------- docs/source/api_losses.md | 27 -------- docs/source/api_service.md | 21 ++++++ docs/source/api_types.md | 41 ------------ docs/source/api_util.md | 89 ------------------------- docs/source/conf.py | 80 +++++++--------------- src/forge/controller/actor.py | 77 +++++++++++++++------ src/forge/controller/service/service.py | 11 +++ 12 files changed, 148 insertions(+), 486 deletions(-) delete mode 100644 docs/source/api_core.md delete mode 100644 docs/source/api_data.md delete mode 100644 docs/source/api_data_models.md delete mode 100644 docs/source/api_losses.md create mode 100644 docs/source/api_service.md delete mode 100644 docs/source/api_types.md delete mode 100644 docs/source/api_util.md diff --git a/docs/source/api.md b/docs/source/api.md index e4c2d064..f4ecd01d 100644 --- a/docs/source/api.md +++ b/docs/source/api.md @@ -1,34 +1,32 @@ # API Reference -This API reference is organized by priority of concepts that users should be exposed to, based on how they're used in the core Forge workflows. - -```{eval-rst} -.. toctree:: - :maxdepth: 2 - - api_core - api_actors - api_data_models - api_types - api_util - api_data - api_losses -``` +This section provides comprehensive API documentation for TorchForge. ## Overview -The Forge API is structured around key concepts in order of priority: +TorchForge is a PyTorch native platform for post-training generative AI models, +designed to streamline reinforcement learning workflows for large language +models. The platform leverages PyTorch's distributed computing capabilities +and is built on top of [Monarch](https://meta-pytorch.org/monarch/), +making extensive use of actors for distributed computation and fault tolerance. + +Key Features of TorchForge include: -1. **[Core Concepts](api_core.md)** - Actor System, ForgeActor, and ForgeService fundamentals -2. **[Built-in Actors](api_actors.md)** - Policy, Trainer, ReplayBuffer, and ReferenceModel -3. **[Data Models](api_data_models.md)** - Completion, Prompt, Episode, and other data structures -4. **[Configuration and Types](api_types.md)** - Core types and configuration classes -5. **[Utilities](api_util.md)** - Distributed computing, logging, and observability tools -6. **[Data Processing](api_data.md)** - Rewards, tokenization, and data handling utilities -7. **[Loss Functions](api_losses.md)** - GRPO and REINFORCE loss implementations +- **Actor-Based Architecture**: TorchForge uses an actor-based system for distributed training, providing excellent scalability and fault tolerance. +- **PyTorch Native**: Built natively on PyTorch, ensuring seamless integration with existing PyTorch workflows. +- **Post-Training Focus**: Specifically designed for post-training techniques like RLHF, SFT, and other alignment methods. +- **Distributed by Design**: Supports multi-GPU and multi-node training out of the box. -## Quick Start -To get started with Forge, begin with the [Core Concepts](api_core.md) to understand the actor system foundation, then explore the [Built-in Actors](api_actors.md) for common RL workflows. +For most use cases, you'll interact with the high-level service +interfaces, which handle the complexity of actor coordination and +distributed training automatically. -For a practical example, see the GRPO implementation in `apps/grpo/main.py` which demonstrates how these components work together in a complete reinforcement learning training loop. +For advanced users who need fine-grained control, the individual actor +APIs provide direct access to the underlying distributed components. + +```{toctree} +:maxdepth: 1 +api_actors +api_service +``` diff --git a/docs/source/api_actors.md b/docs/source/api_actors.md index 1be09712..f086591d 100644 --- a/docs/source/api_actors.md +++ b/docs/source/api_actors.md @@ -1,54 +1,20 @@ -# Built-in Actors - -On top of the services/actors foundation, Forge provides implementations of actors that are useful in RL workflows. - -## Policy - -Inference and generation via vLLM. The {class}`forge.actors.policy.Policy` is a key actor for generating completions from language models. +# ForgeActor ```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.policy.Policy - forge.actors.policy.PolicyWorker - forge.actors.policy.EngineConfig - forge.actors.policy.SamplingConfig +.. currentmodule:: forge.actors ``` -## Trainer - -Training via torchtitan. The {class}`forge.actors.trainer.RLTrainer` handles reinforcement learning training loops. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.trainer.RLTrainer -``` - -## ReplayBuffer - -For storing experience and sampling to the trainer - the glue between policy and trainer. The {class}`forge.actors.replay_buffer.ReplayBuffer` manages experience data for RL training. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.replay_buffer.ReplayBuffer -``` - -## ReferenceModel - -Used for RL correctness. The {class}`forge.actors.reference_model.ReferenceModel` provides reference logprobs for RL algorithms. +The actors module contains the core components for model training +and inference in TorchForge. These pre-built actors provide essential +functionality for reinforcement learning workflows and can be used +as building blocks for complex distributed training systems. ```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: +.. currentmodule:: forge.controller.actor - forge.actors.reference_model.ReferenceModel +.. autoclass:: ForgeActor + :members: + :undoc-members: + :show-inheritance: + :exclude-members: logger, setup, set_env ``` diff --git a/docs/source/api_core.md b/docs/source/api_core.md deleted file mode 100644 index 785302ad..00000000 --- a/docs/source/api_core.md +++ /dev/null @@ -1,68 +0,0 @@ -# Core Concepts - -## Actor System - -Forge is built on top of Monarch and makes extensive use of actors. -Actors are the foundation for building distributed, fault-tolerant systems. - -## ForgeActor - -In Forge, everything centers around the {class}`forge.controller.actor.ForgeActor`, which is a customized version of a Monarch actor tailored for Forge-specific needs. - -The {class}`forge.controller.actor.ForgeActor` differs from a standard Monarch actor by allowing you to specify resource requirements using the `options()` method. This API lets you define the resources your actor needs and create two types of constructs: - -- A regular Monarch actor using `as_actor()` -- A service using `as_service()` - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.controller.actor.ForgeActor -``` - -### Resource Configuration Example - -Options are important because they demonstrate the resource requirements and how we represent them in Forge: - -```python -from forge.controller.actor import ForgeActor - -class MyActor(ForgeActor): - pass - -# Create a service with specific resource requirements -service = MyActor.options( - hosts=1, - procs=8, - replicas=1, - with_gpus=True -).as_service() -``` - -This example creates a service that has 1 replica, where the replica consists of 1 remote host and 8 processes, using Monarch's remote allocations. - -### Key Methods - -**`options()`** -Configures resource requirements for the actor. - -**`setup()`** -Sets up an actor. All actors should implement this for any heavyweight setup (like PyTorch distributed initialization, model checkpoint loading, etc.). - -**`launch()`** -Logic to provision and deploy a new replica. This is what services use to spin up replicas. - -## ForgeService - -Services are replicated, fault-tolerant versions of actors. They provide high availability and load distribution across multiple actor instances. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.controller.service.Service - forge.controller.service.ServiceInterface -``` diff --git a/docs/source/api_data.md b/docs/source/api_data.md deleted file mode 100644 index a652c6d0..00000000 --- a/docs/source/api_data.md +++ /dev/null @@ -1,63 +0,0 @@ -# Data Processing - -Data handling utilities for datasets, rewards, and tokenization. - -## Rewards - -Reward functions for RL training. The {mod}`forge.data.rewards` module provides reward functions like {class}`forge.data.rewards.MathReward` and {class}`forge.data.rewards.ThinkingReward`. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.rewards -``` - -## Tokenization - -Tokenization utilities for processing text data. The {mod}`forge.data.tokenizer` module provides tokenization functions and utilities. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.tokenizer -``` - -## Data Collation - -Data collation utilities for batching and processing. The {mod}`forge.data.collate` module provides functions for collating data into batches. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.collate -``` - -## Data Sharding - -Data sharding utilities for distributed processing. The {mod}`forge.data.sharding` module provides sharding strategies for distributed training. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.sharding -``` - -## Data Utilities - -General data processing utilities. The {mod}`forge.data.utils` module provides miscellaneous data handling functions. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data.utils -``` diff --git a/docs/source/api_data_models.md b/docs/source/api_data_models.md deleted file mode 100644 index a7c02caa..00000000 --- a/docs/source/api_data_models.md +++ /dev/null @@ -1,51 +0,0 @@ -# Data Models - -Base data models for common RL workflows. - -## Completion - -Outputs from vLLM. The {class}`forge.data_models.completion.Completion` represents a model-generated completion for a given prompt. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data_models.completion.Completion -``` - -## Prompt - -Input prompts for models. The {class}`forge.data_models.prompt.Prompt` encapsulates prompt data for language models. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data_models.prompt.Prompt -``` - -## Episode - -Training episodes for RL. The {class}`forge.data_models.episode.Episode` represents a complete interaction episode in reinforcement learning. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data_models.episode.Episode -``` - -## ScoredCompletion - -Completions with associated scores. The {class}`forge.data_models.scored_completion.ScoredCompletion` extends completions with scoring information. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.data_models.scored_completion.ScoredCompletion -``` diff --git a/docs/source/api_losses.md b/docs/source/api_losses.md deleted file mode 100644 index d16f2a6f..00000000 --- a/docs/source/api_losses.md +++ /dev/null @@ -1,27 +0,0 @@ -# Loss Functions - -Loss functions for RL training. - -## GRPO Loss - -Group Relative Policy Optimization loss function. The {mod}`forge.losses.grpo_loss` module provides loss functions for GRPO training. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.losses.grpo_loss -``` - -## REINFORCE Loss - -REINFORCE algorithm loss function. The {mod}`forge.losses.reinforce_loss` module provides loss functions for REINFORCE training. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.losses.reinforce_loss -``` diff --git a/docs/source/api_service.md b/docs/source/api_service.md new file mode 100644 index 00000000..878d4332 --- /dev/null +++ b/docs/source/api_service.md @@ -0,0 +1,21 @@ +# Service + +```{eval-rst} +.. currentmodule:: forge.controller.service.service +``` + +```{eval-rst} +.. autoclass:: Service + :members: call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop + :show-inheritance: + :special-members: __init__ +``` + +## ServiceActor + +```{eval-rst} +.. autoclass:: ServiceActor + :members: call, call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop + :show-inheritance: + :special-members: __init__ +``` diff --git a/docs/source/api_types.md b/docs/source/api_types.md deleted file mode 100644 index e7bd423a..00000000 --- a/docs/source/api_types.md +++ /dev/null @@ -1,41 +0,0 @@ -# Configuration and Types - -## Core Type Definitions - -Core type definitions used throughout Forge. The {mod}`forge.types` module contains fundamental types and configurations. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.types -``` - -## Configuration Classes - -Configuration classes for actors and services. - -### EngineConfig - -Configuration for vLLM engines used in Policy actors. The {class}`forge.actors.policy.EngineConfig` extends vLLM's EngineArgs with worker-specific fields. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.policy.EngineConfig -``` - -### SamplingConfig - -Configuration for sampling parameters in Policy actors. The {class}`forge.actors.policy.SamplingConfig` provides overrides for vLLM's sampling parameters. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.actors.policy.SamplingConfig -``` diff --git a/docs/source/api_util.md b/docs/source/api_util.md deleted file mode 100644 index 06438b35..00000000 --- a/docs/source/api_util.md +++ /dev/null @@ -1,89 +0,0 @@ -# Utilities - -Utility functions and classes for distributed computing, logging, and operations. - -## Distributed Computing - -Utilities for distributed computing operations. The {mod}`forge.util.distributed` module provides functions for setting up distributed environments. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.util.distributed -``` - -## Logging - -Logging utilities for Forge applications. The {mod}`forge.util.logging` module provides logging configuration and utilities. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.util.logging -``` - -## Operations - -Core operations and utilities. The {mod}`forge.util.ops` module contains commonly used operations for RL workflows. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.util.ops -``` - -## Metric Logging - -Metric logging utilities. The {mod}`forge.util.metric_logging` module provides utilities for logging metrics. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.util.metric_logging -``` - -## Observability - -### Metrics - -Metrics collection and reporting. The {mod}`forge.observability.metrics` module provides the core metrics infrastructure. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.observability.metrics -``` - -### Performance Tracking - -Performance tracking utilities. The {mod}`forge.observability.perf_tracker` module provides performance measurement tools. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.observability.perf_tracker -``` - -### Metric Actors - -Actors for metric collection. The {mod}`forge.observability.metric_actors` module provides actors for distributed metric collection. - -```{eval-rst} -.. autosummary:: - :toctree: _autosummary - :recursive: - - forge.observability.metric_actors -``` diff --git a/docs/source/conf.py b/docs/source/conf.py index 2d1c95e6..aa73efa6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,33 +17,10 @@ import pytorch_sphinx_theme2 # Add the source directory to Python path so modules can be imported -sys.path.insert(0, os.path.abspath("../../src/forge")) - - -# Determine the version path for deployment -def get_version_path(): - """Get the version path based on environment variables or git context.""" - # Check if we're in CI/CD and get the target folder - github_ref = os.environ.get("GITHUB_REF", "") - - # Convert refs/tags/v1.12.0rc3 into 1.12. - # Matches the logic in .github/workflows/docs.yml - if github_ref.startswith("refs/tags/v"): - import re - - match = re.match(r"^refs/tags/v([0-9]+\.[0-9]+)\..*", github_ref) - if match: - return match.group(1) + "/" - - # Default to main for main branch or local development - return "main/" - - -# Set base URL based on deployment context -version_path = get_version_path() +sys.path.insert(0, os.path.abspath("../../src")) project = "torchforge" -copyright = "" +copyright = "2025, PyTorch Contributors" author = "PyTorch Contributors" release = "0.1" @@ -61,12 +38,9 @@ def get_version_path(): "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", - "sphinx_gallery.gen_gallery", ] -html_baseurl = ( - f"https://meta-pytorch.org/forge/{version_path}" # needed for sphinx-sitemap -) +html_baseurl = "https://meta-pytorch.org/forge/" # needed for sphinx-sitemap sitemap_locales = [None] sitemap_excludes = [ "search.html", @@ -78,7 +52,7 @@ def get_version_path(): "_templates", os.path.join(os.path.dirname(pytorch_sphinx_theme2.__file__), "templates"), ] -exclude_patterns = ["tutorials/index.rst"] +exclude_patterns = [] sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../src")) @@ -116,7 +90,7 @@ def get_version_path(): }, { "name": "PyPi", - "url": "https://pypi.org/project/torchforge/", + "url": "https://pypi.org/project/forge/", "icon": "fa-brands fa-python", }, ], @@ -135,21 +109,9 @@ def get_version_path(): "github_user": "meta-pytorch", "github_repo": "forge", "feedback_url": "https://github.com/meta-pytorch/forge", - "colab_branch": "gh-pages", "github_version": "main", "doc_path": "docs/source", - "has_sphinx_gallery": True, # Enable tutorial call-to-action links -} - -# For tutorial repository configuration -# Note: github_user and github_repo are combined in the template as "{{ github_user }}/{{ github_repo }}" -# So we keep github_user = "meta-pytorch" and github_repo = "forge" already set above -# and only need to ensure the branch settings are correct -tutorial_repo_config = { - "github_version": "main", # This maps to github_branch in the template - "colab_branch": "gh-pages", } -html_context.update(tutorial_repo_config) myst_enable_extensions = [ "colon_fence", @@ -157,17 +119,25 @@ def get_version_path(): "html_image", ] -# -- Sphinx Gallery configuration ------------------------------------------- -sphinx_gallery_conf = { - "examples_dirs": "tutorial_sources", # Path to examples directory - "gallery_dirs": "tutorials", # Path to generate gallery - "filename_pattern": ".*", # Include all files - "download_all_examples": False, - "first_notebook_cell": "%matplotlib inline", - "plot_gallery": "True", - "promote_jupyter_magic": True, - "backreferences_dir": None, - "write_computation_times": True, - "show_signature": False, +autodoc_default_options = { + "members": True, + "member-order": "bysource", + "special-members": "__init__", + "undoc-members": False, # Changed to False to prevent private members + "exclude-members": "__weakref__", } +# Skip private members (starting with _) by default +def autodoc_skip_member(app, what, name, obj, skip, options): + """Skip private members (starting with _) but keep special methods (__*).""" + # Skip private attributes and methods but keep special methods like __init__ + if name.startswith('_') and not name.startswith('__'): + return True + return skip + +def setup(app): + app.connect('autodoc-skip-member', autodoc_skip_member) + +# Autosummary settings +autosummary_generate = True +autosummary_imported_members = True diff --git a/src/forge/controller/actor.py b/src/forge/controller/actor.py index a899da6f..1bc63a0e 100644 --- a/src/forge/controller/actor.py +++ b/src/forge/controller/actor.py @@ -12,7 +12,7 @@ from monarch.actor import Actor, current_rank, current_size, endpoint -from forge.controller.provisioner import get_proc_mesh, stop_proc_mesh +from forge.controller.proc_mesh import get_proc_mesh, stop_proc_mesh from forge.types import ProcessConfig, ServiceConfig @@ -26,7 +26,6 @@ class ForgeActor(Actor): hosts: int | None = None with_gpus: bool = False num_replicas: int = 1 - mesh_name: str | None = None _extra_config: dict[str, Any] = {} def __init__(self, *args, **kwargs): @@ -59,7 +58,6 @@ def options( hosts: int | None = None, with_gpus: bool = False, num_replicas: int = 1, - mesh_name: str | None = None, **kwargs, ) -> Type[T]: """ @@ -69,34 +67,47 @@ def options( `.as_actor()` or `.as_service()`. Each call creates a separate subclass, so multiple different configurations can coexist without interfering with each other. - ---- Usage Examples ---- + Examples: - # Pre-configure a service with multiple replicas - service = await MyForgeActor.options(num_replicas=2, procs=2).as_service(...) - await service.shutdown() + * Pre-configure a service with multiple replicas: + + .. code-block:: python + + service = await MyForgeActor.options(num_replicas=2, procs=2).as_service(...) + await service.shutdown() - # Default usage without calling options - service = await MyForgeActor.as_service(...) - await service.shutdown() + * Default usage without calling options: - # Pre-configure a single actor - actor = await MyForgeActor.options(procs=1, hosts=1).as_actor(...) - await actor.shutdown() + .. code-block:: python + + service = await MyForgeActor.as_service(...) + await service.shutdown() - # Default usage without calling options - actor = await MyForgeActor.as_actor(...) - await actor.shutdown() + * Pre-configure a single actor + + .. code-block:: python + + actor = await MyForgeActor.options(procs=1, hosts=1).as_actor(...) + await actor.shutdown() + + * Default usage without calling options + + .. code-block:: python + + actor = await MyForgeActor.as_actor(...) + await actor.shutdown() """ attrs = { "procs": procs, + "hosts": hosts, "with_gpus": with_gpus, "num_replicas": num_replicas, - "mesh_name": mesh_name, "_extra_config": kwargs, } + return type(cls.__name__, (cls,), attrs) @classmethod @@ -119,12 +130,11 @@ async def as_service( "hosts": cls.hosts, "with_gpus": cls.with_gpus, "num_replicas": cls.num_replicas, - "mesh_name": cls.mesh_name, **cls._extra_config, # all extra fields } cfg = ServiceConfig(**cfg_kwargs) - logger.info("Spawning Service for %s", cls.__name__) + logger.info("Spawning Service Actor for %s", cls.__name__) service = Service(cfg, cls, actor_args, actor_kwargs) await service.__initialize__() return ServiceInterface(service, cls) @@ -144,6 +154,28 @@ async def setup(self): """ pass + @endpoint + async def set_env(self, addr: str, port: str): + """A temporary workaround to set master addr/port. + + TODO - issues/144. This should be done in proc_mesh creation. + The ideal path: + - Create a host mesh + - Grab a host from host mesh, from proc 0 spawn an actor that + gets addr/port + - Spawn procs on the HostMesh with addr/port, setting the + addr/port in bootstrap. + + We can't currently do this because HostMesh only supports single + proc_mesh creation at the moment. This will be possible once + we have "proper HostMesh support". + + """ + import os + + os.environ["MASTER_ADDR"] = addr + os.environ["MASTER_PORT"] = port + @classmethod async def launch(cls, *args, **kwargs) -> "ForgeActor": """Provisions and deploys a new actor. @@ -163,14 +195,17 @@ async def launch(cls, *args, **kwargs) -> "ForgeActor": procs=cls.procs, hosts=cls.hosts, with_gpus=cls.with_gpus, - mesh_name=cls.mesh_name, ) proc_mesh = await get_proc_mesh(process_config=cfg) actor_name = kwargs.pop("name", cls.__name__) - actor = proc_mesh.spawn(actor_name, cls, *args, **kwargs) + actor = await proc_mesh.spawn(actor_name, cls, *args, **kwargs) actor._proc_mesh = proc_mesh + + if hasattr(proc_mesh, "_hostname") and hasattr(proc_mesh, "_port"): + host, port = proc_mesh._hostname, proc_mesh._port + await actor.set_env.call(addr=host, port=port) await actor.setup.call() return actor diff --git a/src/forge/controller/service/service.py b/src/forge/controller/service/service.py index 0b655fb6..402b9a41 100644 --- a/src/forge/controller/service/service.py +++ b/src/forge/controller/service/service.py @@ -1159,3 +1159,14 @@ def _get_internal_state(self) -> dict: def __repr__(self): return f"Service(actor={self._actor_def.__name__})" + + +# Copy docstrings from Service to ServiceActor methods for Sphinx autodoc +# This ensures ServiceActor methods have complete docstrings while avoiding duplication +ServiceActor.call.__doc__ = Service._call.__doc__ +ServiceActor.call_all.__doc__ = Service.call_all.__doc__ +ServiceActor.start_session.__doc__ = Service.start_session.__doc__ +ServiceActor.get_metrics.__doc__ = Service.get_metrics.__doc__ +ServiceActor.get_metrics_summary.__doc__ = Service.get_metrics_summary.__doc__ +ServiceActor.terminate_session.__doc__ = Service.terminate_session.__doc__ +ServiceActor.stop.__doc__ = Service.stop.__doc__ From 23dda071f38886d8cd68048a4905d457e1cabd99 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 11:48:55 -0700 Subject: [PATCH 04/26] Update --- .gitignore | 2 +- docs/source/conf.py | 77 +- ...r_rl_fundamentals_forge_tutorial_thumb.png | Bin 28056 -> 0 bytes .../sphx_glr_template_tutorial_thumb.png | Bin 28056 -> 0 bytes docs/source/tutorials/index.rst | 63 -- ...l_fundamentals_forge_tutorial.codeobj.json | 215 ----- .../rl_fundamentals_forge_tutorial.ipynb | 158 ---- .../rl_fundamentals_forge_tutorial.py | 558 ------------- .../rl_fundamentals_forge_tutorial.py.md5 | 1 - .../rl_fundamentals_forge_tutorial.rst | 788 ------------------ .../rl_fundamentals_forge_tutorial.zip | Bin 40960 -> 0 bytes docs/source/tutorials/sg_execution_times.rst | 40 - .../tutorials/template_tutorial.codeobj.json | 27 - docs/source/tutorials/template_tutorial.ipynb | 75 -- docs/source/tutorials/template_tutorial.py | 91 -- .../source/tutorials/template_tutorial.py.md5 | 1 - docs/source/tutorials/template_tutorial.rst | 152 ---- docs/source/tutorials/template_tutorial.zip | Bin 5757 -> 0 bytes src/forge/controller/actor.py | 69 +- src/forge/controller/service/service.py | 4 + 20 files changed, 100 insertions(+), 2221 deletions(-) delete mode 100644 docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png delete mode 100644 docs/source/tutorials/images/thumb/sphx_glr_template_tutorial_thumb.png delete mode 100644 docs/source/tutorials/index.rst delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.py delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.rst delete mode 100644 docs/source/tutorials/rl_fundamentals_forge_tutorial.zip delete mode 100644 docs/source/tutorials/sg_execution_times.rst delete mode 100644 docs/source/tutorials/template_tutorial.codeobj.json delete mode 100644 docs/source/tutorials/template_tutorial.ipynb delete mode 100644 docs/source/tutorials/template_tutorial.py delete mode 100644 docs/source/tutorials/template_tutorial.py.md5 delete mode 100644 docs/source/tutorials/template_tutorial.rst delete mode 100644 docs/source/tutorials/template_tutorial.zip diff --git a/.gitignore b/.gitignore index 41306648..c952405d 100644 --- a/.gitignore +++ b/.gitignore @@ -153,7 +153,7 @@ docs/source/generated_examples/ docs/source/gen_modules/ docs/source/generated/ docs/source/sg_execution_times.rst -docs/source/tutorials +docs/source/tutorials/* # pytorch-sphinx-theme gets installed here docs/src diff --git a/docs/source/conf.py b/docs/source/conf.py index aa73efa6..77ee1338 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -17,10 +17,33 @@ import pytorch_sphinx_theme2 # Add the source directory to Python path so modules can be imported -sys.path.insert(0, os.path.abspath("../../src")) +sys.path.insert(0, os.path.abspath("../../src/forge")) + + +# Determine the version path for deployment +def get_version_path(): + """Get the version path based on environment variables or git context.""" + # Check if we're in CI/CD and get the target folder + github_ref = os.environ.get("GITHUB_REF", "") + + # Convert refs/tags/v1.12.0rc3 into 1.12. + # Matches the logic in .github/workflows/docs.yml + if github_ref.startswith("refs/tags/v"): + import re + + match = re.match(r"^refs/tags/v([0-9]+\.[0-9]+)\..*", github_ref) + if match: + return match.group(1) + "/" + + # Default to main for main branch or local development + return "main/" + + +# Set base URL based on deployment context +version_path = get_version_path() project = "torchforge" -copyright = "2025, PyTorch Contributors" +copyright = "" author = "PyTorch Contributors" release = "0.1" @@ -38,9 +61,12 @@ "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", + "sphinx_gallery.gen_gallery", ] -html_baseurl = "https://meta-pytorch.org/forge/" # needed for sphinx-sitemap +html_baseurl = ( + f"https://meta-pytorch.org/forge/{version_path}" # needed for sphinx-sitemap +) sitemap_locales = [None] sitemap_excludes = [ "search.html", @@ -52,7 +78,7 @@ "_templates", os.path.join(os.path.dirname(pytorch_sphinx_theme2.__file__), "templates"), ] -exclude_patterns = [] +exclude_patterns = ["tutorials/index.rst", "tutorials/template_tutorial.rst"] sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../src")) @@ -90,7 +116,7 @@ }, { "name": "PyPi", - "url": "https://pypi.org/project/forge/", + "url": "https://pypi.org/project/torchforge/", "icon": "fa-brands fa-python", }, ], @@ -109,9 +135,21 @@ "github_user": "meta-pytorch", "github_repo": "forge", "feedback_url": "https://github.com/meta-pytorch/forge", + "colab_branch": "gh-pages", "github_version": "main", "doc_path": "docs/source", + "has_sphinx_gallery": True, # Enable tutorial call-to-action links +} + +# For tutorial repository configuration +# Note: github_user and github_repo are combined in the template as "{{ github_user }}/{{ github_repo }}" +# So we keep github_user = "meta-pytorch" and github_repo = "forge" already set above +# and only need to ensure the branch settings are correct +tutorial_repo_config = { + "github_version": "main", # This maps to github_branch in the template + "colab_branch": "gh-pages", } +html_context.update(tutorial_repo_config) myst_enable_extensions = [ "colon_fence", @@ -123,21 +161,22 @@ "members": True, "member-order": "bysource", "special-members": "__init__", - "undoc-members": False, # Changed to False to prevent private members + "undoc-members": True, "exclude-members": "__weakref__", } -# Skip private members (starting with _) by default -def autodoc_skip_member(app, what, name, obj, skip, options): - """Skip private members (starting with _) but keep special methods (__*).""" - # Skip private attributes and methods but keep special methods like __init__ - if name.startswith('_') and not name.startswith('__'): - return True - return skip - -def setup(app): - app.connect('autodoc-skip-member', autodoc_skip_member) -# Autosummary settings -autosummary_generate = True -autosummary_imported_members = True +# -- Sphinx Gallery configuration ------------------------------------------- +sphinx_gallery_conf = { + "examples_dirs": "tutorial_sources", # Path to examples directory + "gallery_dirs": "tutorials", # Path to generate gallery + "filename_pattern": ".*", # Include all files + "download_all_examples": False, + "first_notebook_cell": "%matplotlib inline", + "plot_gallery": "True", + "promote_jupyter_magic": True, + "backreferences_dir": None, + "write_computation_times": True, + "show_signature": False, + "write_computation_times": False +} diff --git a/docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png b/docs/source/tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png deleted file mode 100644 index 3b362cf91299659eeec19a203a6d16971bc138fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28056 zcmd3NV|OM^w03Me6XT9;+qP}nwr$(CHNnKTZF@4&o9C?a>HLECL$B3c-PPTDS8dc) z`-)VM6NiWS1p@>G1TQHeq67p4obYpCK|%bqIO!C40s-0TONt1pc;sAvhC7?SrS3xe z&Axr!_+spkE_6>!Jcwf0Cd;~6Z+2TsGS?<%Vq2z=gV_|ndMi{&(Kj-o? zPygpu1jJ7d5n>$T|Lq&`fBXLb@W368x0XpbDv%2>6i^c)GLrLs=)ta4KDdevk;s+x ztHBE77wzS@At21R{*|b=-tpF&^kr(NZ1|=cQlGgA6*^$8ml(<7y9-re3$gbdRLbBz zX(3o!E+}Ff;q`Y3QtxYL$Mlu!^;g3xogJxn%olIc@cYqun9B3-5#lBs4$wV$F3k+U zX%%sWS%MRO$uE^lU>(>ftpGJv3urKNh)}4JZ2_=CBf&;?Zc0iJR3-tgyzML!En^81 zfE|ZeaHcb&N+u)po+c0hxB~W*WCTyrBak1^9v7)TuoH43jF6tO1bo&qQ6b&0s;LwS zB<-hlH)u0Jz<@$)tne;WLI8zv2Sx-=FJe2y1AK#Za57tp+TI%I%_5K=rJYf}-NDFa zs7Vl_oqN;n`!&1SY)_ZHi4)E9;UE`Hm9asdLYbrh85#@3@-I{rMOQP9D7+$KH18Rs zb_t=V7>&)UE|>>rm@MJES?&=Z=E)P^i;dY8>Y?{bpPnO~;K0NA03L9O4yfrbkggYm zle+D&Ef~Q`;}Dx5JlP-*kO5MaQ81NID;Q#PwW8B#Fm)T|4*ydmC!`HRJ&1W8krP25 z^4SN`*>R>E?ccSpb_WB@&}4kspf}7w@$mmxMA+n52<6#n1#ZCZLhTZ`J1qbdvNWem z-(OCr!-eM?7U>L~38#l5Dv;+$&|#UD87kp|6~ZktaOG8P^-F`Wg1eyS{9%Lvt<@v9 zljP_vxGHKAmYuMADY&uB)w+6Tb7LYS?%_(WazY&G99oVir9jqs(GDZjKzevjlAFd` z5O=hRA8e#4nI&XNh>c-cCb8OEx>d!^9>gvf2wWFjV&*X!;($3Qv>_RmnkXWQRGUbP7-2yy5M}HrFc@Yme3))|ngY$WxNc9_Zu^+`+uR*8QVJpg!&eCe zG0YAv$>)K{Xl7q%r~B<8_uP8#A2>b=e-4T;2baoyd}zL9=>@{&wm=@B6N_) zBX)Cg%(JKFn+G$sZsdToKB=Jb5XUv*=hej)xDStyy5l&U z1){Nx;c_4pcGH;K**k!s)1%RKO|d*LakW;R^gMhsNOMi}GiP*}Z1*YE@_@T3k^Z2u zz=^?2nT~$qly?xZmPkoX)J?qID}yMY5)^z;NqG7f>hMz)Kt!h1jHa zU!$*;g|$Sw95E$K{eH_Dz%y_QZ#!)fWfOW5$YT<@+o_d%;cm`#;Kk%zfp_Y_tH=Z2 z{XNsg<4D3B2-615AC0yiW%rhBCa_i)gph>>^vMgkAQ;Fbe64QLacSD_okgazAj zh}mS7(;@*_anFTvEk0*jJy;za^@IOiBFiu9jXwG&>4NPKKFJAoFE8E15^l>9;x2~4 z1)4_-`6Hl0o^V{b(3ifC?jSR)Cfm5a)vG?$l=)hTleSOzEqIk6mCt5{t-9UGVR=_=_GZ>m%bG$hwa#)~lH z0gAp2p=ime?$b^OV$VtP4ff6Vxwm4B-a1C3sRAvH***f@I_P^P4_F8i%~Gpa!68_0 zXVUTMDHCsZPUwshYBfK*axOR9QC$g*soPddFSSTnhOyBz<2g_UcnNeOib{aZxtZJH zYojLCTVWu*2Csrl$+oYj#ZY>fWS z66kiXPUz7$>chxu0xP2MkfSBN*+M34fw&xG(7QZLTi@Ln72!!{0AIfulD2hvB^KQ7 zNwU-5@i#|YbmAMaohR+?hia|)Q5OTv+=PLeg9g%za{usQP(D%&m<2iae1@?^- z6ffX1T8+)T-q^Pc{pb2y0iLiXLW_m~6NkvB`0D(@nxr&8kB-?)j!ZuvfWp!^R^9Z6_@04F6vPEHA%M z`k>EyqUTsW6Nkpq)LWZBcrvU-K8irZ1z8Sqq`i*YdE!J=zy5gfe z>Jz{&r@guwMQOLvt0_USM3#YgRUCPs&%`mhJDPG_`4xWSbdDkF;YuUUs(cvE_N>B5 zfOV?Ev9uo@wlKoSnD+P4n!d-_+d+fe(91;!(tGnZsIPOa=MOQ+wS zNVo3LE3imMv67n+ZwS?2y+WYo%cFkm35?(0Yn`)8$MJ{j%65=kzxn^3mNtKQxqvqF zRSZQCwj^L&E~Ll|Tjv!8zS6?Cu_-a&vYB(pgfd6Fsn@yQGgU$SX7V}i|8VqzNvtLD zA`Bkdd0+?6uyr|pZnnPew3fxjrhdE+Irk*~WqyiMt=;4>OQ%QRwY6Gr1P*eQL)l;E zxq@frgxYCgSx3vx^!q~r#cUK+M=GN_&lGJBtveVf5rRcu%tAMPB{r>2r#6u~U!kf8 zX0XMXFOZpNtAUy~j$riPu7q0Y$W_8P#TvH-p-cL%sPhfniz>f>@2);bq-4IvqRG<2 z21F4g69>)NU42^jKH^O6$+`|VKemd#Q=H!fBLOWb**_y!?;Dh7X}?=*t7bv;>LRb$ zOgoj_Zmv`(Z~y6hEXgzDfjVe&R-~raQs>KeiybS=J|^KLJcT<$Bzf+l7JsLgnEQ_W zgFKD+HDpM1=9Ya|@qf7i51Q;=PWb0X;(JN{P-e<6u{MnJpxC%eGme+D^mKilK>m9Q z)BSsyG?spIoa$64(<}6in4R{HSFEzyL!_~aP{j%|Gw!C2oz<(k!n^?g-lN8i{`6|N z47iEKC4)A&2t0pGL9R2pS*gPZ+$`tP!AFNL@X@+0YF3~qkvDv1+{-7g+<&S*ak@xM z@<-r}2T3nm@b352`XkNX9S|}tkiIqEp%wxVkjGTvV&T&CuH0T&GP9IVNT?H+OoWL$ zMT%30t&Nd8%J_p8LjRC~P}?_*l%IBLygF5!eOx(QXQTEA7!EBTB>H0JK8=s60VdOn z3!1p)J&(jbmN0umU{tSS6E_bcqshcZrh7DOShDvq(hKPd{wHKjFO(kG@u;9};T8gI znRycNN?QMaeC)uISq4|RM*AmZgaKA>uJsR zYuj(f+`qcx7o3xblS-YIl&9)4v3wJO*5g6gfPigT;@Ujmqxj` zV{=Lu(RY9yKK|?X8oKS-uip54i9`c)H=FF2e=Ogz~rB^cM z?4=RtE14^hZ)l-Y?=u}<#Qk)|TrTjD(fRCnrsJIRfcr97oEwarT@|E?5(n}<+>&k| z*y&CGndM4Dg@2X}(Sy<2KOtl@24uQbM(_K}=iQnXI-idF%W~EQscHvKT!#+7SZPv7 zi)^mKYYsVnndfDh2Tyb22*=;U_&VC(w(8?gVc`VwPA9|>M$<{Y;Y-c|!Ut zBPr&p#~rTmG9?b32ha9f=RVgY?@9Ex(nmVZ?R)B!HqIV$yoWQ;{vx47#jMnRVE7BF`+ zRv3Nl7^LQZ9L3kZQ&Mwornbu;bJCkLz=<*(?wEfIyBtBV`zu*N)s=h?%uAxn+F^SgQYYg4zM%%O_@yFPR zf>Ol>*k@QDo2hj_afA5;=O`&U#(YI?jSp3|P!y(Glv7e1BEWJZQ|w|GchA6h71t>@ zO9+$C+W7JRD^j>%w0eHg3$#RCOrdcw(z7&; zi-0!8nzBo4YWSKLwsI-TNwJ=mM=0LX1~gt=3^m|`kS2EmiQ&%gCoRScI~udX_ML2@ zk78Z0V?PVP=jH@k@Kl*(3Nr3(G=&XK>zWyj6;TkU(&!FG_71eCD zh>$CsciIZZL0iz8ZQHB+yx%U4!Uz3_o|&O}B%B!OjjZJDc`iq{cy0}minmzmUFX1N zDUY>fczX!L5OiX1(HNzRv;D>gTjvubX`&qHLy6{mo5xMc%aPsLwKVzji$KrzYqPYD zjfPKNr1&K^<9&@-M!JT5dWKwPHVxO;sx5Ugf0pgbgBDR=E%Er814iPMcgyo}_v`@Y zR`{OAXCx(fw5F(N^|BFOym7U`+3Iv9B{~u@>8QBrVaDC@jrto_=&u?-G159|5%9X# zpcdwKwry&G7Wo2WP!8a{p8JvQ9mJ9T3ZRak+=iuAhuD_o1*-13%0szCryg)r ztlYPu_XQnMHqf7CQ$^GPoTUNdON-5mGyYA~UJ5Eo?Rehg<7PqiK02-5DNvAtIwTlv zTjD0s9qp9H#2JnF2s*pzaksmON}C+RYxiU*@M9Ro0ruv(T~#Rs71V`SYmJd)Vr|ks z^QCA<384KAOYqZ?OTL{q4Ceu-0kUa+nCCDKVXXaeHpd8pasvQzg{!bkN{u4^UkSBk zrvA-kICy%|Y|4UVh4ImNWSu@knb%TE+^y>-AR>5 zEjBn|b5z%8T;ud`!!kRNw5Ztq^fFQY2_Mml%A4S^p2UqMdOiN_na>EOr(?(qK4B`0sjk3IkSe1wCNg6cbAaCg2m9+jTP41o~3!0|D=aL@g3dl z<0||Ey`AvwmPGqaK&#!Xz~|T8P(ap$z%3TrF?}RYRNpVb+kUH4f}tX=@uW>MqP}mo zM6CE-BauV_*N8})^z1+H?hVb$&|2na^J;S!>UQk?-fo@GqaZoocm0BCiR^t*kN$V% z8y&r1$0ReIlufYsXr?(iA)ZtQ5<2Gu zMi~-{K&WuF29?DAHo0dmg}w3x26sqo=?`^3D1`|$gfCKe{LeuAV#e*F%PkjEVZa9J z<#D5rF%=7mpaSC~6qL%Wz>WhvRW;_RR4&8GOS80SHal8lfLFrTIbFi4P5Hw`Jxfwo z^%inBMmsnEh*Yg<|E8fYen4koUO>@&@l}BKOOQTW11lEgUNzW_JPpFjobyUV%L z#T{m3_$tGa6b+|qZT{%*RqaZqyyf$7NlI<8`wysDzn65Qsp=LFSeTPhRl zfjqAy68Q{$SPw8nGDFKCHVY+1HiO!1P-J@7cuK=UXa!cczbypVGXFD|Y9VqVyRHQ_ ze^b^xGFH6RG(p9sqo1*%nZ;YGsuRonRIz*168Rf>BD$87PLu6G2TiP^O4XA7Z#P=q zvj8o<5DI9`d#+`oNOku=7v!X}rl?JuvP}IJb441#Qyo_olBwhJH!tS$un2(pKnaH8 zAyOHW8$LqQ4Ya{PGRWxHllc=HM-&Pds;Hxgzcyldfw`lN1~3_Bs33zbT8v*sd0h6og}3Z!S~FsD7((ftexqs)?{2y9U&mCmNq*{fUS3+2x2 zru;X`QOE z&}uU_5{A=~mw+u7lfxJU0x((Kn(p}Ev71A>@*o_uwy@zdk$f*yNHbI{0tqtR()H*mJ1ynrhBa;ID0lRu~PAull9rmNQ(s_04M z-#^%zewR^FcC5OtAb?G`y~J>%>|mrQK^og-A8b-sgULVn7Vl*jk>>0~UU*=LsBwxuY-q?0$pW@Su|RjA{i}uWcTJ+>zqE9OVc@yx zbx4viwg~u)DL}4KJ@(NSLcAZh4p9@dAfoIpN2IhXwPO-}v@YP{bd*^U-=d;S>!2aR$!q<|b!Wqbt%BvBSb-BV6tvUyQG>&pw zB+2Oe-yC^mC3UHST;WBu|K?u9*dFDMdY0fIQ=_&Z7iY{y;m?LzgqWXmq-l@#ED^BZ zqru3wCD{0lNr$Y=JMPldWy1=+!m4#}a2t@f2=?0ZRkQE*)-Q(44_og!7(lQj^x)<$tGY@WxSlg)2v|NJZ)qs$ssMmBfeLY5G|0q*(`jeQlKo;> zCXT48KL;PDEANKQ7phw6Ifv2?ewK6L?b`n>)Exxcr{1plbM(8{5*^S(HcfnqdNa=z zS51J>=7qIAIwJTy1Iwo9R1?L$2#tv7Vf6)TQ6@2&MgBSK87VlupQnysiuFi~Jc4x+ z#b&1$tgD`wxXz9K&%56#(j()(_4Bj++=@~%;Qq1#pU5Y=v%+&0WJqF&(|V5o6kx1mu+wix?=ly->K84?UVr*0&fuzYW@suL2^k6bdQ*~aM~4Zaeyb_l!<#U-yd{>ZTV$x?C&9)Ow>+>1^@9+w zoEz=jj3lbL9$iUiVn$4l=F=XQl$la^t7q>Bc|&?b}9m~<@4Dkv}gPvbQ0^2%W`tJY@|7gdoU`P zHfw0w#@biZYb!ALAkfG^iFJf++Hi65nL@fYcf@ojJ56*jE^Igw@ySgUDMiReb@mg$_b!9kiHJyNRPx| zQ=Z6qgT`d@(`QdfXQAjDaiFnJt@UV`&}1l2prnVIUS}ibLR@}edq3H&tOzI*#~3po zcV5?Fxs}%o+*5C-SiAO%pO(mqx>llqj}iYKySfb+Bqy*)8vVL4CJYqveXcPth}gcc695VCqd_mj6IzXg2f5xdd}| z&4+q=)?=zbGf`79r&`p;OYHm5SeP0O{NV?p zMA{%W2M-UV9jamb zdT6#+38Bc1@l9^ZT5+;)`0^)_{G*2xpa-=0ULkj^lF0{_A5*}OPt@f56D655Anq#n zmvX6)>u6jb>dlKV3oJGQa*Ar$y@<0XK1&xc`cOnM0**^@+ffgW{WLDfhnAj}I%_?R z5$4qD2}&e_hN)^~Pi{W#r@5%qS{%q-aCR4aK6URTGJW5geSeE&D-x`W9aOb2XMo*l zjlZ3aO7+3JVoIDdLA9%tnv$#->5ruuVn5|T;ihJtJlxlQt8YzGYpFqiKV>tBwR$USyaFNzEjFmP1ig&EUDzMPq*8;+ZEwJWTZ3 z3_G|d@I8Em+d!IlV~Bl<@(07#^ecM#JQ(%yFTtGzmz=G+`&lCFJP7YeUl9lymOXA% zr{$Ik<}U z4-`pXt>Cj@&xnOgo(5cxhoWmP7;T@tFSo(lX6pxpabwd=EzHN2e!Di!L``i^SrcweJKh!s zdnxe}Pe< z1NzI?;ew1xFsYEqbZO*En20Czv*Y{y@8hY@iYbr5|FzqBy&a>Ofm^E#1sPYxTqg-b z(iM6{<5`g4c2U@I#^#Q&HTXboYzTf{g-2!B4{%BH4#ZxPPNi^cWEWZU=g zvORrQcsv)Z1)-F5w&^o&I!)y8o}*^=QUQ8}bw_pXR{%KTRQXga`HWxkC5UOZ?Jlrk z=GYP_si8KqG2Q2V<2yUp;*vem6AY9+=zguvwyFNkxAWf9El|S`um<^q8Yl;u`#k++ z_)_Do)#qtm@HMa7)U5E_eHai|HO&dAihR}5_!9wfJgV$6pU{a#B+w`{=SkkglU6p} zbUFpboOcGEDLk_rce+p&go&l0AQTx<)hiV4f#-*dcocqRSU>3#N~lh1?H?)8A78jw zTH<%cA4u&_#zW5{l=f_$9RG#wau!=R`3`x0r zA9vT*&86-)>1R;9$9w;&2D-qRYijEFupOn-NG&K4H`tvj094}CH(NMLrfF9Q1Tq-eH>1X}6~`2N`CVQcd# zuk{pvvSMD90y53DGbcbJnNlH>&ezcG!W{7tf8v31m%`6)JWs&! zQM}Zu5AX${`-ZeQ#cgpB+FR8a?rSeEm)g0cy3GwX77)?w1 z*J!vDj}NA*2mW|3NZ@@m|KGnbUEqGrd|ACcq8@*IF3Z>8_1wWv_Q%I26hJQ=~wk zo{xBxtZ!Z&)FSp~s#(q^5B%bRZVLNdP0hK3J?#?ryP4pN{f=oYU;Oe#yW)mlRHYAH zXic69-6>B7$ss=Xp&Q*kG^@^8bQ?sZJ7A_V%stm-2W<6JaXnws(+Q^U0h^E$<^(mE z?`7=p$1)bB2M1uMJctbSD?QE|Sce4F(u4G1HT7T^5(RLtTsDlS$ObFide?W>TSi-2 zuIST*@IkGqoI(ZJ5tjGzaq42odZ2GX$pk$+;v!eWTDxsz9WaS5&^e%rc%S*YtCv?E zJ!EZZ2t#B=Vnv|=O?9uzbb=1gO#a5R=kp^${)c`MsvLlD|B%7&nPEs!Z;2)+J9kk z?cB-|$1%$%{BAlMUCg!~iNrs04vh%s{~&8ZTxO4Lu9`j{m}>T}0q)3IDZgXr(TtTaou);*JI~GFL(s$h)pnA?0doY$YOgvlB${W!>HBw7ElII zf#UvnQpsa_TbZ|b8*aABmQP90avoInuDJQPB$zr2OL7{jsl#2VT_DYyb{&?9QbpY_HzLl7Ijm*DLFg-t4P$>H)^=c6fYcPBBU`cN`F#y`Uk~|pgLX3I5NA= z^Q*&nZLq&Hcsd#Wm5Ao=x@(7`1x^F%!!FoO7O=mqenVw8eO;A~L$uX&&Z6!gqw|$R^DkRJ zz?9TVUvm0eg$ar+N7ubUKVP*M{DLbQ@(`-!1jd`1KUdT~y`@HTygpGEyR_E?Y z&uWB+(F~7#kRcHju{-6Vs*&auUcmhuXFA(TvU(ZB) zrzY0$H7}yTJkmjANM4}Q&nYg+J;Tryp|MGf-G#5XPdiUD z|8dU(MLXWyCPQJTS&5qdn}5>i&1_K?v%dA!P=wiF71jz{$jjCsy@FEBR9=T^ciMdW zrSeOk^>V36`iFX{@sepayg4}s4@X2L&-_D9f6KQ==%+g`=1)_o$y25cx_w1BY9=5B zSWtv2Kn|jn9rDNh7(|z6b++H%*=Ka$9r{b#MEq|iGaN6K zjYn_F%k#&$ULCoi=eR6cm86rocfYOb>QNPpm!Sbm!B(jS=;&1pmYRa-R3e~>C^bAC z?4^-4v)!xAEA;3q-q!n|+#7y?g<$-Ew2yErbwge-lSgi@l^8Q8QZT#7 zvnC3RYG^~^{rRL*!}=}Rj6rGy>mk!sTw<y_0u+@Am=eMHjM+i{_jdKw zuZmn`ysA6FQ-(hXhonrI3Hcx<6R7gY^Bm0xs1;U)bvu()o>ZY5it~7aFY60b6KxIX z5T)*$nWl+f@N3R2Fy6{n{@31R)HZA-ltHEdU8RwvD!RyvB=|s-M1Wy?!`8GN&CvFn zb@A7+v2f|PZRDFK88h8jMu#MZ!)cMs(?Gz%4K-3Humz;m_PN!07niL4x|$p#7o)%A)H2Cu{8P)HLwZGo4KWvk2hoX zIVb-b{fGNL7=c|gJ3&v#LO?n=+$I@E;4kRN_4azBX)93c=&>$up_%Jdc@V$JM@N%d z#n?%w7L>-Bk37??<@OSSyEW&Ibt+S<^(I4u;1!J;P*Lg$#jQ~A9jNYxCyW3!X}s6i zDB3+WO&U7LK~1Na(K-!1G!nKN;`k!M2N_h*K0;_2oz~mna>9fPcRQw%#GDVL4hLrz zk&~vUHf3*pkl#yz)dvW{o%(hE1a|lwSni{nt=sd=8Ehk=+dJ!fP{J}_Z2k)?m z?g6W$awMiI)oFz*b*hV!IX0u-JFublzT8sss{>@Iqt+!#UGzLcc}juROF=UO8UpK0 zV6)d4+%M4^mfQ^%wvf#;iNTh)HbKgX^ieIr_OIUNkN)mUJ!D1|E9Y4bC&r2i{s3WM z*G$Uczh&Ql36^ENg_hH}Jn#{oMoR&yQtL<*Dg{Z5&v24(B+dnxuy=NU;IYj1<*}b` z9$W}3$mNEoa{MhY=|n{&Vy3}o!~IjH##AloADGar1d#+6wS{adNUFRrWxX<0y7=5$ zyr{(Ym#X|(x!?&(hY8F9=SoZTETIF|4A$Be7$5Qu1&EnAkyq-65yc;4o2Y-{Hxi@m zu86;-{(vTVpxT28UC9Y2a|AX5t5rdT>H2%! zD9wpX6Mrv>#*r?ZP=AK2S~B5kgh-u04rNm>J~(i(FXI7#5)l8t^g$jmyS*kN?(x*x@FB4UB zCg)L?!(>lvNP0)`ymY|wlVwdibfb{q`G`0XG7?|Hq{3O}>CZigmk`juA3wwha@7ksMzl{&`THm{g;v}DsTUY)X zt!gW`$CNX!j%Bi`o#f`m%Z}(!e}0RKYUA#py(#>BX&uG}xf^az8ptocM;+`m!?6^0$~uo7jStBQ(PR8k zrQ*oVUs*n-y}Spl zppAOQp4eVb;B>~@@MchFCCgfPH6@9tUXofxrpoxq0^7#}xAb0>b~S>6lJef8@Kr@& zY-Z{N}p)1fkVw{u}iZ8b>=%gRj|fy&vF5|@sBVMw{oZ9dop2pazisW*kzhT(b@)8X;g--kIwyy!Xj%>o$6?k zLw9&dQ?%HOy_busXNMIfhi-wI{xW(B3Qq zEm0`zKnvAOH4RhrUuyH5?(np`Kh?IH&^q6qNXt=mm(P{%yjutxjMm?`bX)CyaYc6& zOust`rr7)be)sk{eWsU%A*v+E25cjlzA<>}n-3z1glANinMq_QfdEe0DvkQ5orokZ z2109~8G9-oeRX|t;PO!`7E!KiOpUaRsH-$r%Sfpjec3vF7CWslnpGvzGPVYF=vJ$a zy7KX!t&^8c{se}NH(I);Mmulu@Af>-xSHTZ;7c6Ai1Y!C+Y50S&oLmt4-v03kdm+1 zDK{-&7P_8-oKDj-h0(4f$( zy33HPFUw`oXks9&F^e?6FERKo2Qk=i8~xuE`e(K<6=!k!7^Kb3p|akTEopRm<$?rBXus;+iH6mXz* zVpXhWRVa$0s?;(KoJ5h402BiVPT;W+YE&iJE9PHX&}4 ztc;NmL9kLI8e0UaliHSuhAc?k=|Ik)w%7EJDUV2bHQe}(Eo1Qovq7oxP5^PIvZ?)h zd_IVQ^9+f1$PYWpRfMjv^{L+R()u=FVpNe54{{#(sEMFNxri;9nCCWwKdeMewTP*} zG(hSMo`)&FC}R9oui?8=!H~*{WJIOnfcs;p7~+;-dLHquUf%f{ad$;1cI#-D744sf zlrAL(=Q+xREm6yyo{pgP8^W4C*xpyB<$;Ng=3I17l5w9pct#^4i$W}LE#@yoo5Ob$Is!s58M7S;8Uyyk65IKykaD=vMJoexl#IzMFU z$JV`MOvz^Kaqhzs!=#9^mh{broY3}4&)!V@I~7t1ufiK66RXA@o^gkM0{3j*^~crQ zrDyN9?^L6eP|1Pd3h5GCRZA3?$W+Xcm@sL3>O0?4X@Ua3gGGt3@At2X1Bd@B3iuW$wXO`RFluZ(;r!>kUru z<}LT7qyrs^!+C=bs;5?g~`tv*Exkxs+VsHe7-5OlOHX_0{?3e#>WgZ+v&% zAH-PZT`2jNkM80=?;-6~V*?DD$B~{PkU))-kU2`R-LMycNwqC>Eiz^otAnNF#TR$M z1EpCqojEn|j!Mv;l|p9j?O3S08n<0vCgLhy80zRE^gJ_ME>VGy$<4?=kCgYX8Slq~ zX#ZEa_we1*^ROA$ZxijmQ!~uJh0=W?!;qV%@Nd-zg}6sYRHucxi|6S~z=&Q)!x}Um zeP$RV$(1ul{YEps=KN($#DC(1Zmf==`Ht0X3Kh-gq z0MYDGu+%7t_c*pXI>UV#P{~eeK3c(T#{_<`GGsEU#P&f-@2VZ`1{=kJnd88}{n16GOYJ;P zH=}c_0e{LbDE)~|`3ej6Nh#5bM`>#v=tW5R|GgSQ-Z#JtCswrYl{Is|nxRQoVI?OJ zC>0=!*wdSJtX&`dzvj-lq0;t?_u1IluE{ng+um7|ZP!kdZM!DdWY;v+WKK=CZR70c zcV58x_q>5y*LAOTt+l?NwPIpO98}T*Ry71fbcef}K1lzNKvkjWOflC7Pci98U$_dKtFRM+BS?Q=T^_p8Z=(acaG+#I@qn% z^j;-~caKH>J@FT=yW(X+Xcs||~1ag19YqG1z`#4INpa!{W6Y74{ey*-H?E?6 zLxeMGCgE>#qnM61nlzNzlW6irrGR_4gpw1n!!WaVyIo74A* z3@^Ve%KdD=gH(yl_$4wf5}XJKI)meSWzn;{40~wNpE;Y)j-LeFB1@V#VriTi__i^% zUfR^qihFG*KkU*k*6*;kN}X2#@^(&UW@~=(75u$9i3nhX>F3oPMvLE%Anr<0^_32$ zH#@1#^NYN1+%oz}8XUmhns?)9ghN(Xq3Yt5V2b_Jd~)?t)<95H<0gGN`4pz!b`30Mp^Y(v7Gr zO_&_^4Pyh=-`;sdI|=$e?(LtdO3(c8#DsU((xgh@MhDEMQBBYF-|j+Wu0 zvMFW&p@vB{lVnR!7dE7HPvz?4|SPS>dPr{ zfDoZ;Gr^$W_Hb3;ShE=!rL@wAd$U%0(xhAmv=m%ElO<|;mG@?(QKXH3M19er5&>&W z^LeNziUJNcVA{}9I=T18)Xehl@x%Bs2Vm{FZ$qG4c zPfbA0B1VrtRksYN{x}ur+Sb>?zR8Q<2?hPc=kez=%eE|!pJV*A__A*7afe*$`aU54 z@ZhTZxhFXp;5tYFuH-9X^ZDr|Ho%Rt%319TW%Clwb!&tlL_|Fkcn0QX85Y$L;?>D$ zgsCZ}hyE$#2wA)@n>qtdQJA;+;T}X8**df)8NQsHpVY}}#9i-~ z2UcPCk3=FL{6`+aMM3e!Wg!F47edh#Akp%x;R48`RC*d11xo3_|3S1Y23CCRCY49F z^6U2iaHkE+*?4OMpYS^d%iovTe&pYQd9ef)cI1*DzaSAkZpz++lfO|vKLzwWC%mes z0rpfUo8rPqG@3-Gk`BIT64j6HI@fddjl`8wyJ}|HDy%!7D|h)&(XfzJmE)UAd^?Bgbc>W+w3jo4qdFNwKs3l6synUMC(;}pq5w^zn9r#HA zSa3zlZr(tjGvo2+Iw9KG=vnJ3s7z(cW8e(%%>pRV=88HEq zRa1jwl^>Y8r(Ye6`s9-HwA+CYUnTb>Te^jPa`jAHA8gOb@{NFGaAv05ORl!bx-mZ?=~mudysN zRkHx6%xdYHD+)DWMFTTZU(tnBiQ5yqO3A$XlFF$M(U_;FJRvjrOd{W3JIE_r?A9}p zU50CDmCY)k-js7&0)Z!F%et;{360OMBWsl<2`Ig%i1gK?g-DcPUF_Ht&m zl^fUf9t9^EbU9`9L<{!$SXDO#obxsz>14Ug_|I&zM{?IXe*}Wm-~&vX&$#iruh+jH zWEfWg`?xgrKVu1ge6v_IiG*CulWuK~es*f5cDL$(qZE}P?uuUE#obJ>o45R}9gRFc zc*0b_y$*84DB?s`%c=QbOAl-SACj4C*2lD%Zq>@#DONFSW@mgZ3AjcJ56SsFT5#Ub z(*+esFcd0ecG<~yQ2R;}5y?S_E#EY%K462;FpKo?bV~pBxvf^nUx3TCsD2(Q5O>(-|ZSO`UWMoZrZu=2970A zBO_B&{>Evm^FkFj`jGYfi?Uzv$GDY*sG>XWZ9HLQ5Lg@uUBv2D4AP<<{z!EA?+y^B z7RNk+1^WN8kC=MMoqrU%7tYw7VEKJ*H4!b!ir*-7<;2wv)(V%w!IVF(8Wy>Je%LTI zb{VN&3^3}>xbUErZZ)xYR0-cy{)xgr5F^&?8^GKRyH&kg`RcxW?S%HC`bg;Vhf7C* zM}a80hdEAR6U9x0ufB*Mi70AJXoV3mF8o{dHszIOn2dS@WANYDQHeM)Rc}lNwW1S+ zsK_U4k=98fdiHr!*mxs5%W4q!=JTR0sS#d(HVRP>!Z&llNYat5hSFhR@YViIc23#B zxg6uNR`G8WQnOp(d_{boMo9ZnR@^{HeGSu~(z=XK?{;p0b7Eozr5cXRa4X5ZQ8|J| zmcNs(I5_?lsjqVTr@w7GN&A^l#VGF!BI*ASBRe)C2jl4ykk6!gRv%Vb?_-aVcl^{6 z1(46s6aHSjL=wQ&a32aZb$my=MgNMgs&v(n-)o7Rtwj^2^T7$DxR*iW2Pck{rmx8{p(Nvcv>F z2swds^b%S-JtQZ;?ToYA#ydWIM7Xc%&u!yO+vBvGzCzX$ou| zZv$9d-&`sr-*%9x+Bc5&rqNzYvAu@>(@n0b;Pn$Eoe4w?YsvCerV1+*+E7*jT-1oM z@roQut(cH#T$b4A!v5GOmXQAv*V%yC`UYoC9umWY!iz{*BI#Q^;cY{&L4tmRheB+A za0sCYesM3(Syd+yZJHY=39IW*?DP!~Nii+LC^9)=>*$QsS5@N`fySYlg53c>HYyW- zRK5!(7$0vm$zLbA1jMg&EC$n&cRP1UwPnUj$Q;3N3(2$s$$%b?)zKBJ^wqw+>W@Ci zZV|Au`5f%}-uDpBbIv(G@0eDa41_?B@AU-GJJ?V9qYU~_;XN8eV@2LX$o-%Jy_xeZA{57;9Gnp*%I z+lF+_1TEmI$a3pvL2+_)ZfINwGa)irmLx5u%$HC~XQ`JkvkTj<4#CABRg5E)(D)uM z1A#PHAg&nq%6%_r@xQ0Z@=O$3lkkP12MbIc`* ziyzr#k&H_V06wMM03278#y6i&pH;WiBzPUrbf!)VgFoqw8XKC|O(NaeUSWb$8(1s! zpk{p{-iWLF8IcX?qH#@lhVL64FqHl{z@{u(AS>%=>A#x>9+YSCG6S-&WPZS{X)7}YlB4+Z^q#R(b4}MB(*yn zm%qZIlGTj9v%?AS74QDe)0GtiAq!W`BKk~)ijWi+KA~v}M~#PgO^61t+hRHhhv0@M zwmmrM>za9=D=WL`Ync5VD%`s+58l_|P|F-QQ%Z1hhP9`;F=CSMT1*^lzDV`BZDS#_ zczcb1yptP>tBVaeg}thSDWUTSb|Nt?Nn7|I$JafNrDVro#Rn4aJ(-4l@f7GW39FHm3%q{#1-?q>7%)qjW3oUIsTjrRz4`X zTf7stQ?XhL2I*^mAdWx*(j_}_>$1adRMnjEng2{71}3KI*uf>BEs4xQo0b=zbr$QE zhppLyO7@rd0f`*m9FeL(X9b8qLG+K=>`@(?O8;HykV*W(?`Voy&g+<4{gDIX@c;*` z|7P=j>3XLrh~_ghnTutvhYWB>lf=mV-XPockeRuta9m76Kt4p%V9O+(98}&}%*ac) z45|1jD-qMt&2Ag>c>xrJAxpApX+!@^>sns#P$*;#kQwt1B7C}U&$#> zgO;lrNIhJj7zGekX7hhVjn?F04@3D-#>QA5_moZPgS?xbFR{HjrxZ2~z$@X()Q_RK z-vYypmmOXx>!%d~w%4)n>Nq?GiBQ#CGln`L;Ixxk+I>m^n!6stIPhA=K~#;wEa5}AAYn+!i&iKv*>g*ZI)WCY7+rc8 z(10#WtC*TW*3#BkL{TCaG6?I}%Z-)&&rG_McaqI$HCPQlQJj>31`u9-_Mk+|hp#Cu ziFE%cMz&K24r^+qX@bK|E5^ml<3erN-5ZX+Mxvx{T(O*PT645VncJh3$QXw(lC_=Q zHT>{3IgwrNY1URa9baFQZ?Jy|Gu(X2UxB=nK@pt1<$(Ncc^O&=MHctR64U8{S!RCb zJXX*^r5FMnwE%V^!v(scEYysU%`eHD=ojh|`&X~&$ZI?bu z$%)FS%C6Su?E$YlyoyYvSJn+s_vETk-0hH`6hMU!i2WaxOSq6qM~438@9zo#mt(Le zCRlyW6k-EuV*!&(4Bb$V020cznf7iuv7s>4!zvbp$%Cu0VXjjX3^h~L87QSfQG_k_ ztw~-r*fDF}55pACq?W;wG`O%alA$b^_iPeQ-Rk_>xor#lxZ_XfWh~1wQs`mt;e5ptLZM1pF<=UiSQ(4RJX7 z4R?yn!jcmuK?2G7M0O1Ew3yD7#$kAAVX8P2i2k>qi!v-aT9v(@xse|1d+sw$%!UXj z?alPqyI3yN;m4qDsiI>kkSBbhw}y8Mm)c{518{2(Q&~T(T*i- zI&qC6g{&}G**X8Y8$K*erWh6v2_4lmZaM)>U+xt(pv%l1v6z}isV&D+!WlR&Be=GA zqTdCj&^nwk_B#uK7venzlQA#lowhOAjHF*QRy(q?4UtH+hbtB7*~Wm3H*+ z@+^$Y4U&Fe^55W!B<~OnqtNv%kp1qg3BqW#-*3l+;OYMJeY8>#4=7QVbofupU48i_ za0h1-@lHYq%v?hL$6bE0GyVMOQ#lkzOe=ksc%y71exo3Js!%cYCN$Cmb&BWh4!w1NUv`!Xvat)d*7B7;aPjA^-;1oN$c zROQD>z8hseYH@Xsz{L{}5jJIdK(j9M-wWO-X`ZA^=w(+rXzvz0!C6H^dp05j^OjsB za7RRH16AW`Hvb!-8X*zf(O$$SEQglIg;RcAhhT;#(ZB5NXoyOx*ao+n*5sj}MRJB# zDMVC0CeDhom7HgsW#a$LGhOIaBmcCd_A}?v?s7@yBo8;o2#44dMI|3~O94FCMeptt z2*C`~GKdQ2wu=CfE)T@2re-;>192Nj9pjZsygXSyoa6_~@7Zn_#5gdc>`wRg5j8 z6ST2we^>h_3}R3yR8Z0tchni@c+fbaUT<2)rwK#^ofh_FC9UyM>6zFxPGIP0k<2es zB8hFW(MIpDh0TOrBym#Z#;9ygZ{dh*(e3iuYBQ+OF@(?@lZ(e?+*ty^I_+Y7B!Sk| z9|=H?s_+!uCTxDkn^^~uB{-tlNXE?GaMPUdu;+c!@VQ(5cSB$MiJ^lOZ?H?!t+!z zRpy8V7kDvIOdYrwx5+2~@gy`-3mJddIn+$i={yfrdEcnYSof2M^>%WLcIZpJR& zd9WOS{&MAsr5U>rWlutRYmzI|+Dcu0m}9+ONq3d&Hmc^0K56E5W~rWO{)PNHQJEJw zx~X~fs`*vbRYJxpOdjj(iLHNuSr{9YlGknM%z$=HxGjCM-jZWGI3aE9I5m1;&Mawi zs9ahfg?+p+%`v|ZsPHyv6^;dJIFlj14TShpg0U7iGTLk zyU}s|n5gDNrtCx*;-Adk6n$L8OVZp7&8(StNWq*;KKbY9anm=UwMkiaD%Nt^9pUD} z+3n*t@bXpUmwau_VMM*a49%k1UuSb<@W0YUZUj{Z%sa?{=tV6o@-fzDUrCrOaTb}G zDFn)1v2^UIb6igdz3Q>o`vW5O5bgfh9h;R+@Dg|%Yfe|4_~#$a-Y+pN#(a?^$Ly@q zNiO^+^ptGX_^1!QIQZBfy0W#0NeJAKzF$Ii4m1MGPg3)mqBI_DawYTEZX3E~e%_|y zLITMrS?&5)xvt*=|HcPE$r?cpT6)N^^DEuet36=X4YqQQ}! zpwp>wL&(pOh|2E9d@_vFt{W0;%SV z9saE7-ul=;N%NK?<%HzRocTv+j-HOy()Ogv4Fb1?{jgb*yX|vr@TLfeE!KP`j(^(? z(COidwtvPf{7v52s19@=k;dDL0?V=&Q)Vbh(wwy<9+)4FePV?~zK{J%x!MOp{)P9i zNI!DUcd-??E_m9M|Gqi(OMB)~ zgrS5Dj-a66r)3Zs#1>|<69q~GmA@*yScoE9A z)UGe)5H>y{xFhQx=|s3vyx~dcCZyfcb)(M$Ze}3%pqgJD8b15ua_fo%Z>SCSeHwBg z+r#rhtxPyvyaOBe^535{_qVJ^S>d);eP-G;ooQhdAQ&?Z&$K}(75LcFi2zr`My9F^ zLc@2l<=HgMdF-H2!G7rV7O=rNKa$_N^Y^-b-7M?A4}c(xJnE)@>~~4!^??~GKB)~2 zcj)!Wk*!N|?;{mZ=&L_|@*xy{j;Z5%C;&HqJTsU}{=-UI-35A5tkrbde=11>d}szW z73*;9%P?M->Fm5N#f4JNQ4d+Db-!PjJ$BIytVZoFgOieuwBS?)EqsP_^5MsIAj!(V za8#b;x6u40qg=liw&=X_IBHai6o7W2YdpoKhVpdBSW+vI0LNJG3(``-MGVb_rlP{W z^Ul*Z4K2`K2gRYZ`b5YTv|50^rjzqV-0owR~PIoCv`$T5!6n4>I-uJQJK8TlqQwrj9Ya{x9xyJ z(iwa*JMX+-0No`gWWna#xc9WDIa;4fsL=a1rQRwz6JSZBm2~Ly2nd!p*DW}TS}VyA zYO@I~ukl}|H_Vr{`B0tg(e!-suoUJxl2}8QWFHE~oRZD&iJz*~)fL{9UgSeDejzAG zjmHguT*SAM`1`m;!;lT_@T=LpgCLQq>E$CQs5`>{92xDo3;+EHQWL!!j-?0TU|M9K zs2E*6g9rW&V=iqg4IKAPT)-f@9mf zE&RKHXgyi&%iEEvfUcMS0K4$CsK8OlIz?2ZD||35Tdne?pzsPT-w_c$duoehH`7n} zAFb|7E(nSEk6QnkaHS16)5{<0=vL4uBDII!@Yt${Z5ZCgPwVQ{uYGMA;$5X^4UdQn z=qMfm|6cFhK!VO>U_s^UNfDOwC@aJh&KU~}=fewQL2~mrP1pH zkdgKC>tzJmCvVUvD+G#swmIX^-tV6h>+`Fu@#&w_pJ(DvCst^4;#Dhm6WTrx$!ybX z?g+(|7ezg-@@%yeiot@qGykgT4Mo&%$qK5Trqz@5W9@mtPa`MW8+Nf{EPHyu?2aqL zWp_b)7*F>`aoA2a-vTG#bD}i*HKF8_^|i^B2)ABAv>iHy2C~jo-AP{H!e782tM`RP z9%~W*sSnzOKVf}>lKiv4a%xj=qA4`5aC*}QPNV?VTE%YeFCD(D1TrWX9HlOB`FV;n$G$B+liIu^Jr1eh)7AHKvB-v6OEO>7<$oLM|CeQ z|Ks(yNzS_k!%)vpc)P5y$3@roo!%-tA(7lb$B8Z=@pa<2GwPP$uTo(B7L(AD3hoV2 zS^(C_O4CS&Bjq%BY=bI-?+m1tmQplB*$3q{7ZYxeZ;;=0dU15c55i2wzX$F*?Y8;z zDai zB3Oy22$O95-QBUo?X}wUW1W-XnO^x+1bJr{DYi@9_Qb_YA>cBc)zj7WP59mb6?Zxv zmtMEx08{)nM}CxqFyMnb^@}jv&G>{Ps`IXim2H)u@Gx_~2ads%79AFe@wof88RO?z zi?mgX;+@{fvjvorn8(QT;C**D<=Z@K|58U$tw%y?6tYh;IoKGL#|a<7z5LfAGti8V zUPMC$gVnri?kJ@-WTC(uz2GZ}J%@nr)~T)3uczj!%sOR*qnk$Fsc)%I>YSxd3v3xYU% zm5Y*=Bj14=sjsCmO@~i&dOp6=`fYrP(5o^9*uaqbpz9t&A5ZlC8o%tt>r{}~VB22&-MtY8FzbS>H{iN9Tq1QWv?4Se#eRE;Izn<(&vlfp z55M+$j*O1D6>ORBQ8|}snBbv~?Eaj?>0X$)K?V{%|LYZL^Ib?we%C5K?B0DZ&7>%P z&p(1liuWzRjKh)DwCWN*=pCduT~y4jlCWvJ8}2h|M9#R$D@m z$XF^_;(n&@H>a5 zI0Y9pl)tGiJ6xNRO0#3A`lZ~-a?*7oF{Om-;$&%=XF#?M+=oW+lZYTHYyS#_Ox*}} zYZFUDWYd?}s$}MO>nf;=d~=}<2J69;KJ ze9iJmDm~j&J^YNPN`9wh{f}vIo6{-dZq`~ssOR}HB3K@3zX|ILjcRfZmwD~*DZhy}0&C5yPV zi9C54(UVaf8pCcA*!5El35)&_C#@2_S|dw)5q)|sdJQkQ>iO7f+j;%VDf+7>+!l>; z+`;_sEsr*$0CE{JIO^Q+h7K&%qB}?rHx5!IP<*H8wJ$iAD)F2hmg|43#yK!%TJC6W zWQMCm@Cxfqqv#Ga?G3P)cs|UXKO}G$FN%PSm8i7EabifWfN^5b(583DXp`;4D={!$lzf^n$l2garA{l6coQGGoxVSTD_pzH z+|g@+2Ipeu$AuCRaMGR2 zU`nHGDM&zV(*ZpswT}<1NQ^C?GbrC_esMy1)OKS=qhVyL+J-dSXmKhEppPCV{UzAX zde8lfzsCw=JV{px%IAOH7b|H0I@}RH(x*Q*V4r?kr09O4EaN!Yc2fp{;%5?_byB4V zjWb*-KWHJUP@0XIa;{^)$5CB5euuf?6)-D&nxIrMj{9!LS%lkbej;v46|oAjQnjL8 zwouq!N!^6C&y}TU4Vvs~N&bQS`-~wCrVny!9$LUSiI$u&@kiIJJtAOr(kIe;FAfyH z@ASc{T&A^i0mq?lWbhaLqXID%L zl@79%ygAAmCmM$?{g;l}t70d%WCquGx+bn8b=)nl%bU~Jh%wGjRD&CvpiU@tODk04 zpLec)#|RBIP}TPGA!93Myl*KWP}4np)4348+QT1>h@W8|saY}<_!i|D;JFW`l!WTm zsPgn93?J=QLYg(e9|-1SzBq*g-7zQ(%yhL3?n-jNm_LJ_1vPuY=`>;AEy~<(tMl|h zr+Zf_=k=wZIMu{fJMn2oYr9+nMAE za>t95f92>2*E?5m{*4c8G;rnrScBlvl`@?7E=dW5dlL_- zgCAfgBbvy6Ls<R6LzojULQh_9r0>FQ0>0e zFHHWD{V(-E%=CMvB=)Pd>iTn8Ca);PpE*ljoLl(o z0#)WaJT}4ziHme%t4~Y3PYtNx>{)vD2RQToe`kJ~nN$3SS^a-C&HLECL$B3c-PPTDS8dc) z`-)VM6NiWS1p@>G1TQHeq67p4obYpCK|%bqIO!C40s-0TONt1pc;sAvhC7?SrS3xe z&Axr!_+spkE_6>!Jcwf0Cd;~6Z+2TsGS?<%Vq2z=gV_|ndMi{&(Kj-o? zPygpu1jJ7d5n>$T|Lq&`fBXLb@W368x0XpbDv%2>6i^c)GLrLs=)ta4KDdevk;s+x ztHBE77wzS@At21R{*|b=-tpF&^kr(NZ1|=cQlGgA6*^$8ml(<7y9-re3$gbdRLbBz zX(3o!E+}Ff;q`Y3QtxYL$Mlu!^;g3xogJxn%olIc@cYqun9B3-5#lBs4$wV$F3k+U zX%%sWS%MRO$uE^lU>(>ftpGJv3urKNh)}4JZ2_=CBf&;?Zc0iJR3-tgyzML!En^81 zfE|ZeaHcb&N+u)po+c0hxB~W*WCTyrBak1^9v7)TuoH43jF6tO1bo&qQ6b&0s;LwS zB<-hlH)u0Jz<@$)tne;WLI8zv2Sx-=FJe2y1AK#Za57tp+TI%I%_5K=rJYf}-NDFa zs7Vl_oqN;n`!&1SY)_ZHi4)E9;UE`Hm9asdLYbrh85#@3@-I{rMOQP9D7+$KH18Rs zb_t=V7>&)UE|>>rm@MJES?&=Z=E)P^i;dY8>Y?{bpPnO~;K0NA03L9O4yfrbkggYm zle+D&Ef~Q`;}Dx5JlP-*kO5MaQ81NID;Q#PwW8B#Fm)T|4*ydmC!`HRJ&1W8krP25 z^4SN`*>R>E?ccSpb_WB@&}4kspf}7w@$mmxMA+n52<6#n1#ZCZLhTZ`J1qbdvNWem z-(OCr!-eM?7U>L~38#l5Dv;+$&|#UD87kp|6~ZktaOG8P^-F`Wg1eyS{9%Lvt<@v9 zljP_vxGHKAmYuMADY&uB)w+6Tb7LYS?%_(WazY&G99oVir9jqs(GDZjKzevjlAFd` z5O=hRA8e#4nI&XNh>c-cCb8OEx>d!^9>gvf2wWFjV&*X!;($3Qv>_RmnkXWQRGUbP7-2yy5M}HrFc@Yme3))|ngY$WxNc9_Zu^+`+uR*8QVJpg!&eCe zG0YAv$>)K{Xl7q%r~B<8_uP8#A2>b=e-4T;2baoyd}zL9=>@{&wm=@B6N_) zBX)Cg%(JKFn+G$sZsdToKB=Jb5XUv*=hej)xDStyy5l&U z1){Nx;c_4pcGH;K**k!s)1%RKO|d*LakW;R^gMhsNOMi}GiP*}Z1*YE@_@T3k^Z2u zz=^?2nT~$qly?xZmPkoX)J?qID}yMY5)^z;NqG7f>hMz)Kt!h1jHa zU!$*;g|$Sw95E$K{eH_Dz%y_QZ#!)fWfOW5$YT<@+o_d%;cm`#;Kk%zfp_Y_tH=Z2 z{XNsg<4D3B2-615AC0yiW%rhBCa_i)gph>>^vMgkAQ;Fbe64QLacSD_okgazAj zh}mS7(;@*_anFTvEk0*jJy;za^@IOiBFiu9jXwG&>4NPKKFJAoFE8E15^l>9;x2~4 z1)4_-`6Hl0o^V{b(3ifC?jSR)Cfm5a)vG?$l=)hTleSOzEqIk6mCt5{t-9UGVR=_=_GZ>m%bG$hwa#)~lH z0gAp2p=ime?$b^OV$VtP4ff6Vxwm4B-a1C3sRAvH***f@I_P^P4_F8i%~Gpa!68_0 zXVUTMDHCsZPUwshYBfK*axOR9QC$g*soPddFSSTnhOyBz<2g_UcnNeOib{aZxtZJH zYojLCTVWu*2Csrl$+oYj#ZY>fWS z66kiXPUz7$>chxu0xP2MkfSBN*+M34fw&xG(7QZLTi@Ln72!!{0AIfulD2hvB^KQ7 zNwU-5@i#|YbmAMaohR+?hia|)Q5OTv+=PLeg9g%za{usQP(D%&m<2iae1@?^- z6ffX1T8+)T-q^Pc{pb2y0iLiXLW_m~6NkvB`0D(@nxr&8kB-?)j!ZuvfWp!^R^9Z6_@04F6vPEHA%M z`k>EyqUTsW6Nkpq)LWZBcrvU-K8irZ1z8Sqq`i*YdE!J=zy5gfe z>Jz{&r@guwMQOLvt0_USM3#YgRUCPs&%`mhJDPG_`4xWSbdDkF;YuUUs(cvE_N>B5 zfOV?Ev9uo@wlKoSnD+P4n!d-_+d+fe(91;!(tGnZsIPOa=MOQ+wS zNVo3LE3imMv67n+ZwS?2y+WYo%cFkm35?(0Yn`)8$MJ{j%65=kzxn^3mNtKQxqvqF zRSZQCwj^L&E~Ll|Tjv!8zS6?Cu_-a&vYB(pgfd6Fsn@yQGgU$SX7V}i|8VqzNvtLD zA`Bkdd0+?6uyr|pZnnPew3fxjrhdE+Irk*~WqyiMt=;4>OQ%QRwY6Gr1P*eQL)l;E zxq@frgxYCgSx3vx^!q~r#cUK+M=GN_&lGJBtveVf5rRcu%tAMPB{r>2r#6u~U!kf8 zX0XMXFOZpNtAUy~j$riPu7q0Y$W_8P#TvH-p-cL%sPhfniz>f>@2);bq-4IvqRG<2 z21F4g69>)NU42^jKH^O6$+`|VKemd#Q=H!fBLOWb**_y!?;Dh7X}?=*t7bv;>LRb$ zOgoj_Zmv`(Z~y6hEXgzDfjVe&R-~raQs>KeiybS=J|^KLJcT<$Bzf+l7JsLgnEQ_W zgFKD+HDpM1=9Ya|@qf7i51Q;=PWb0X;(JN{P-e<6u{MnJpxC%eGme+D^mKilK>m9Q z)BSsyG?spIoa$64(<}6in4R{HSFEzyL!_~aP{j%|Gw!C2oz<(k!n^?g-lN8i{`6|N z47iEKC4)A&2t0pGL9R2pS*gPZ+$`tP!AFNL@X@+0YF3~qkvDv1+{-7g+<&S*ak@xM z@<-r}2T3nm@b352`XkNX9S|}tkiIqEp%wxVkjGTvV&T&CuH0T&GP9IVNT?H+OoWL$ zMT%30t&Nd8%J_p8LjRC~P}?_*l%IBLygF5!eOx(QXQTEA7!EBTB>H0JK8=s60VdOn z3!1p)J&(jbmN0umU{tSS6E_bcqshcZrh7DOShDvq(hKPd{wHKjFO(kG@u;9};T8gI znRycNN?QMaeC)uISq4|RM*AmZgaKA>uJsR zYuj(f+`qcx7o3xblS-YIl&9)4v3wJO*5g6gfPigT;@Ujmqxj` zV{=Lu(RY9yKK|?X8oKS-uip54i9`c)H=FF2e=Ogz~rB^cM z?4=RtE14^hZ)l-Y?=u}<#Qk)|TrTjD(fRCnrsJIRfcr97oEwarT@|E?5(n}<+>&k| z*y&CGndM4Dg@2X}(Sy<2KOtl@24uQbM(_K}=iQnXI-idF%W~EQscHvKT!#+7SZPv7 zi)^mKYYsVnndfDh2Tyb22*=;U_&VC(w(8?gVc`VwPA9|>M$<{Y;Y-c|!Ut zBPr&p#~rTmG9?b32ha9f=RVgY?@9Ex(nmVZ?R)B!HqIV$yoWQ;{vx47#jMnRVE7BF`+ zRv3Nl7^LQZ9L3kZQ&Mwornbu;bJCkLz=<*(?wEfIyBtBV`zu*N)s=h?%uAxn+F^SgQYYg4zM%%O_@yFPR zf>Ol>*k@QDo2hj_afA5;=O`&U#(YI?jSp3|P!y(Glv7e1BEWJZQ|w|GchA6h71t>@ zO9+$C+W7JRD^j>%w0eHg3$#RCOrdcw(z7&; zi-0!8nzBo4YWSKLwsI-TNwJ=mM=0LX1~gt=3^m|`kS2EmiQ&%gCoRScI~udX_ML2@ zk78Z0V?PVP=jH@k@Kl*(3Nr3(G=&XK>zWyj6;TkU(&!FG_71eCD zh>$CsciIZZL0iz8ZQHB+yx%U4!Uz3_o|&O}B%B!OjjZJDc`iq{cy0}minmzmUFX1N zDUY>fczX!L5OiX1(HNzRv;D>gTjvubX`&qHLy6{mo5xMc%aPsLwKVzji$KrzYqPYD zjfPKNr1&K^<9&@-M!JT5dWKwPHVxO;sx5Ugf0pgbgBDR=E%Er814iPMcgyo}_v`@Y zR`{OAXCx(fw5F(N^|BFOym7U`+3Iv9B{~u@>8QBrVaDC@jrto_=&u?-G159|5%9X# zpcdwKwry&G7Wo2WP!8a{p8JvQ9mJ9T3ZRak+=iuAhuD_o1*-13%0szCryg)r ztlYPu_XQnMHqf7CQ$^GPoTUNdON-5mGyYA~UJ5Eo?Rehg<7PqiK02-5DNvAtIwTlv zTjD0s9qp9H#2JnF2s*pzaksmON}C+RYxiU*@M9Ro0ruv(T~#Rs71V`SYmJd)Vr|ks z^QCA<384KAOYqZ?OTL{q4Ceu-0kUa+nCCDKVXXaeHpd8pasvQzg{!bkN{u4^UkSBk zrvA-kICy%|Y|4UVh4ImNWSu@knb%TE+^y>-AR>5 zEjBn|b5z%8T;ud`!!kRNw5Ztq^fFQY2_Mml%A4S^p2UqMdOiN_na>EOr(?(qK4B`0sjk3IkSe1wCNg6cbAaCg2m9+jTP41o~3!0|D=aL@g3dl z<0||Ey`AvwmPGqaK&#!Xz~|T8P(ap$z%3TrF?}RYRNpVb+kUH4f}tX=@uW>MqP}mo zM6CE-BauV_*N8})^z1+H?hVb$&|2na^J;S!>UQk?-fo@GqaZoocm0BCiR^t*kN$V% z8y&r1$0ReIlufYsXr?(iA)ZtQ5<2Gu zMi~-{K&WuF29?DAHo0dmg}w3x26sqo=?`^3D1`|$gfCKe{LeuAV#e*F%PkjEVZa9J z<#D5rF%=7mpaSC~6qL%Wz>WhvRW;_RR4&8GOS80SHal8lfLFrTIbFi4P5Hw`Jxfwo z^%inBMmsnEh*Yg<|E8fYen4koUO>@&@l}BKOOQTW11lEgUNzW_JPpFjobyUV%L z#T{m3_$tGa6b+|qZT{%*RqaZqyyf$7NlI<8`wysDzn65Qsp=LFSeTPhRl zfjqAy68Q{$SPw8nGDFKCHVY+1HiO!1P-J@7cuK=UXa!cczbypVGXFD|Y9VqVyRHQ_ ze^b^xGFH6RG(p9sqo1*%nZ;YGsuRonRIz*168Rf>BD$87PLu6G2TiP^O4XA7Z#P=q zvj8o<5DI9`d#+`oNOku=7v!X}rl?JuvP}IJb441#Qyo_olBwhJH!tS$un2(pKnaH8 zAyOHW8$LqQ4Ya{PGRWxHllc=HM-&Pds;Hxgzcyldfw`lN1~3_Bs33zbT8v*sd0h6og}3Z!S~FsD7((ftexqs)?{2y9U&mCmNq*{fUS3+2x2 zru;X`QOE z&}uU_5{A=~mw+u7lfxJU0x((Kn(p}Ev71A>@*o_uwy@zdk$f*yNHbI{0tqtR()H*mJ1ynrhBa;ID0lRu~PAull9rmNQ(s_04M z-#^%zewR^FcC5OtAb?G`y~J>%>|mrQK^og-A8b-sgULVn7Vl*jk>>0~UU*=LsBwxuY-q?0$pW@Su|RjA{i}uWcTJ+>zqE9OVc@yx zbx4viwg~u)DL}4KJ@(NSLcAZh4p9@dAfoIpN2IhXwPO-}v@YP{bd*^U-=d;S>!2aR$!q<|b!Wqbt%BvBSb-BV6tvUyQG>&pw zB+2Oe-yC^mC3UHST;WBu|K?u9*dFDMdY0fIQ=_&Z7iY{y;m?LzgqWXmq-l@#ED^BZ zqru3wCD{0lNr$Y=JMPldWy1=+!m4#}a2t@f2=?0ZRkQE*)-Q(44_og!7(lQj^x)<$tGY@WxSlg)2v|NJZ)qs$ssMmBfeLY5G|0q*(`jeQlKo;> zCXT48KL;PDEANKQ7phw6Ifv2?ewK6L?b`n>)Exxcr{1plbM(8{5*^S(HcfnqdNa=z zS51J>=7qIAIwJTy1Iwo9R1?L$2#tv7Vf6)TQ6@2&MgBSK87VlupQnysiuFi~Jc4x+ z#b&1$tgD`wxXz9K&%56#(j()(_4Bj++=@~%;Qq1#pU5Y=v%+&0WJqF&(|V5o6kx1mu+wix?=ly->K84?UVr*0&fuzYW@suL2^k6bdQ*~aM~4Zaeyb_l!<#U-yd{>ZTV$x?C&9)Ow>+>1^@9+w zoEz=jj3lbL9$iUiVn$4l=F=XQl$la^t7q>Bc|&?b}9m~<@4Dkv}gPvbQ0^2%W`tJY@|7gdoU`P zHfw0w#@biZYb!ALAkfG^iFJf++Hi65nL@fYcf@ojJ56*jE^Igw@ySgUDMiReb@mg$_b!9kiHJyNRPx| zQ=Z6qgT`d@(`QdfXQAjDaiFnJt@UV`&}1l2prnVIUS}ibLR@}edq3H&tOzI*#~3po zcV5?Fxs}%o+*5C-SiAO%pO(mqx>llqj}iYKySfb+Bqy*)8vVL4CJYqveXcPth}gcc695VCqd_mj6IzXg2f5xdd}| z&4+q=)?=zbGf`79r&`p;OYHm5SeP0O{NV?p zMA{%W2M-UV9jamb zdT6#+38Bc1@l9^ZT5+;)`0^)_{G*2xpa-=0ULkj^lF0{_A5*}OPt@f56D655Anq#n zmvX6)>u6jb>dlKV3oJGQa*Ar$y@<0XK1&xc`cOnM0**^@+ffgW{WLDfhnAj}I%_?R z5$4qD2}&e_hN)^~Pi{W#r@5%qS{%q-aCR4aK6URTGJW5geSeE&D-x`W9aOb2XMo*l zjlZ3aO7+3JVoIDdLA9%tnv$#->5ruuVn5|T;ihJtJlxlQt8YzGYpFqiKV>tBwR$USyaFNzEjFmP1ig&EUDzMPq*8;+ZEwJWTZ3 z3_G|d@I8Em+d!IlV~Bl<@(07#^ecM#JQ(%yFTtGzmz=G+`&lCFJP7YeUl9lymOXA% zr{$Ik<}U z4-`pXt>Cj@&xnOgo(5cxhoWmP7;T@tFSo(lX6pxpabwd=EzHN2e!Di!L``i^SrcweJKh!s zdnxe}Pe< z1NzI?;ew1xFsYEqbZO*En20Czv*Y{y@8hY@iYbr5|FzqBy&a>Ofm^E#1sPYxTqg-b z(iM6{<5`g4c2U@I#^#Q&HTXboYzTf{g-2!B4{%BH4#ZxPPNi^cWEWZU=g zvORrQcsv)Z1)-F5w&^o&I!)y8o}*^=QUQ8}bw_pXR{%KTRQXga`HWxkC5UOZ?Jlrk z=GYP_si8KqG2Q2V<2yUp;*vem6AY9+=zguvwyFNkxAWf9El|S`um<^q8Yl;u`#k++ z_)_Do)#qtm@HMa7)U5E_eHai|HO&dAihR}5_!9wfJgV$6pU{a#B+w`{=SkkglU6p} zbUFpboOcGEDLk_rce+p&go&l0AQTx<)hiV4f#-*dcocqRSU>3#N~lh1?H?)8A78jw zTH<%cA4u&_#zW5{l=f_$9RG#wau!=R`3`x0r zA9vT*&86-)>1R;9$9w;&2D-qRYijEFupOn-NG&K4H`tvj094}CH(NMLrfF9Q1Tq-eH>1X}6~`2N`CVQcd# zuk{pvvSMD90y53DGbcbJnNlH>&ezcG!W{7tf8v31m%`6)JWs&! zQM}Zu5AX${`-ZeQ#cgpB+FR8a?rSeEm)g0cy3GwX77)?w1 z*J!vDj}NA*2mW|3NZ@@m|KGnbUEqGrd|ACcq8@*IF3Z>8_1wWv_Q%I26hJQ=~wk zo{xBxtZ!Z&)FSp~s#(q^5B%bRZVLNdP0hK3J?#?ryP4pN{f=oYU;Oe#yW)mlRHYAH zXic69-6>B7$ss=Xp&Q*kG^@^8bQ?sZJ7A_V%stm-2W<6JaXnws(+Q^U0h^E$<^(mE z?`7=p$1)bB2M1uMJctbSD?QE|Sce4F(u4G1HT7T^5(RLtTsDlS$ObFide?W>TSi-2 zuIST*@IkGqoI(ZJ5tjGzaq42odZ2GX$pk$+;v!eWTDxsz9WaS5&^e%rc%S*YtCv?E zJ!EZZ2t#B=Vnv|=O?9uzbb=1gO#a5R=kp^${)c`MsvLlD|B%7&nPEs!Z;2)+J9kk z?cB-|$1%$%{BAlMUCg!~iNrs04vh%s{~&8ZTxO4Lu9`j{m}>T}0q)3IDZgXr(TtTaou);*JI~GFL(s$h)pnA?0doY$YOgvlB${W!>HBw7ElII zf#UvnQpsa_TbZ|b8*aABmQP90avoInuDJQPB$zr2OL7{jsl#2VT_DYyb{&?9QbpY_HzLl7Ijm*DLFg-t4P$>H)^=c6fYcPBBU`cN`F#y`Uk~|pgLX3I5NA= z^Q*&nZLq&Hcsd#Wm5Ao=x@(7`1x^F%!!FoO7O=mqenVw8eO;A~L$uX&&Z6!gqw|$R^DkRJ zz?9TVUvm0eg$ar+N7ubUKVP*M{DLbQ@(`-!1jd`1KUdT~y`@HTygpGEyR_E?Y z&uWB+(F~7#kRcHju{-6Vs*&auUcmhuXFA(TvU(ZB) zrzY0$H7}yTJkmjANM4}Q&nYg+J;Tryp|MGf-G#5XPdiUD z|8dU(MLXWyCPQJTS&5qdn}5>i&1_K?v%dA!P=wiF71jz{$jjCsy@FEBR9=T^ciMdW zrSeOk^>V36`iFX{@sepayg4}s4@X2L&-_D9f6KQ==%+g`=1)_o$y25cx_w1BY9=5B zSWtv2Kn|jn9rDNh7(|z6b++H%*=Ka$9r{b#MEq|iGaN6K zjYn_F%k#&$ULCoi=eR6cm86rocfYOb>QNPpm!Sbm!B(jS=;&1pmYRa-R3e~>C^bAC z?4^-4v)!xAEA;3q-q!n|+#7y?g<$-Ew2yErbwge-lSgi@l^8Q8QZT#7 zvnC3RYG^~^{rRL*!}=}Rj6rGy>mk!sTw<y_0u+@Am=eMHjM+i{_jdKw zuZmn`ysA6FQ-(hXhonrI3Hcx<6R7gY^Bm0xs1;U)bvu()o>ZY5it~7aFY60b6KxIX z5T)*$nWl+f@N3R2Fy6{n{@31R)HZA-ltHEdU8RwvD!RyvB=|s-M1Wy?!`8GN&CvFn zb@A7+v2f|PZRDFK88h8jMu#MZ!)cMs(?Gz%4K-3Humz;m_PN!07niL4x|$p#7o)%A)H2Cu{8P)HLwZGo4KWvk2hoX zIVb-b{fGNL7=c|gJ3&v#LO?n=+$I@E;4kRN_4azBX)93c=&>$up_%Jdc@V$JM@N%d z#n?%w7L>-Bk37??<@OSSyEW&Ibt+S<^(I4u;1!J;P*Lg$#jQ~A9jNYxCyW3!X}s6i zDB3+WO&U7LK~1Na(K-!1G!nKN;`k!M2N_h*K0;_2oz~mna>9fPcRQw%#GDVL4hLrz zk&~vUHf3*pkl#yz)dvW{o%(hE1a|lwSni{nt=sd=8Ehk=+dJ!fP{J}_Z2k)?m z?g6W$awMiI)oFz*b*hV!IX0u-JFublzT8sss{>@Iqt+!#UGzLcc}juROF=UO8UpK0 zV6)d4+%M4^mfQ^%wvf#;iNTh)HbKgX^ieIr_OIUNkN)mUJ!D1|E9Y4bC&r2i{s3WM z*G$Uczh&Ql36^ENg_hH}Jn#{oMoR&yQtL<*Dg{Z5&v24(B+dnxuy=NU;IYj1<*}b` z9$W}3$mNEoa{MhY=|n{&Vy3}o!~IjH##AloADGar1d#+6wS{adNUFRrWxX<0y7=5$ zyr{(Ym#X|(x!?&(hY8F9=SoZTETIF|4A$Be7$5Qu1&EnAkyq-65yc;4o2Y-{Hxi@m zu86;-{(vTVpxT28UC9Y2a|AX5t5rdT>H2%! zD9wpX6Mrv>#*r?ZP=AK2S~B5kgh-u04rNm>J~(i(FXI7#5)l8t^g$jmyS*kN?(x*x@FB4UB zCg)L?!(>lvNP0)`ymY|wlVwdibfb{q`G`0XG7?|Hq{3O}>CZigmk`juA3wwha@7ksMzl{&`THm{g;v}DsTUY)X zt!gW`$CNX!j%Bi`o#f`m%Z}(!e}0RKYUA#py(#>BX&uG}xf^az8ptocM;+`m!?6^0$~uo7jStBQ(PR8k zrQ*oVUs*n-y}Spl zppAOQp4eVb;B>~@@MchFCCgfPH6@9tUXofxrpoxq0^7#}xAb0>b~S>6lJef8@Kr@& zY-Z{N}p)1fkVw{u}iZ8b>=%gRj|fy&vF5|@sBVMw{oZ9dop2pazisW*kzhT(b@)8X;g--kIwyy!Xj%>o$6?k zLw9&dQ?%HOy_busXNMIfhi-wI{xW(B3Qq zEm0`zKnvAOH4RhrUuyH5?(np`Kh?IH&^q6qNXt=mm(P{%yjutxjMm?`bX)CyaYc6& zOust`rr7)be)sk{eWsU%A*v+E25cjlzA<>}n-3z1glANinMq_QfdEe0DvkQ5orokZ z2109~8G9-oeRX|t;PO!`7E!KiOpUaRsH-$r%Sfpjec3vF7CWslnpGvzGPVYF=vJ$a zy7KX!t&^8c{se}NH(I);Mmulu@Af>-xSHTZ;7c6Ai1Y!C+Y50S&oLmt4-v03kdm+1 zDK{-&7P_8-oKDj-h0(4f$( zy33HPFUw`oXks9&F^e?6FERKo2Qk=i8~xuE`e(K<6=!k!7^Kb3p|akTEopRm<$?rBXus;+iH6mXz* zVpXhWRVa$0s?;(KoJ5h402BiVPT;W+YE&iJE9PHX&}4 ztc;NmL9kLI8e0UaliHSuhAc?k=|Ik)w%7EJDUV2bHQe}(Eo1Qovq7oxP5^PIvZ?)h zd_IVQ^9+f1$PYWpRfMjv^{L+R()u=FVpNe54{{#(sEMFNxri;9nCCWwKdeMewTP*} zG(hSMo`)&FC}R9oui?8=!H~*{WJIOnfcs;p7~+;-dLHquUf%f{ad$;1cI#-D744sf zlrAL(=Q+xREm6yyo{pgP8^W4C*xpyB<$;Ng=3I17l5w9pct#^4i$W}LE#@yoo5Ob$Is!s58M7S;8Uyyk65IKykaD=vMJoexl#IzMFU z$JV`MOvz^Kaqhzs!=#9^mh{broY3}4&)!V@I~7t1ufiK66RXA@o^gkM0{3j*^~crQ zrDyN9?^L6eP|1Pd3h5GCRZA3?$W+Xcm@sL3>O0?4X@Ua3gGGt3@At2X1Bd@B3iuW$wXO`RFluZ(;r!>kUru z<}LT7qyrs^!+C=bs;5?g~`tv*Exkxs+VsHe7-5OlOHX_0{?3e#>WgZ+v&% zAH-PZT`2jNkM80=?;-6~V*?DD$B~{PkU))-kU2`R-LMycNwqC>Eiz^otAnNF#TR$M z1EpCqojEn|j!Mv;l|p9j?O3S08n<0vCgLhy80zRE^gJ_ME>VGy$<4?=kCgYX8Slq~ zX#ZEa_we1*^ROA$ZxijmQ!~uJh0=W?!;qV%@Nd-zg}6sYRHucxi|6S~z=&Q)!x}Um zeP$RV$(1ul{YEps=KN($#DC(1Zmf==`Ht0X3Kh-gq z0MYDGu+%7t_c*pXI>UV#P{~eeK3c(T#{_<`GGsEU#P&f-@2VZ`1{=kJnd88}{n16GOYJ;P zH=}c_0e{LbDE)~|`3ej6Nh#5bM`>#v=tW5R|GgSQ-Z#JtCswrYl{Is|nxRQoVI?OJ zC>0=!*wdSJtX&`dzvj-lq0;t?_u1IluE{ng+um7|ZP!kdZM!DdWY;v+WKK=CZR70c zcV58x_q>5y*LAOTt+l?NwPIpO98}T*Ry71fbcef}K1lzNKvkjWOflC7Pci98U$_dKtFRM+BS?Q=T^_p8Z=(acaG+#I@qn% z^j;-~caKH>J@FT=yW(X+Xcs||~1ag19YqG1z`#4INpa!{W6Y74{ey*-H?E?6 zLxeMGCgE>#qnM61nlzNzlW6irrGR_4gpw1n!!WaVyIo74A* z3@^Ve%KdD=gH(yl_$4wf5}XJKI)meSWzn;{40~wNpE;Y)j-LeFB1@V#VriTi__i^% zUfR^qihFG*KkU*k*6*;kN}X2#@^(&UW@~=(75u$9i3nhX>F3oPMvLE%Anr<0^_32$ zH#@1#^NYN1+%oz}8XUmhns?)9ghN(Xq3Yt5V2b_Jd~)?t)<95H<0gGN`4pz!b`30Mp^Y(v7Gr zO_&_^4Pyh=-`;sdI|=$e?(LtdO3(c8#DsU((xgh@MhDEMQBBYF-|j+Wu0 zvMFW&p@vB{lVnR!7dE7HPvz?4|SPS>dPr{ zfDoZ;Gr^$W_Hb3;ShE=!rL@wAd$U%0(xhAmv=m%ElO<|;mG@?(QKXH3M19er5&>&W z^LeNziUJNcVA{}9I=T18)Xehl@x%Bs2Vm{FZ$qG4c zPfbA0B1VrtRksYN{x}ur+Sb>?zR8Q<2?hPc=kez=%eE|!pJV*A__A*7afe*$`aU54 z@ZhTZxhFXp;5tYFuH-9X^ZDr|Ho%Rt%319TW%Clwb!&tlL_|Fkcn0QX85Y$L;?>D$ zgsCZ}hyE$#2wA)@n>qtdQJA;+;T}X8**df)8NQsHpVY}}#9i-~ z2UcPCk3=FL{6`+aMM3e!Wg!F47edh#Akp%x;R48`RC*d11xo3_|3S1Y23CCRCY49F z^6U2iaHkE+*?4OMpYS^d%iovTe&pYQd9ef)cI1*DzaSAkZpz++lfO|vKLzwWC%mes z0rpfUo8rPqG@3-Gk`BIT64j6HI@fddjl`8wyJ}|HDy%!7D|h)&(XfzJmE)UAd^?Bgbc>W+w3jo4qdFNwKs3l6synUMC(;}pq5w^zn9r#HA zSa3zlZr(tjGvo2+Iw9KG=vnJ3s7z(cW8e(%%>pRV=88HEq zRa1jwl^>Y8r(Ye6`s9-HwA+CYUnTb>Te^jPa`jAHA8gOb@{NFGaAv05ORl!bx-mZ?=~mudysN zRkHx6%xdYHD+)DWMFTTZU(tnBiQ5yqO3A$XlFF$M(U_;FJRvjrOd{W3JIE_r?A9}p zU50CDmCY)k-js7&0)Z!F%et;{360OMBWsl<2`Ig%i1gK?g-DcPUF_Ht&m zl^fUf9t9^EbU9`9L<{!$SXDO#obxsz>14Ug_|I&zM{?IXe*}Wm-~&vX&$#iruh+jH zWEfWg`?xgrKVu1ge6v_IiG*CulWuK~es*f5cDL$(qZE}P?uuUE#obJ>o45R}9gRFc zc*0b_y$*84DB?s`%c=QbOAl-SACj4C*2lD%Zq>@#DONFSW@mgZ3AjcJ56SsFT5#Ub z(*+esFcd0ecG<~yQ2R;}5y?S_E#EY%K462;FpKo?bV~pBxvf^nUx3TCsD2(Q5O>(-|ZSO`UWMoZrZu=2970A zBO_B&{>Evm^FkFj`jGYfi?Uzv$GDY*sG>XWZ9HLQ5Lg@uUBv2D4AP<<{z!EA?+y^B z7RNk+1^WN8kC=MMoqrU%7tYw7VEKJ*H4!b!ir*-7<;2wv)(V%w!IVF(8Wy>Je%LTI zb{VN&3^3}>xbUErZZ)xYR0-cy{)xgr5F^&?8^GKRyH&kg`RcxW?S%HC`bg;Vhf7C* zM}a80hdEAR6U9x0ufB*Mi70AJXoV3mF8o{dHszIOn2dS@WANYDQHeM)Rc}lNwW1S+ zsK_U4k=98fdiHr!*mxs5%W4q!=JTR0sS#d(HVRP>!Z&llNYat5hSFhR@YViIc23#B zxg6uNR`G8WQnOp(d_{boMo9ZnR@^{HeGSu~(z=XK?{;p0b7Eozr5cXRa4X5ZQ8|J| zmcNs(I5_?lsjqVTr@w7GN&A^l#VGF!BI*ASBRe)C2jl4ykk6!gRv%Vb?_-aVcl^{6 z1(46s6aHSjL=wQ&a32aZb$my=MgNMgs&v(n-)o7Rtwj^2^T7$DxR*iW2Pck{rmx8{p(Nvcv>F z2swds^b%S-JtQZ;?ToYA#ydWIM7Xc%&u!yO+vBvGzCzX$ou| zZv$9d-&`sr-*%9x+Bc5&rqNzYvAu@>(@n0b;Pn$Eoe4w?YsvCerV1+*+E7*jT-1oM z@roQut(cH#T$b4A!v5GOmXQAv*V%yC`UYoC9umWY!iz{*BI#Q^;cY{&L4tmRheB+A za0sCYesM3(Syd+yZJHY=39IW*?DP!~Nii+LC^9)=>*$QsS5@N`fySYlg53c>HYyW- zRK5!(7$0vm$zLbA1jMg&EC$n&cRP1UwPnUj$Q;3N3(2$s$$%b?)zKBJ^wqw+>W@Ci zZV|Au`5f%}-uDpBbIv(G@0eDa41_?B@AU-GJJ?V9qYU~_;XN8eV@2LX$o-%Jy_xeZA{57;9Gnp*%I z+lF+_1TEmI$a3pvL2+_)ZfINwGa)irmLx5u%$HC~XQ`JkvkTj<4#CABRg5E)(D)uM z1A#PHAg&nq%6%_r@xQ0Z@=O$3lkkP12MbIc`* ziyzr#k&H_V06wMM03278#y6i&pH;WiBzPUrbf!)VgFoqw8XKC|O(NaeUSWb$8(1s! zpk{p{-iWLF8IcX?qH#@lhVL64FqHl{z@{u(AS>%=>A#x>9+YSCG6S-&WPZS{X)7}YlB4+Z^q#R(b4}MB(*yn zm%qZIlGTj9v%?AS74QDe)0GtiAq!W`BKk~)ijWi+KA~v}M~#PgO^61t+hRHhhv0@M zwmmrM>za9=D=WL`Ync5VD%`s+58l_|P|F-QQ%Z1hhP9`;F=CSMT1*^lzDV`BZDS#_ zczcb1yptP>tBVaeg}thSDWUTSb|Nt?Nn7|I$JafNrDVro#Rn4aJ(-4l@f7GW39FHm3%q{#1-?q>7%)qjW3oUIsTjrRz4`X zTf7stQ?XhL2I*^mAdWx*(j_}_>$1adRMnjEng2{71}3KI*uf>BEs4xQo0b=zbr$QE zhppLyO7@rd0f`*m9FeL(X9b8qLG+K=>`@(?O8;HykV*W(?`Voy&g+<4{gDIX@c;*` z|7P=j>3XLrh~_ghnTutvhYWB>lf=mV-XPockeRuta9m76Kt4p%V9O+(98}&}%*ac) z45|1jD-qMt&2Ag>c>xrJAxpApX+!@^>sns#P$*;#kQwt1B7C}U&$#> zgO;lrNIhJj7zGekX7hhVjn?F04@3D-#>QA5_moZPgS?xbFR{HjrxZ2~z$@X()Q_RK z-vYypmmOXx>!%d~w%4)n>Nq?GiBQ#CGln`L;Ixxk+I>m^n!6stIPhA=K~#;wEa5}AAYn+!i&iKv*>g*ZI)WCY7+rc8 z(10#WtC*TW*3#BkL{TCaG6?I}%Z-)&&rG_McaqI$HCPQlQJj>31`u9-_Mk+|hp#Cu ziFE%cMz&K24r^+qX@bK|E5^ml<3erN-5ZX+Mxvx{T(O*PT645VncJh3$QXw(lC_=Q zHT>{3IgwrNY1URa9baFQZ?Jy|Gu(X2UxB=nK@pt1<$(Ncc^O&=MHctR64U8{S!RCb zJXX*^r5FMnwE%V^!v(scEYysU%`eHD=ojh|`&X~&$ZI?bu z$%)FS%C6Su?E$YlyoyYvSJn+s_vETk-0hH`6hMU!i2WaxOSq6qM~438@9zo#mt(Le zCRlyW6k-EuV*!&(4Bb$V020cznf7iuv7s>4!zvbp$%Cu0VXjjX3^h~L87QSfQG_k_ ztw~-r*fDF}55pACq?W;wG`O%alA$b^_iPeQ-Rk_>xor#lxZ_XfWh~1wQs`mt;e5ptLZM1pF<=UiSQ(4RJX7 z4R?yn!jcmuK?2G7M0O1Ew3yD7#$kAAVX8P2i2k>qi!v-aT9v(@xse|1d+sw$%!UXj z?alPqyI3yN;m4qDsiI>kkSBbhw}y8Mm)c{518{2(Q&~T(T*i- zI&qC6g{&}G**X8Y8$K*erWh6v2_4lmZaM)>U+xt(pv%l1v6z}isV&D+!WlR&Be=GA zqTdCj&^nwk_B#uK7venzlQA#lowhOAjHF*QRy(q?4UtH+hbtB7*~Wm3H*+ z@+^$Y4U&Fe^55W!B<~OnqtNv%kp1qg3BqW#-*3l+;OYMJeY8>#4=7QVbofupU48i_ za0h1-@lHYq%v?hL$6bE0GyVMOQ#lkzOe=ksc%y71exo3Js!%cYCN$Cmb&BWh4!w1NUv`!Xvat)d*7B7;aPjA^-;1oN$c zROQD>z8hseYH@Xsz{L{}5jJIdK(j9M-wWO-X`ZA^=w(+rXzvz0!C6H^dp05j^OjsB za7RRH16AW`Hvb!-8X*zf(O$$SEQglIg;RcAhhT;#(ZB5NXoyOx*ao+n*5sj}MRJB# zDMVC0CeDhom7HgsW#a$LGhOIaBmcCd_A}?v?s7@yBo8;o2#44dMI|3~O94FCMeptt z2*C`~GKdQ2wu=CfE)T@2re-;>192Nj9pjZsygXSyoa6_~@7Zn_#5gdc>`wRg5j8 z6ST2we^>h_3}R3yR8Z0tchni@c+fbaUT<2)rwK#^ofh_FC9UyM>6zFxPGIP0k<2es zB8hFW(MIpDh0TOrBym#Z#;9ygZ{dh*(e3iuYBQ+OF@(?@lZ(e?+*ty^I_+Y7B!Sk| z9|=H?s_+!uCTxDkn^^~uB{-tlNXE?GaMPUdu;+c!@VQ(5cSB$MiJ^lOZ?H?!t+!z zRpy8V7kDvIOdYrwx5+2~@gy`-3mJddIn+$i={yfrdEcnYSof2M^>%WLcIZpJR& zd9WOS{&MAsr5U>rWlutRYmzI|+Dcu0m}9+ONq3d&Hmc^0K56E5W~rWO{)PNHQJEJw zx~X~fs`*vbRYJxpOdjj(iLHNuSr{9YlGknM%z$=HxGjCM-jZWGI3aE9I5m1;&Mawi zs9ahfg?+p+%`v|ZsPHyv6^;dJIFlj14TShpg0U7iGTLk zyU}s|n5gDNrtCx*;-Adk6n$L8OVZp7&8(StNWq*;KKbY9anm=UwMkiaD%Nt^9pUD} z+3n*t@bXpUmwau_VMM*a49%k1UuSb<@W0YUZUj{Z%sa?{=tV6o@-fzDUrCrOaTb}G zDFn)1v2^UIb6igdz3Q>o`vW5O5bgfh9h;R+@Dg|%Yfe|4_~#$a-Y+pN#(a?^$Ly@q zNiO^+^ptGX_^1!QIQZBfy0W#0NeJAKzF$Ii4m1MGPg3)mqBI_DawYTEZX3E~e%_|y zLITMrS?&5)xvt*=|HcPE$r?cpT6)N^^DEuet36=X4YqQQ}! zpwp>wL&(pOh|2E9d@_vFt{W0;%SV z9saE7-ul=;N%NK?<%HzRocTv+j-HOy()Ogv4Fb1?{jgb*yX|vr@TLfeE!KP`j(^(? z(COidwtvPf{7v52s19@=k;dDL0?V=&Q)Vbh(wwy<9+)4FePV?~zK{J%x!MOp{)P9i zNI!DUcd-??E_m9M|Gqi(OMB)~ zgrS5Dj-a66r)3Zs#1>|<69q~GmA@*yScoE9A z)UGe)5H>y{xFhQx=|s3vyx~dcCZyfcb)(M$Ze}3%pqgJD8b15ua_fo%Z>SCSeHwBg z+r#rhtxPyvyaOBe^535{_qVJ^S>d);eP-G;ooQhdAQ&?Z&$K}(75LcFi2zr`My9F^ zLc@2l<=HgMdF-H2!G7rV7O=rNKa$_N^Y^-b-7M?A4}c(xJnE)@>~~4!^??~GKB)~2 zcj)!Wk*!N|?;{mZ=&L_|@*xy{j;Z5%C;&HqJTsU}{=-UI-35A5tkrbde=11>d}szW z73*;9%P?M->Fm5N#f4JNQ4d+Db-!PjJ$BIytVZoFgOieuwBS?)EqsP_^5MsIAj!(V za8#b;x6u40qg=liw&=X_IBHai6o7W2YdpoKhVpdBSW+vI0LNJG3(``-MGVb_rlP{W z^Ul*Z4K2`K2gRYZ`b5YTv|50^rjzqV-0owR~PIoCv`$T5!6n4>I-uJQJK8TlqQwrj9Ya{x9xyJ z(iwa*JMX+-0No`gWWna#xc9WDIa;4fsL=a1rQRwz6JSZBm2~Ly2nd!p*DW}TS}VyA zYO@I~ukl}|H_Vr{`B0tg(e!-suoUJxl2}8QWFHE~oRZD&iJz*~)fL{9UgSeDejzAG zjmHguT*SAM`1`m;!;lT_@T=LpgCLQq>E$CQs5`>{92xDo3;+EHQWL!!j-?0TU|M9K zs2E*6g9rW&V=iqg4IKAPT)-f@9mf zE&RKHXgyi&%iEEvfUcMS0K4$CsK8OlIz?2ZD||35Tdne?pzsPT-w_c$duoehH`7n} zAFb|7E(nSEk6QnkaHS16)5{<0=vL4uBDII!@Yt${Z5ZCgPwVQ{uYGMA;$5X^4UdQn z=qMfm|6cFhK!VO>U_s^UNfDOwC@aJh&KU~}=fewQL2~mrP1pH zkdgKC>tzJmCvVUvD+G#swmIX^-tV6h>+`Fu@#&w_pJ(DvCst^4;#Dhm6WTrx$!ybX z?g+(|7ezg-@@%yeiot@qGykgT4Mo&%$qK5Trqz@5W9@mtPa`MW8+Nf{EPHyu?2aqL zWp_b)7*F>`aoA2a-vTG#bD}i*HKF8_^|i^B2)ABAv>iHy2C~jo-AP{H!e782tM`RP z9%~W*sSnzOKVf}>lKiv4a%xj=qA4`5aC*}QPNV?VTE%YeFCD(D1TrWX9HlOB`FV;n$G$B+liIu^Jr1eh)7AHKvB-v6OEO>7<$oLM|CeQ z|Ks(yNzS_k!%)vpc)P5y$3@roo!%-tA(7lb$B8Z=@pa<2GwPP$uTo(B7L(AD3hoV2 zS^(C_O4CS&Bjq%BY=bI-?+m1tmQplB*$3q{7ZYxeZ;;=0dU15c55i2wzX$F*?Y8;z zDai zB3Oy22$O95-QBUo?X}wUW1W-XnO^x+1bJr{DYi@9_Qb_YA>cBc)zj7WP59mb6?Zxv zmtMEx08{)nM}CxqFyMnb^@}jv&G>{Ps`IXim2H)u@Gx_~2ads%79AFe@wof88RO?z zi?mgX;+@{fvjvorn8(QT;C**D<=Z@K|58U$tw%y?6tYh;IoKGL#|a<7z5LfAGti8V zUPMC$gVnri?kJ@-WTC(uz2GZ}J%@nr)~T)3uczj!%sOR*qnk$Fsc)%I>YSxd3v3xYU% zm5Y*=Bj14=sjsCmO@~i&dOp6=`fYrP(5o^9*uaqbpz9t&A5ZlC8o%tt>r{}~VB22&-MtY8FzbS>H{iN9Tq1QWv?4Se#eRE;Izn<(&vlfp z55M+$j*O1D6>ORBQ8|}snBbv~?Eaj?>0X$)K?V{%|LYZL^Ib?we%C5K?B0DZ&7>%P z&p(1liuWzRjKh)DwCWN*=pCduT~y4jlCWvJ8}2h|M9#R$D@m z$XF^_;(n&@H>a5 zI0Y9pl)tGiJ6xNRO0#3A`lZ~-a?*7oF{Om-;$&%=XF#?M+=oW+lZYTHYyS#_Ox*}} zYZFUDWYd?}s$}MO>nf;=d~=}<2J69;KJ ze9iJmDm~j&J^YNPN`9wh{f}vIo6{-dZq`~ssOR}HB3K@3zX|ILjcRfZmwD~*DZhy}0&C5yPV zi9C54(UVaf8pCcA*!5El35)&_C#@2_S|dw)5q)|sdJQkQ>iO7f+j;%VDf+7>+!l>; z+`;_sEsr*$0CE{JIO^Q+h7K&%qB}?rHx5!IP<*H8wJ$iAD)F2hmg|43#yK!%TJC6W zWQMCm@Cxfqqv#Ga?G3P)cs|UXKO}G$FN%PSm8i7EabifWfN^5b(583DXp`;4D={!$lzf^n$l2garA{l6coQGGoxVSTD_pzH z+|g@+2Ipeu$AuCRaMGR2 zU`nHGDM&zV(*ZpswT}<1NQ^C?GbrC_esMy1)OKS=qhVyL+J-dSXmKhEppPCV{UzAX zde8lfzsCw=JV{px%IAOH7b|H0I@}RH(x*Q*V4r?kr09O4EaN!Yc2fp{;%5?_byB4V zjWb*-KWHJUP@0XIa;{^)$5CB5euuf?6)-D&nxIrMj{9!LS%lkbej;v46|oAjQnjL8 zwouq!N!^6C&y}TU4Vvs~N&bQS`-~wCrVny!9$LUSiI$u&@kiIJJtAOr(kIe;FAfyH z@ASc{T&A^i0mq?lWbhaLqXID%L zl@79%ygAAmCmM$?{g;l}t70d%WCquGx+bn8b=)nl%bU~Jh%wGjRD&CvpiU@tODk04 zpLec)#|RBIP}TPGA!93Myl*KWP}4np)4348+QT1>h@W8|saY}<_!i|D;JFW`l!WTm zsPgn93?J=QLYg(e9|-1SzBq*g-7zQ(%yhL3?n-jNm_LJ_1vPuY=`>;AEy~<(tMl|h zr+Zf_=k=wZIMu{fJMn2oYr9+nMAE za>t95f92>2*E?5m{*4c8G;rnrScBlvl`@?7E=dW5dlL_- zgCAfgBbvy6Ls<R6LzojULQh_9r0>FQ0>0e zFHHWD{V(-E%=CMvB=)Pd>iTn8Ca);PpE*ljoLl(o z0#)WaJT}4ziHme%t4~Y3PYtNx>{)vD2RQToe`kJ~nN$3SS^a-C& - -.. thumbnail-parent-div-open - -.. raw:: html - -
- -.. only:: html - - .. image:: /tutorials/images/thumb/sphx_glr_template_tutorial_thumb.png - :alt: - - :ref:`sphx_glr_tutorials_template_tutorial.py` - -.. raw:: html - -
Template Tutorial
-
- - -.. raw:: html - -
- -.. only:: html - - .. image:: /tutorials/images/thumb/sphx_glr_rl_fundamentals_forge_tutorial_thumb.png - :alt: - - :ref:`sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py` - -.. raw:: html - -
RL Fundamentals Using Forge Terminology
-
- - -.. thumbnail-parent-div-close - -.. raw:: html - - - - -.. toctree:: - :hidden: - - /tutorials/template_tutorial - /tutorials/rl_fundamentals_forge_tutorial - diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json b/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json deleted file mode 100644 index c1c95a0b..00000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.codeobj.json +++ /dev/null @@ -1,215 +0,0 @@ -{ - "Any": [ - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "_SpecialForm" - }, - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "Any" - } - ], - "Dict": [ - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "_SpecialGenericAlias" - }, - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "Dict" - } - ], - "MockResponse": [ - { - "is_class": true, - "is_explicit": false, - "module": "__main__", - "module_short": "__main__", - "name": "MockResponse" - } - ], - "Optional": [ - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "_SpecialForm" - }, - { - "is_class": false, - "is_explicit": false, - "module": "typing", - "module_short": "typing", - "name": "Optional" - } - ], - "asyncio.gather": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - }, - { - "is_class": false, - "is_explicit": false, - "module": "asyncio", - "module_short": "asyncio", - "name": "gather" - } - ], - "conceptual_rl_step": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_advantages_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_dataset_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_episode_from_response": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_policy_service": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_reference_model_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_replay_buffer_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_reward_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "create_trainer_actor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "demonstrate_production_scaling": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "example_experience": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "dict" - } - ], - "show_infrastructure_challenges": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "function" - } - ], - "torch.cat": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "builtin_function_or_method" - }, - { - "is_class": false, - "is_explicit": false, - "module": "torch", - "module_short": "torch", - "name": "cat" - } - ], - "torch.tensor": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "builtin_function_or_method" - }, - { - "is_class": false, - "is_explicit": false, - "module": "torch", - "module_short": "torch", - "name": "tensor" - } - ] -} \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb b/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb deleted file mode 100644 index 3981a531..00000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.ipynb +++ /dev/null @@ -1,158 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n# RL Fundamentals Using Forge Terminology\n\n**Author:** [Your Name](https://github.com/yourusername)\n\n.. grid:: 2\n\n .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn\n :class-card: card-prerequisites\n\n * Core RL components in Forge (Dataset, Policy, Reward Model, etc.)\n * How RL concepts map to distributed Forge services\n * Building scalable RL training loops with fault tolerance\n * Resource management and independent scaling patterns\n\n .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites\n :class-card: card-prerequisites\n\n * PyTorch v2.0.0+\n * GPU access recommended\n * Basic understanding of reinforcement learning\n * Familiarity with async/await in Python\n\nThis tutorial teaches RL fundamentals using Forge's exact terminology and architecture.\nWe'll start with a simple math tutoring example to understand how traditional RL concepts\nmap to Forge's distributed service model.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Core RL Components in Forge\n\nLet's start with a simple math tutoring example to understand RL concepts\nwith the exact names Forge uses. Think of it as teaching an AI student:\n\n- **Dataset**: Provides questions (like \"What is 2+2?\")\n- **Policy**: The AI student being trained (generates answers like \"The answer is 4\")\n- **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95)\n- **Reference Model**: Copy of original student (prevents drift from baseline)\n- **Replay Buffer**: Notebook that stores experiences (question + answer + score)\n- **Trainer**: The tutor that improves the student based on experiences\n\nHere's how these components interact in a conceptual RL step:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import asyncio\nfrom typing import Any, Dict, Optional\n\nimport torch\n\n\ndef conceptual_rl_step():\n \"\"\"\n Conceptual example showing the RL learning flow.\n See apps/grpo/main.py for actual GRPO implementation.\n \"\"\"\n # 1. Get a math problem\n question = \"What is 2+2?\" # dataset.sample()\n\n # 2. Student generates answer\n answer = \"The answer is 4\" # policy.generate(question)\n\n # 3. Teacher grades it\n score = 0.95 # reward_model.evaluate(question, answer)\n\n # 4. Compare to original student\n baseline = 0.85 # reference_model.compute_logprobs(question, answer)\n\n # 5. Store the experience\n experience = {\n \"question\": question,\n \"answer\": answer,\n \"score\": score,\n \"baseline\": baseline,\n }\n # replay_buffer.add(experience)\n\n # 6. When enough experiences collected, improve student\n # trainer.train_step(batch) # Student gets better!\n\n return experience\n\n\nexample_experience = conceptual_rl_step()\nprint(\"Example RL experience:\", example_experience)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## From Concepts to Forge Services\n\nHere's the key insight: **Each RL component becomes a Forge service**.\nThe toy example above maps directly to Forge's distributed architecture:\n\n* Dataset \u2192 DatasetActor\n* Policy \u2192 Policy\n* Reward Model \u2192 RewardActor\n* Reference Model \u2192 ReferenceModel\n* Replay Buffer \u2192 ReplayBuffer\n* Trainer \u2192 RLTrainer\n\nLet's see how the conceptual example translates to actual Forge service calls:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]:\n \"\"\"\n RL step using actual Forge service APIs.\n This shows the same logic as conceptual_rl_step but with real service calls.\n \"\"\"\n # 1. Get a math problem - Using actual DatasetActor API\n sample = await services[\"dataloader\"].sample.call_one()\n prompt, target = sample[\"request\"], sample[\"target\"]\n\n # 2. Student generates answer - Using actual Policy API\n responses = await services[\"policy\"].generate.route(prompt=prompt)\n answer = responses[0].text\n\n # 3. Teacher grades it - Using actual RewardActor API\n score = await services[\"reward_actor\"].evaluate_response.route(\n prompt=prompt, response=answer, target=target\n )\n\n # 4. Compare to baseline - Using actual ReferenceModel API\n # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs\n input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids])\n ref_logprobs = await services[\"ref_model\"].forward.route(\n input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True\n )\n\n # 5. Store experience - Using actual Episode structure from apps/grpo/main.py\n episode = create_episode_from_response(responses[0], score, ref_logprobs, step)\n await services[\"replay_buffer\"].add.call_one(episode)\n\n # 6. Improve student - Using actual trainer pattern\n batch = await services[\"replay_buffer\"].sample.call_one(curr_policy_version=step)\n if batch is not None:\n inputs, targets = batch # GRPO returns (inputs, targets) tuple\n loss = await services[\"trainer\"].train_step.call(inputs, targets)\n\n # 7. Policy synchronization - Using actual weight update pattern\n await services[\"trainer\"].push_weights.call(step + 1)\n await services[\"policy\"].update_weights.fanout(step + 1)\n\n return loss\n\n return None\n\n\ndef create_episode_from_response(response, score, ref_logprobs, step):\n \"\"\"Helper function to create episode from response data\"\"\"\n return {\n \"response\": response,\n \"score\": score,\n \"ref_logprobs\": ref_logprobs,\n \"step\": step,\n }" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setting Up Forge Services\n\nHere's how to initialize the complete RL system with proper resource allocation.\nEach service can scale independently based on its computational needs:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "async def setup_forge_rl_system():\n \"\"\"\n Complete setup of Forge RL services with proper resource allocation.\n This example uses Qwen 3.1-1.7B model for demonstration.\n \"\"\"\n # Note: In actual Forge environment, imports would be:\n # from forge.actors.policy import Policy\n # from forge.actors.replay_buffer import ReplayBuffer\n # from forge.actors.reference_model import ReferenceModel\n # from forge.actors.trainer import RLTrainer\n # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages\n # from forge.data.rewards import MathReward, ThinkingReward\n\n model = \"Qwen/Qwen3-1.7B\"\n group_size = 1\n\n # Initialize all services with appropriate resource allocation\n services = await asyncio.gather(\n # Dataset actor (CPU intensive for I/O)\n create_dataset_actor(model),\n # Policy service (GPU for inference)\n create_policy_service(model, group_size),\n # Trainer actor (GPU for training)\n create_trainer_actor(model),\n # Replay buffer (CPU for memory management)\n create_replay_buffer_actor(),\n # Advantage computation (CPU)\n create_advantages_actor(),\n # Reference model (GPU for baseline)\n create_reference_model_actor(model),\n # Reward actor (CPU/small GPU for evaluation)\n create_reward_actor(),\n )\n\n service_names = [\n \"dataloader\",\n \"policy\",\n \"trainer\",\n \"replay_buffer\",\n \"compute_advantages\",\n \"ref_model\",\n \"reward_actor\",\n ]\n\n return dict(zip(service_names, services))\n\n\n# Service creation functions (would use actual Forge APIs)\nasync def create_dataset_actor(model):\n \"\"\"DatasetActor for loading training data\"\"\"\n return {\n \"name\": \"DatasetActor\",\n \"config\": {\n \"path\": \"openai/gsm8k\",\n \"revision\": \"main\",\n \"data_split\": \"train\",\n \"streaming\": True,\n \"model\": model,\n },\n \"resources\": \"CPU\",\n \"sample\": lambda: {\n \"call_one\": lambda: {\"request\": \"What is 2+2?\", \"target\": \"4\"}\n },\n }\n\n\nasync def create_policy_service(model, group_size):\n \"\"\"Policy service for text generation\"\"\"\n return {\n \"name\": \"Policy\",\n \"config\": {\n \"engine_config\": {\n \"model\": model,\n \"tensor_parallel_size\": 1,\n \"pipeline_parallel_size\": 1,\n \"enforce_eager\": False,\n },\n \"sampling_config\": {\n \"n\": group_size,\n \"max_tokens\": 16,\n \"temperature\": 1.0,\n \"top_p\": 1.0,\n },\n },\n \"resources\": \"GPU\",\n \"generate\": lambda: {\"route\": lambda prompt: [MockResponse()]},\n }\n\n\nasync def create_trainer_actor(model):\n \"\"\"RLTrainer for policy optimization\"\"\"\n return {\n \"name\": \"RLTrainer\",\n \"config\": {\n \"model\": {\n \"name\": \"qwen3\",\n \"flavor\": \"1.7B\",\n \"hf_assets_path\": f\"hf://{model}\",\n },\n \"optimizer\": {\"name\": \"AdamW\", \"lr\": 1e-5},\n \"training\": {\"local_batch_size\": 2, \"seq_len\": 2048},\n },\n \"resources\": \"GPU\",\n \"train_step\": lambda: {\"call\": lambda inputs, targets: 0.5},\n }\n\n\nasync def create_replay_buffer_actor():\n \"\"\"ReplayBuffer for experience storage\"\"\"\n return {\n \"name\": \"ReplayBuffer\",\n \"config\": {\"batch_size\": 2, \"max_policy_age\": 1, \"dp_size\": 1},\n \"resources\": \"CPU\",\n \"add\": lambda: {\"call_one\": lambda episode: None},\n \"sample\": lambda: {\"call_one\": lambda curr_policy_version: ([], [])},\n }\n\n\nasync def create_advantages_actor():\n \"\"\"ComputeAdvantages for advantage estimation\"\"\"\n return {\"name\": \"ComputeAdvantages\", \"resources\": \"CPU\"}\n\n\nasync def create_reference_model_actor(model):\n \"\"\"ReferenceModel for baseline computation\"\"\"\n return {\n \"name\": \"ReferenceModel\",\n \"config\": {\n \"model\": {\n \"name\": \"qwen3\",\n \"flavor\": \"1.7B\",\n \"hf_assets_path\": f\"hf://{model}\",\n },\n \"training\": {\"dtype\": \"bfloat16\"},\n },\n \"resources\": \"GPU\",\n \"forward\": lambda: {\n \"route\": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor(\n [0.1, 0.2]\n )\n },\n }\n\n\nasync def create_reward_actor():\n \"\"\"RewardActor for response evaluation\"\"\"\n return {\n \"name\": \"RewardActor\",\n \"config\": {\"reward_functions\": [\"MathReward\", \"ThinkingReward\"]},\n \"resources\": \"CPU\",\n \"evaluate_response\": lambda: {\"route\": lambda prompt, response, target: 0.95},\n }\n\n\nclass MockResponse:\n \"\"\"Mock response object for demonstration\"\"\"\n\n def __init__(self):\n self.text = \"The answer is 4\"\n self.prompt_ids = torch.tensor([1, 2, 3])\n self.token_ids = torch.tensor([4, 5, 6])\n\n\n# Demonstrate the setup\nprint(\"Setting up Forge RL system...\")\n# services = await setup_forge_rl_system() # Would work in async context\nprint(\"Forge services configured with independent scaling capabilities\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Why Forge Architecture Matters\n\nTraditional ML infrastructure fails for RL because each component has\ndifferent resource needs, scaling patterns, and failure modes:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "def show_infrastructure_challenges():\n \"\"\"\n Demonstrate why traditional monolithic RL fails and how Forge solves it.\n \"\"\"\n print(\"=== Infrastructure Challenges ===\\n\")\n\n print(\"Problem 1: Different Resource Needs\")\n resource_requirements = {\n \"Policy (Student AI)\": {\n \"generates\": \"'The answer is 4'\",\n \"needs\": \"Large GPU memory\",\n \"scaling\": \"Multiple replicas for speed\",\n },\n \"Reward Model (Teacher)\": {\n \"scores\": \"answers: 0.95\",\n \"needs\": \"Moderate compute\",\n \"scaling\": \"CPU or small GPU\",\n },\n \"Trainer (Tutor)\": {\n \"improves\": \"student weights\",\n \"needs\": \"Massive GPU compute\",\n \"scaling\": \"Distributed training\",\n },\n \"Dataset (Question Bank)\": {\n \"provides\": \"'What is 2+2?'\",\n \"needs\": \"CPU intensive I/O\",\n \"scaling\": \"High memory bandwidth\",\n },\n }\n\n for component, reqs in resource_requirements.items():\n print(f\"{component}:\")\n for key, value in reqs.items():\n print(f\" {key}: {value}\")\n print()\n\n print(\"Problem 2: Coordination Complexity\")\n print(\"Unlike supervised learning with independent batches,\")\n print(\"RL requires complex coordination between components:\")\n print(\"- Policy waits idle while reward model works\")\n print(\"- Training waits for single episode (batch size = 1)\")\n print(\"- Everything stops if any component fails\")\n print()\n\n print(\"=== Forge Solutions ===\\n\")\n\n print(\"\u2705 Automatic Resource Management\")\n print(\"- Routing to least loaded replica\")\n print(\"- GPU memory management\")\n print(\"- Batch optimization\")\n print(\"- Failure recovery\")\n print(\"- Auto-scaling based on demand\")\n print()\n\n print(\"\u2705 Independent Scaling\")\n print(\"- Policy: num_replicas=8 for high inference demand\")\n print(\"- RewardActor: num_replicas=16 for parallel evaluation\")\n print(\"- Trainer: Multiple actors for distributed training\")\n print()\n\n print(\"\u2705 Fault Tolerance\")\n print(\"- Automatic routing to healthy replicas\")\n print(\"- Background replica respawn\")\n print(\"- Graceful degradation\")\n print(\"- System continues during component failures\")\n\n\nshow_infrastructure_challenges()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Production Scaling Example\n\nHere's how you would scale the system for production workloads:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "def demonstrate_production_scaling():\n \"\"\"\n Show how Forge services scale independently for production.\n \"\"\"\n print(\"=== Production Scaling Configuration ===\\n\")\n\n scaling_config = {\n \"Policy Service\": {\n \"replicas\": 8,\n \"reason\": \"High inference demand from multiple training runs\",\n \"resources\": \"GPU-heavy instances\",\n },\n \"RewardActor Service\": {\n \"replicas\": 16,\n \"reason\": \"Parallel evaluation of many responses\",\n \"resources\": \"CPU/small GPU instances\",\n },\n \"Trainer Actor\": {\n \"replicas\": 4,\n \"reason\": \"Distributed training across multiple nodes\",\n \"resources\": \"Large GPU clusters\",\n },\n \"Dataset Actor\": {\n \"replicas\": 2,\n \"reason\": \"I/O intensive data loading\",\n \"resources\": \"High-bandwidth CPU instances\",\n },\n \"ReplayBuffer Actor\": {\n \"replicas\": 1,\n \"reason\": \"Centralized experience storage\",\n \"resources\": \"High-memory instances\",\n },\n }\n\n for service, config in scaling_config.items():\n print(f\"{service}:\")\n print(f\" Replicas: {config['replicas']}\")\n print(f\" Reason: {config['reason']}\")\n print(f\" Resources: {config['resources']}\")\n print()\n\n print(\"Key Benefits:\")\n print(\"- Each service scales based on its bottlenecks\")\n print(\"- Resource utilization is optimized\")\n print(\"- Costs are minimized (no idle GPUs)\")\n print(\"- System maintains performance under load\")\n\n\ndemonstrate_production_scaling()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Complete RL Training Loop\n\nHere's a complete example showing multiple RL training steps:\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "async def complete_rl_training_example(num_steps: int = 5):\n \"\"\"\n Complete RL training loop using Forge services.\n \"\"\"\n print(f\"=== Running {num_steps} RL Training Steps ===\\n\")\n\n # Setup services (mock for demonstration)\n services = {\n \"dataloader\": await create_dataset_actor(\"Qwen/Qwen3-1.7B\"),\n \"policy\": await create_policy_service(\"Qwen/Qwen3-1.7B\", 1),\n \"trainer\": await create_trainer_actor(\"Qwen/Qwen3-1.7B\"),\n \"replay_buffer\": await create_replay_buffer_actor(),\n \"ref_model\": await create_reference_model_actor(\"Qwen/Qwen3-1.7B\"),\n \"reward_actor\": await create_reward_actor(),\n }\n\n losses = []\n\n for step in range(num_steps):\n print(f\"Step {step + 1}:\")\n\n # Simulate the RL step (would use actual forge_rl_step in practice)\n sample = await services[\"dataloader\"][\"sample\"]()[\"call_one\"]()\n print(f\" Question: {sample['request']}\")\n print(f\" Target: {sample['target']}\")\n\n # Generate response\n responses = await services[\"policy\"][\"generate\"]()[\"route\"](sample[\"request\"])\n print(f\" Generated: {responses[0].text}\")\n\n # Get reward\n score = await services[\"reward_actor\"][\"evaluate_response\"]()[\"route\"](\n sample[\"request\"], responses[0].text, sample[\"target\"]\n )\n print(f\" Reward: {score}\")\n\n # Simulate training (every few steps when buffer has enough data)\n if step >= 2: # Start training after accumulating some experience\n loss = await services[\"trainer\"][\"train_step\"]()[\"call\"]([], [])\n losses.append(loss)\n print(f\" Training Loss: {loss:.4f}\")\n\n print()\n\n print(f\"Training completed! Average loss: {sum(losses)/len(losses):.4f}\")\n return losses\n\n\n# Run the example (would work in async context)\nprint(\"Complete RL training example:\")\nprint(\"(In real usage, run: await complete_rl_training_example(5))\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Conclusion\n\nThis tutorial demonstrated how RL fundamentals map to Forge's distributed\nservice architecture. Key takeaways:\n\n1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.)\n becomes an independent, scalable Forge service\n\n2. **Resource Optimization**: Services scale independently based on their\n computational needs (GPU for inference/training, CPU for data/rewards)\n\n3. **Fault Tolerance**: Individual service failures don't stop the entire\n training pipeline - Forge handles routing and recovery automatically\n\n4. **Simple Interface**: Complex distributed systems are hidden behind\n simple async function calls\n\nThe same RL logic that works conceptually scales to production workloads\nwithout infrastructure code - Forge handles distribution, scaling, and\nfault tolerance automatically.\n\n## Further Reading\n\n* [Forge Architecture Documentation](#)\n* [GRPO Implementation (apps/grpo/main.py)](#)\n* [Forge Service APIs](#)\n* [Production RL Scaling Guide](#)\n\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.18" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py deleted file mode 100644 index 3e77639a..00000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py +++ /dev/null @@ -1,558 +0,0 @@ -""" -RL Fundamentals Using Forge Terminology -======================================== - -**Author:** `Your Name `_ - -.. grid:: 2 - - .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn - :class-card: card-prerequisites - - * Core RL components in Forge (Dataset, Policy, Reward Model, etc.) - * How RL concepts map to distributed Forge services - * Building scalable RL training loops with fault tolerance - * Resource management and independent scaling patterns - - .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - :class-card: card-prerequisites - - * PyTorch v2.0.0+ - * GPU access recommended - * Basic understanding of reinforcement learning - * Familiarity with async/await in Python - -This tutorial teaches RL fundamentals using Forge's exact terminology and architecture. -We'll start with a simple math tutoring example to understand how traditional RL concepts -map to Forge's distributed service model. - -""" - -######################################################################## -# Core RL Components in Forge -# ---------------------------- -# -# Let's start with a simple math tutoring example to understand RL concepts -# with the exact names Forge uses. Think of it as teaching an AI student: -# -# - **Dataset**: Provides questions (like "What is 2+2?") -# - **Policy**: The AI student being trained (generates answers like "The answer is 4") -# - **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95) -# - **Reference Model**: Copy of original student (prevents drift from baseline) -# - **Replay Buffer**: Notebook that stores experiences (question + answer + score) -# - **Trainer**: The tutor that improves the student based on experiences -# -# Here's how these components interact in a conceptual RL step: - -import asyncio -from typing import Any, Dict, Optional - -import torch - - -def conceptual_rl_step(): - """ - Conceptual example showing the RL learning flow. - See apps/grpo/main.py for actual GRPO implementation. - """ - # 1. Get a math problem - question = "What is 2+2?" # dataset.sample() - - # 2. Student generates answer - answer = "The answer is 4" # policy.generate(question) - - # 3. Teacher grades it - score = 0.95 # reward_model.evaluate(question, answer) - - # 4. Compare to original student - baseline = 0.85 # reference_model.compute_logprobs(question, answer) - - # 5. Store the experience - experience = { - "question": question, - "answer": answer, - "score": score, - "baseline": baseline, - } - # replay_buffer.add(experience) - - # 6. When enough experiences collected, improve student - # trainer.train_step(batch) # Student gets better! - - return experience - - -example_experience = conceptual_rl_step() -print("Example RL experience:", example_experience) - -######################################################################## -# From Concepts to Forge Services -# -------------------------------- -# -# Here's the key insight: **Each RL component becomes a Forge service**. -# The toy example above maps directly to Forge's distributed architecture: -# -# * Dataset → DatasetActor -# * Policy → Policy -# * Reward Model → RewardActor -# * Reference Model → ReferenceModel -# * Replay Buffer → ReplayBuffer -# * Trainer → RLTrainer -# -# Let's see how the conceptual example translates to actual Forge service calls: - - -async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]: - """ - RL step using actual Forge service APIs. - This shows the same logic as conceptual_rl_step but with real service calls. - """ - # 1. Get a math problem - Using actual DatasetActor API - sample = await services["dataloader"].sample.call_one() - prompt, target = sample["request"], sample["target"] - - # 2. Student generates answer - Using actual Policy API - responses = await services["policy"].generate.route(prompt=prompt) - answer = responses[0].text - - # 3. Teacher grades it - Using actual RewardActor API - score = await services["reward_actor"].evaluate_response.route( - prompt=prompt, response=answer, target=target - ) - - # 4. Compare to baseline - Using actual ReferenceModel API - # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs - input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids]) - ref_logprobs = await services["ref_model"].forward.route( - input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True - ) - - # 5. Store experience - Using actual Episode structure from apps/grpo/main.py - episode = create_episode_from_response(responses[0], score, ref_logprobs, step) - await services["replay_buffer"].add.call_one(episode) - - # 6. Improve student - Using actual trainer pattern - batch = await services["replay_buffer"].sample.call_one(curr_policy_version=step) - if batch is not None: - inputs, targets = batch # GRPO returns (inputs, targets) tuple - loss = await services["trainer"].train_step.call(inputs, targets) - - # 7. Policy synchronization - Using actual weight update pattern - await services["trainer"].push_weights.call(step + 1) - await services["policy"].update_weights.fanout(step + 1) - - return loss - - return None - - -def create_episode_from_response(response, score, ref_logprobs, step): - """Helper function to create episode from response data""" - return { - "response": response, - "score": score, - "ref_logprobs": ref_logprobs, - "step": step, - } - - -######################################################################## -# Setting Up Forge Services -# -------------------------- -# -# Here's how to initialize the complete RL system with proper resource allocation. -# Each service can scale independently based on its computational needs: - - -async def setup_forge_rl_system(): - """ - Complete setup of Forge RL services with proper resource allocation. - This example uses Qwen 3.1-1.7B model for demonstration. - """ - # Note: In actual Forge environment, imports would be: - # from forge.actors.policy import Policy - # from forge.actors.replay_buffer import ReplayBuffer - # from forge.actors.reference_model import ReferenceModel - # from forge.actors.trainer import RLTrainer - # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages - # from forge.data.rewards import MathReward, ThinkingReward - - model = "Qwen/Qwen3-1.7B" - group_size = 1 - - # Initialize all services with appropriate resource allocation - services = await asyncio.gather( - # Dataset actor (CPU intensive for I/O) - create_dataset_actor(model), - # Policy service (GPU for inference) - create_policy_service(model, group_size), - # Trainer actor (GPU for training) - create_trainer_actor(model), - # Replay buffer (CPU for memory management) - create_replay_buffer_actor(), - # Advantage computation (CPU) - create_advantages_actor(), - # Reference model (GPU for baseline) - create_reference_model_actor(model), - # Reward actor (CPU/small GPU for evaluation) - create_reward_actor(), - ) - - service_names = [ - "dataloader", - "policy", - "trainer", - "replay_buffer", - "compute_advantages", - "ref_model", - "reward_actor", - ] - - return dict(zip(service_names, services)) - - -# Service creation functions (would use actual Forge APIs) -async def create_dataset_actor(model): - """DatasetActor for loading training data""" - return { - "name": "DatasetActor", - "config": { - "path": "openai/gsm8k", - "revision": "main", - "data_split": "train", - "streaming": True, - "model": model, - }, - "resources": "CPU", - "sample": lambda: { - "call_one": lambda: {"request": "What is 2+2?", "target": "4"} - }, - } - - -async def create_policy_service(model, group_size): - """Policy service for text generation""" - return { - "name": "Policy", - "config": { - "engine_config": { - "model": model, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "enforce_eager": False, - }, - "sampling_config": { - "n": group_size, - "max_tokens": 16, - "temperature": 1.0, - "top_p": 1.0, - }, - }, - "resources": "GPU", - "generate": lambda: {"route": lambda prompt: [MockResponse()]}, - } - - -async def create_trainer_actor(model): - """RLTrainer for policy optimization""" - return { - "name": "RLTrainer", - "config": { - "model": { - "name": "qwen3", - "flavor": "1.7B", - "hf_assets_path": f"hf://{model}", - }, - "optimizer": {"name": "AdamW", "lr": 1e-5}, - "training": {"local_batch_size": 2, "seq_len": 2048}, - }, - "resources": "GPU", - "train_step": lambda: {"call": lambda inputs, targets: 0.5}, - } - - -async def create_replay_buffer_actor(): - """ReplayBuffer for experience storage""" - return { - "name": "ReplayBuffer", - "config": {"batch_size": 2, "max_policy_age": 1, "dp_size": 1}, - "resources": "CPU", - "add": lambda: {"call_one": lambda episode: None}, - "sample": lambda: {"call_one": lambda curr_policy_version: ([], [])}, - } - - -async def create_advantages_actor(): - """ComputeAdvantages for advantage estimation""" - return {"name": "ComputeAdvantages", "resources": "CPU"} - - -async def create_reference_model_actor(model): - """ReferenceModel for baseline computation""" - return { - "name": "ReferenceModel", - "config": { - "model": { - "name": "qwen3", - "flavor": "1.7B", - "hf_assets_path": f"hf://{model}", - }, - "training": {"dtype": "bfloat16"}, - }, - "resources": "GPU", - "forward": lambda: { - "route": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor( - [0.1, 0.2] - ) - }, - } - - -async def create_reward_actor(): - """RewardActor for response evaluation""" - return { - "name": "RewardActor", - "config": {"reward_functions": ["MathReward", "ThinkingReward"]}, - "resources": "CPU", - "evaluate_response": lambda: {"route": lambda prompt, response, target: 0.95}, - } - - -class MockResponse: - """Mock response object for demonstration""" - - def __init__(self): - self.text = "The answer is 4" - self.prompt_ids = torch.tensor([1, 2, 3]) - self.token_ids = torch.tensor([4, 5, 6]) - - -# Demonstrate the setup -print("Setting up Forge RL system...") -# services = await setup_forge_rl_system() # Would work in async context -print("Forge services configured with independent scaling capabilities") - -######################################################################## -# Why Forge Architecture Matters -# ------------------------------- -# -# Traditional ML infrastructure fails for RL because each component has -# different resource needs, scaling patterns, and failure modes: - - -def show_infrastructure_challenges(): - """ - Demonstrate why traditional monolithic RL fails and how Forge solves it. - """ - print("=== Infrastructure Challenges ===\n") - - print("Problem 1: Different Resource Needs") - resource_requirements = { - "Policy (Student AI)": { - "generates": "'The answer is 4'", - "needs": "Large GPU memory", - "scaling": "Multiple replicas for speed", - }, - "Reward Model (Teacher)": { - "scores": "answers: 0.95", - "needs": "Moderate compute", - "scaling": "CPU or small GPU", - }, - "Trainer (Tutor)": { - "improves": "student weights", - "needs": "Massive GPU compute", - "scaling": "Distributed training", - }, - "Dataset (Question Bank)": { - "provides": "'What is 2+2?'", - "needs": "CPU intensive I/O", - "scaling": "High memory bandwidth", - }, - } - - for component, reqs in resource_requirements.items(): - print(f"{component}:") - for key, value in reqs.items(): - print(f" {key}: {value}") - print() - - print("Problem 2: Coordination Complexity") - print("Unlike supervised learning with independent batches,") - print("RL requires complex coordination between components:") - print("- Policy waits idle while reward model works") - print("- Training waits for single episode (batch size = 1)") - print("- Everything stops if any component fails") - print() - - print("=== Forge Solutions ===\n") - - print("✅ Automatic Resource Management") - print("- Routing to least loaded replica") - print("- GPU memory management") - print("- Batch optimization") - print("- Failure recovery") - print("- Auto-scaling based on demand") - print() - - print("✅ Independent Scaling") - print("- Policy: num_replicas=8 for high inference demand") - print("- RewardActor: num_replicas=16 for parallel evaluation") - print("- Trainer: Multiple actors for distributed training") - print() - - print("✅ Fault Tolerance") - print("- Automatic routing to healthy replicas") - print("- Background replica respawn") - print("- Graceful degradation") - print("- System continues during component failures") - - -show_infrastructure_challenges() - -######################################################################## -# Production Scaling Example -# --------------------------- -# -# Here's how you would scale the system for production workloads: - - -def demonstrate_production_scaling(): - """ - Show how Forge services scale independently for production. - """ - print("=== Production Scaling Configuration ===\n") - - scaling_config = { - "Policy Service": { - "replicas": 8, - "reason": "High inference demand from multiple training runs", - "resources": "GPU-heavy instances", - }, - "RewardActor Service": { - "replicas": 16, - "reason": "Parallel evaluation of many responses", - "resources": "CPU/small GPU instances", - }, - "Trainer Actor": { - "replicas": 4, - "reason": "Distributed training across multiple nodes", - "resources": "Large GPU clusters", - }, - "Dataset Actor": { - "replicas": 2, - "reason": "I/O intensive data loading", - "resources": "High-bandwidth CPU instances", - }, - "ReplayBuffer Actor": { - "replicas": 1, - "reason": "Centralized experience storage", - "resources": "High-memory instances", - }, - } - - for service, config in scaling_config.items(): - print(f"{service}:") - print(f" Replicas: {config['replicas']}") - print(f" Reason: {config['reason']}") - print(f" Resources: {config['resources']}") - print() - - print("Key Benefits:") - print("- Each service scales based on its bottlenecks") - print("- Resource utilization is optimized") - print("- Costs are minimized (no idle GPUs)") - print("- System maintains performance under load") - - -demonstrate_production_scaling() - -######################################################################## -# Complete RL Training Loop -# -------------------------- -# -# Here's a complete example showing multiple RL training steps: - - -async def complete_rl_training_example(num_steps: int = 5): - """ - Complete RL training loop using Forge services. - """ - print(f"=== Running {num_steps} RL Training Steps ===\n") - - # Setup services (mock for demonstration) - services = { - "dataloader": await create_dataset_actor("Qwen/Qwen3-1.7B"), - "policy": await create_policy_service("Qwen/Qwen3-1.7B", 1), - "trainer": await create_trainer_actor("Qwen/Qwen3-1.7B"), - "replay_buffer": await create_replay_buffer_actor(), - "ref_model": await create_reference_model_actor("Qwen/Qwen3-1.7B"), - "reward_actor": await create_reward_actor(), - } - - losses = [] - - for step in range(num_steps): - print(f"Step {step + 1}:") - - # Simulate the RL step (would use actual forge_rl_step in practice) - sample = await services["dataloader"]["sample"]()["call_one"]() - print(f" Question: {sample['request']}") - print(f" Target: {sample['target']}") - - # Generate response - responses = await services["policy"]["generate"]()["route"](sample["request"]) - print(f" Generated: {responses[0].text}") - - # Get reward - score = await services["reward_actor"]["evaluate_response"]()["route"]( - sample["request"], responses[0].text, sample["target"] - ) - print(f" Reward: {score}") - - # Simulate training (every few steps when buffer has enough data) - if step >= 2: # Start training after accumulating some experience - loss = await services["trainer"]["train_step"]()["call"]([], []) - losses.append(loss) - print(f" Training Loss: {loss:.4f}") - - print() - - print(f"Training completed! Average loss: {sum(losses)/len(losses):.4f}") - return losses - - -# Run the example (would work in async context) -print("Complete RL training example:") -print("(In real usage, run: await complete_rl_training_example(5))") - -######################################################################## -# Conclusion -# ---------- -# -# This tutorial demonstrated how RL fundamentals map to Forge's distributed -# service architecture. Key takeaways: -# -# 1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.) -# becomes an independent, scalable Forge service -# -# 2. **Resource Optimization**: Services scale independently based on their -# computational needs (GPU for inference/training, CPU for data/rewards) -# -# 3. **Fault Tolerance**: Individual service failures don't stop the entire -# training pipeline - Forge handles routing and recovery automatically -# -# 4. **Simple Interface**: Complex distributed systems are hidden behind -# simple async function calls -# -# The same RL logic that works conceptually scales to production workloads -# without infrastructure code - Forge handles distribution, scaling, and -# fault tolerance automatically. -# -# Further Reading -# --------------- -# -# * `Forge Architecture Documentation <#>`_ -# * `GRPO Implementation (apps/grpo/main.py) <#>`_ -# * `Forge Service APIs <#>`_ -# * `Production RL Scaling Guide <#>`_ - diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 b/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 deleted file mode 100644 index d93f4a3a..00000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.py.md5 +++ /dev/null @@ -1 +0,0 @@ -d1dbe1df9283c920a3039b06b0d5ce4d \ No newline at end of file diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst b/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst deleted file mode 100644 index 95ad7c0b..00000000 --- a/docs/source/tutorials/rl_fundamentals_forge_tutorial.rst +++ /dev/null @@ -1,788 +0,0 @@ - -.. DO NOT EDIT. -.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. -.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: -.. "tutorials/rl_fundamentals_forge_tutorial.py" -.. LINE NUMBERS ARE GIVEN BELOW. - -.. only:: html - - .. note:: - :class: sphx-glr-download-link-note - - :ref:`Go to the end ` - to download the full example code. - -.. rst-class:: sphx-glr-example-title - -.. _sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py: - - -RL Fundamentals Using Forge Terminology -======================================== - -**Author:** `Your Name `_ - -.. grid:: 2 - - .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn - :class-card: card-prerequisites - - * Core RL components in Forge (Dataset, Policy, Reward Model, etc.) - * How RL concepts map to distributed Forge services - * Building scalable RL training loops with fault tolerance - * Resource management and independent scaling patterns - - .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - :class-card: card-prerequisites - - * PyTorch v2.0.0+ - * GPU access recommended - * Basic understanding of reinforcement learning - * Familiarity with async/await in Python - -This tutorial teaches RL fundamentals using Forge's exact terminology and architecture. -We'll start with a simple math tutoring example to understand how traditional RL concepts -map to Forge's distributed service model. - -.. GENERATED FROM PYTHON SOURCE LINES 32-46 - -Core RL Components in Forge ----------------------------- - -Let's start with a simple math tutoring example to understand RL concepts -with the exact names Forge uses. Think of it as teaching an AI student: - -- **Dataset**: Provides questions (like "What is 2+2?") -- **Policy**: The AI student being trained (generates answers like "The answer is 4") -- **Reward Model**: The teacher that evaluates answer quality (gives scores like 0.95) -- **Reference Model**: Copy of original student (prevents drift from baseline) -- **Replay Buffer**: Notebook that stores experiences (question + answer + score) -- **Trainer**: The tutor that improves the student based on experiences - -Here's how these components interact in a conceptual RL step: - -.. GENERATED FROM PYTHON SOURCE LINES 46-88 - -.. code-block:: Python - - - import asyncio - from typing import Any, Dict, Optional - - import torch - - - def conceptual_rl_step(): - """ - Conceptual example showing the RL learning flow. - See apps/grpo/main.py for actual GRPO implementation. - """ - # 1. Get a math problem - question = "What is 2+2?" # dataset.sample() - - # 2. Student generates answer - answer = "The answer is 4" # policy.generate(question) - - # 3. Teacher grades it - score = 0.95 # reward_model.evaluate(question, answer) - - # 4. Compare to original student - baseline = 0.85 # reference_model.compute_logprobs(question, answer) - - # 5. Store the experience - experience = { - "question": question, - "answer": answer, - "score": score, - "baseline": baseline, - } - # replay_buffer.add(experience) - - # 6. When enough experiences collected, improve student - # trainer.train_step(batch) # Student gets better! - - return experience - - - example_experience = conceptual_rl_step() - print("Example RL experience:", example_experience) - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - Example RL experience: {'question': 'What is 2+2?', 'answer': 'The answer is 4', 'score': 0.95, 'baseline': 0.85} - - - - -.. GENERATED FROM PYTHON SOURCE LINES 89-103 - -From Concepts to Forge Services --------------------------------- - -Here's the key insight: **Each RL component becomes a Forge service**. -The toy example above maps directly to Forge's distributed architecture: - -* Dataset → DatasetActor -* Policy → Policy -* Reward Model → RewardActor -* Reference Model → ReferenceModel -* Replay Buffer → ReplayBuffer -* Trainer → RLTrainer - -Let's see how the conceptual example translates to actual Forge service calls: - -.. GENERATED FROM PYTHON SOURCE LINES 103-159 - -.. code-block:: Python - - - - async def forge_rl_step(services: Dict[str, Any], step: int) -> Optional[float]: - """ - RL step using actual Forge service APIs. - This shows the same logic as conceptual_rl_step but with real service calls. - """ - # 1. Get a math problem - Using actual DatasetActor API - sample = await services["dataloader"].sample.call_one() - prompt, target = sample["request"], sample["target"] - - # 2. Student generates answer - Using actual Policy API - responses = await services["policy"].generate.route(prompt=prompt) - answer = responses[0].text - - # 3. Teacher grades it - Using actual RewardActor API - score = await services["reward_actor"].evaluate_response.route( - prompt=prompt, response=answer, target=target - ) - - # 4. Compare to baseline - Using actual ReferenceModel API - # Note: ReferenceModel.forward requires input_ids, max_req_tokens, return_logprobs - input_ids = torch.cat([responses[0].prompt_ids, responses[0].token_ids]) - ref_logprobs = await services["ref_model"].forward.route( - input_ids.unsqueeze(0), max_req_tokens=512, return_logprobs=True - ) - - # 5. Store experience - Using actual Episode structure from apps/grpo/main.py - episode = create_episode_from_response(responses[0], score, ref_logprobs, step) - await services["replay_buffer"].add.call_one(episode) - - # 6. Improve student - Using actual trainer pattern - batch = await services["replay_buffer"].sample.call_one(curr_policy_version=step) - if batch is not None: - inputs, targets = batch # GRPO returns (inputs, targets) tuple - loss = await services["trainer"].train_step.call(inputs, targets) - - # 7. Policy synchronization - Using actual weight update pattern - await services["trainer"].push_weights.call(step + 1) - await services["policy"].update_weights.fanout(step + 1) - - return loss - - return None - - - def create_episode_from_response(response, score, ref_logprobs, step): - """Helper function to create episode from response data""" - return { - "response": response, - "score": score, - "ref_logprobs": ref_logprobs, - "step": step, - } - - - - - - - - - -.. GENERATED FROM PYTHON SOURCE LINES 160-165 - -Setting Up Forge Services --------------------------- - -Here's how to initialize the complete RL system with proper resource allocation. -Each service can scale independently based on its computational needs: - -.. GENERATED FROM PYTHON SOURCE LINES 165-335 - -.. code-block:: Python - - - - async def setup_forge_rl_system(): - """ - Complete setup of Forge RL services with proper resource allocation. - This example uses Qwen 3.1-1.7B model for demonstration. - """ - # Note: In actual Forge environment, imports would be: - # from forge.actors.policy import Policy - # from forge.actors.replay_buffer import ReplayBuffer - # from forge.actors.reference_model import ReferenceModel - # from forge.actors.trainer import RLTrainer - # from apps.grpo.main import DatasetActor, RewardActor, ComputeAdvantages - # from forge.data.rewards import MathReward, ThinkingReward - - model = "Qwen/Qwen3-1.7B" - group_size = 1 - - # Initialize all services with appropriate resource allocation - services = await asyncio.gather( - # Dataset actor (CPU intensive for I/O) - create_dataset_actor(model), - # Policy service (GPU for inference) - create_policy_service(model, group_size), - # Trainer actor (GPU for training) - create_trainer_actor(model), - # Replay buffer (CPU for memory management) - create_replay_buffer_actor(), - # Advantage computation (CPU) - create_advantages_actor(), - # Reference model (GPU for baseline) - create_reference_model_actor(model), - # Reward actor (CPU/small GPU for evaluation) - create_reward_actor(), - ) - - service_names = [ - "dataloader", - "policy", - "trainer", - "replay_buffer", - "compute_advantages", - "ref_model", - "reward_actor", - ] - - return dict(zip(service_names, services)) - - - # Service creation functions (would use actual Forge APIs) - async def create_dataset_actor(model): - """DatasetActor for loading training data""" - return { - "name": "DatasetActor", - "config": { - "path": "openai/gsm8k", - "revision": "main", - "data_split": "train", - "streaming": True, - "model": model, - }, - "resources": "CPU", - "sample": lambda: { - "call_one": lambda: {"request": "What is 2+2?", "target": "4"} - }, - } - - - async def create_policy_service(model, group_size): - """Policy service for text generation""" - return { - "name": "Policy", - "config": { - "engine_config": { - "model": model, - "tensor_parallel_size": 1, - "pipeline_parallel_size": 1, - "enforce_eager": False, - }, - "sampling_config": { - "n": group_size, - "max_tokens": 16, - "temperature": 1.0, - "top_p": 1.0, - }, - }, - "resources": "GPU", - "generate": lambda: {"route": lambda prompt: [MockResponse()]}, - } - - - async def create_trainer_actor(model): - """RLTrainer for policy optimization""" - return { - "name": "RLTrainer", - "config": { - "model": { - "name": "qwen3", - "flavor": "1.7B", - "hf_assets_path": f"hf://{model}", - }, - "optimizer": {"name": "AdamW", "lr": 1e-5}, - "training": {"local_batch_size": 2, "seq_len": 2048}, - }, - "resources": "GPU", - "train_step": lambda: {"call": lambda inputs, targets: 0.5}, - } - - - async def create_replay_buffer_actor(): - """ReplayBuffer for experience storage""" - return { - "name": "ReplayBuffer", - "config": {"batch_size": 2, "max_policy_age": 1, "dp_size": 1}, - "resources": "CPU", - "add": lambda: {"call_one": lambda episode: None}, - "sample": lambda: {"call_one": lambda curr_policy_version: ([], [])}, - } - - - async def create_advantages_actor(): - """ComputeAdvantages for advantage estimation""" - return {"name": "ComputeAdvantages", "resources": "CPU"} - - - async def create_reference_model_actor(model): - """ReferenceModel for baseline computation""" - return { - "name": "ReferenceModel", - "config": { - "model": { - "name": "qwen3", - "flavor": "1.7B", - "hf_assets_path": f"hf://{model}", - }, - "training": {"dtype": "bfloat16"}, - }, - "resources": "GPU", - "forward": lambda: { - "route": lambda input_ids, max_req_tokens, return_logprobs: torch.tensor( - [0.1, 0.2] - ) - }, - } - - - async def create_reward_actor(): - """RewardActor for response evaluation""" - return { - "name": "RewardActor", - "config": {"reward_functions": ["MathReward", "ThinkingReward"]}, - "resources": "CPU", - "evaluate_response": lambda: {"route": lambda prompt, response, target: 0.95}, - } - - - class MockResponse: - """Mock response object for demonstration""" - - def __init__(self): - self.text = "The answer is 4" - self.prompt_ids = torch.tensor([1, 2, 3]) - self.token_ids = torch.tensor([4, 5, 6]) - - - # Demonstrate the setup - print("Setting up Forge RL system...") - # services = await setup_forge_rl_system() # Would work in async context - print("Forge services configured with independent scaling capabilities") - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - Setting up Forge RL system... - Forge services configured with independent scaling capabilities - - - - -.. GENERATED FROM PYTHON SOURCE LINES 336-341 - -Why Forge Architecture Matters -------------------------------- - -Traditional ML infrastructure fails for RL because each component has -different resource needs, scaling patterns, and failure modes: - -.. GENERATED FROM PYTHON SOURCE LINES 341-412 - -.. code-block:: Python - - - - def show_infrastructure_challenges(): - """ - Demonstrate why traditional monolithic RL fails and how Forge solves it. - """ - print("=== Infrastructure Challenges ===\n") - - print("Problem 1: Different Resource Needs") - resource_requirements = { - "Policy (Student AI)": { - "generates": "'The answer is 4'", - "needs": "Large GPU memory", - "scaling": "Multiple replicas for speed", - }, - "Reward Model (Teacher)": { - "scores": "answers: 0.95", - "needs": "Moderate compute", - "scaling": "CPU or small GPU", - }, - "Trainer (Tutor)": { - "improves": "student weights", - "needs": "Massive GPU compute", - "scaling": "Distributed training", - }, - "Dataset (Question Bank)": { - "provides": "'What is 2+2?'", - "needs": "CPU intensive I/O", - "scaling": "High memory bandwidth", - }, - } - - for component, reqs in resource_requirements.items(): - print(f"{component}:") - for key, value in reqs.items(): - print(f" {key}: {value}") - print() - - print("Problem 2: Coordination Complexity") - print("Unlike supervised learning with independent batches,") - print("RL requires complex coordination between components:") - print("- Policy waits idle while reward model works") - print("- Training waits for single episode (batch size = 1)") - print("- Everything stops if any component fails") - print() - - print("=== Forge Solutions ===\n") - - print("✅ Automatic Resource Management") - print("- Routing to least loaded replica") - print("- GPU memory management") - print("- Batch optimization") - print("- Failure recovery") - print("- Auto-scaling based on demand") - print() - - print("✅ Independent Scaling") - print("- Policy: num_replicas=8 for high inference demand") - print("- RewardActor: num_replicas=16 for parallel evaluation") - print("- Trainer: Multiple actors for distributed training") - print() - - print("✅ Fault Tolerance") - print("- Automatic routing to healthy replicas") - print("- Background replica respawn") - print("- Graceful degradation") - print("- System continues during component failures") - - - show_infrastructure_challenges() - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - === Infrastructure Challenges === - - Problem 1: Different Resource Needs - Policy (Student AI): - generates: 'The answer is 4' - needs: Large GPU memory - scaling: Multiple replicas for speed - - Reward Model (Teacher): - scores: answers: 0.95 - needs: Moderate compute - scaling: CPU or small GPU - - Trainer (Tutor): - improves: student weights - needs: Massive GPU compute - scaling: Distributed training - - Dataset (Question Bank): - provides: 'What is 2+2?' - needs: CPU intensive I/O - scaling: High memory bandwidth - - Problem 2: Coordination Complexity - Unlike supervised learning with independent batches, - RL requires complex coordination between components: - - Policy waits idle while reward model works - - Training waits for single episode (batch size = 1) - - Everything stops if any component fails - - === Forge Solutions === - - ✅ Automatic Resource Management - - Routing to least loaded replica - - GPU memory management - - Batch optimization - - Failure recovery - - Auto-scaling based on demand - - ✅ Independent Scaling - - Policy: num_replicas=8 for high inference demand - - RewardActor: num_replicas=16 for parallel evaluation - - Trainer: Multiple actors for distributed training - - ✅ Fault Tolerance - - Automatic routing to healthy replicas - - Background replica respawn - - Graceful degradation - - System continues during component failures - - - - -.. GENERATED FROM PYTHON SOURCE LINES 413-417 - -Production Scaling Example ---------------------------- - -Here's how you would scale the system for production workloads: - -.. GENERATED FROM PYTHON SOURCE LINES 417-469 - -.. code-block:: Python - - - - def demonstrate_production_scaling(): - """ - Show how Forge services scale independently for production. - """ - print("=== Production Scaling Configuration ===\n") - - scaling_config = { - "Policy Service": { - "replicas": 8, - "reason": "High inference demand from multiple training runs", - "resources": "GPU-heavy instances", - }, - "RewardActor Service": { - "replicas": 16, - "reason": "Parallel evaluation of many responses", - "resources": "CPU/small GPU instances", - }, - "Trainer Actor": { - "replicas": 4, - "reason": "Distributed training across multiple nodes", - "resources": "Large GPU clusters", - }, - "Dataset Actor": { - "replicas": 2, - "reason": "I/O intensive data loading", - "resources": "High-bandwidth CPU instances", - }, - "ReplayBuffer Actor": { - "replicas": 1, - "reason": "Centralized experience storage", - "resources": "High-memory instances", - }, - } - - for service, config in scaling_config.items(): - print(f"{service}:") - print(f" Replicas: {config['replicas']}") - print(f" Reason: {config['reason']}") - print(f" Resources: {config['resources']}") - print() - - print("Key Benefits:") - print("- Each service scales based on its bottlenecks") - print("- Resource utilization is optimized") - print("- Costs are minimized (no idle GPUs)") - print("- System maintains performance under load") - - - demonstrate_production_scaling() - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - === Production Scaling Configuration === - - Policy Service: - Replicas: 8 - Reason: High inference demand from multiple training runs - Resources: GPU-heavy instances - - RewardActor Service: - Replicas: 16 - Reason: Parallel evaluation of many responses - Resources: CPU/small GPU instances - - Trainer Actor: - Replicas: 4 - Reason: Distributed training across multiple nodes - Resources: Large GPU clusters - - Dataset Actor: - Replicas: 2 - Reason: I/O intensive data loading - Resources: High-bandwidth CPU instances - - ReplayBuffer Actor: - Replicas: 1 - Reason: Centralized experience storage - Resources: High-memory instances - - Key Benefits: - - Each service scales based on its bottlenecks - - Resource utilization is optimized - - Costs are minimized (no idle GPUs) - - System maintains performance under load - - - - -.. GENERATED FROM PYTHON SOURCE LINES 470-474 - -Complete RL Training Loop --------------------------- - -Here's a complete example showing multiple RL training steps: - -.. GENERATED FROM PYTHON SOURCE LINES 474-528 - -.. code-block:: Python - - - - async def complete_rl_training_example(num_steps: int = 5): - """ - Complete RL training loop using Forge services. - """ - print(f"=== Running {num_steps} RL Training Steps ===\n") - - # Setup services (mock for demonstration) - services = { - "dataloader": await create_dataset_actor("Qwen/Qwen3-1.7B"), - "policy": await create_policy_service("Qwen/Qwen3-1.7B", 1), - "trainer": await create_trainer_actor("Qwen/Qwen3-1.7B"), - "replay_buffer": await create_replay_buffer_actor(), - "ref_model": await create_reference_model_actor("Qwen/Qwen3-1.7B"), - "reward_actor": await create_reward_actor(), - } - - losses = [] - - for step in range(num_steps): - print(f"Step {step + 1}:") - - # Simulate the RL step (would use actual forge_rl_step in practice) - sample = await services["dataloader"]["sample"]()["call_one"]() - print(f" Question: {sample['request']}") - print(f" Target: {sample['target']}") - - # Generate response - responses = await services["policy"]["generate"]()["route"](sample["request"]) - print(f" Generated: {responses[0].text}") - - # Get reward - score = await services["reward_actor"]["evaluate_response"]()["route"]( - sample["request"], responses[0].text, sample["target"] - ) - print(f" Reward: {score}") - - # Simulate training (every few steps when buffer has enough data) - if step >= 2: # Start training after accumulating some experience - loss = await services["trainer"]["train_step"]()["call"]([], []) - losses.append(loss) - print(f" Training Loss: {loss:.4f}") - - print() - - print(f"Training completed! Average loss: {sum(losses)/len(losses):.4f}") - return losses - - - # Run the example (would work in async context) - print("Complete RL training example:") - print("(In real usage, run: await complete_rl_training_example(5))") - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - Complete RL training example: - (In real usage, run: await complete_rl_training_example(5)) - - - - -.. GENERATED FROM PYTHON SOURCE LINES 529-558 - -Conclusion ----------- - -This tutorial demonstrated how RL fundamentals map to Forge's distributed -service architecture. Key takeaways: - -1. **Service Mapping**: Each RL component (Dataset, Policy, Reward, etc.) - becomes an independent, scalable Forge service - -2. **Resource Optimization**: Services scale independently based on their - computational needs (GPU for inference/training, CPU for data/rewards) - -3. **Fault Tolerance**: Individual service failures don't stop the entire - training pipeline - Forge handles routing and recovery automatically - -4. **Simple Interface**: Complex distributed systems are hidden behind - simple async function calls - -The same RL logic that works conceptually scales to production workloads -without infrastructure code - Forge handles distribution, scaling, and -fault tolerance automatically. - -Further Reading ---------------- - -* `Forge Architecture Documentation <#>`_ -* `GRPO Implementation (apps/grpo/main.py) <#>`_ -* `Forge Service APIs <#>`_ -* `Production RL Scaling Guide <#>`_ - - -.. rst-class:: sphx-glr-timing - - **Total running time of the script:** (0 minutes 0.006 seconds) - - -.. _sphx_glr_download_tutorials_rl_fundamentals_forge_tutorial.py: - -.. only:: html - - .. container:: sphx-glr-footer sphx-glr-footer-example - - .. container:: sphx-glr-download sphx-glr-download-jupyter - - :download:`Download Jupyter notebook: rl_fundamentals_forge_tutorial.ipynb ` - - .. container:: sphx-glr-download sphx-glr-download-python - - :download:`Download Python source code: rl_fundamentals_forge_tutorial.py ` - - .. container:: sphx-glr-download sphx-glr-download-zip - - :download:`Download zipped: rl_fundamentals_forge_tutorial.zip ` diff --git a/docs/source/tutorials/rl_fundamentals_forge_tutorial.zip b/docs/source/tutorials/rl_fundamentals_forge_tutorial.zip deleted file mode 100644 index c77b9084e4c0471638195cc647797a00896fc0af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40960 zcmdU2&2Ah?a+ZG&ycyUBs4d!R<;m;HKC zPv@gz+?`$JdA@P@b@nW1lzmf;Cg<5RHqBm^^UKj>I-Z_iZ9HD{&&I~~_LD_@F`e&k zZ)e~A<#aL6ehn6~pI_AVtlGbS|9n(mEKa+F>E-<^JY7`fe1ZnQ_^!XP(d}mE^U-jB zKYO^bk>QW|V0ToPm%D>v&PV&xK|LBwC;Q)BPUm$o-#wk;!JqGymw*0U_U%PcXP}$C z8I8x;xGd%qAp~0cgK<$+N+lz$-PydHm)|c&706fmLbTh?o=)dw2FZZO*>nQ&RoQ4_ zgwpw`$$qvgmAeOzD4$JXQR@Q^=mVM=R_RHxTr8F6oGtjv#W?4PW zhNG&Uk4_hLIW%-3hS#G3*x<17XFL0RF&Ym!8`Yo~7pG%ksh$_32|pZ9r?U#9tS_>& zVll2kbzII1yv9>FEGtNOP~xqV;=H6HWW{8N!3@h;iT~ku5+}V`QP-FWBZk#-AA|ed z#bi1kLVi~A2azy!5I2zd!PU#@d~lJye%O7`eee@Mh_4R5$%+9aQ)TlKGlEeJ%b|b% zc~Ol98FaXuS2ae*!A#H4b~J&;4|LR`2zcVVJu5Cp<54jm)mJLTqPm(4?iX*0QO(&t zxPlT-Ha1>fjH=8onyfC1!9`hdO2f64E&Q_isLIN>#Q+oHmybxj!0@1GgL*M9yBpt@ zA3;IE@w_(g&#KYo46}S$U{3Xo7z${~r%)O{#_R%W1yvZ1>d|ykj6)UMFe)Z%7|evK zW|S1HfRAhHt*=q#->|1PumLzPj^@R+1Nybuge+?-&Q8FV0D>+maL5WqBN61 z6H}R`29r~DGfeE{6*UF(U0{kiv(%kpl0A72CKg;C`@+|5w!LkZ{r2{Lb}*m59t}b6 z`$buC0;;St9=$5FT&xfD<>5~r{zbmUA)}|-33hx5c0BFu6e@boX&g;p20G{E1Pd6J z2rN`@pv{>QWiS0C1L;SOZfJKbb=A*#R?`fYuZ!^_AP9c3)}im6^U-U(rW(M`nYTRX z{_GRW@L_p|bqz_XO(X56)7cee1=n#tqQV*ZWF6Sh*J2@u^U+zIoz170*(nSpj7sSV z&c?+R4B8n8liaVTb$L3SzEVCbcqH&qzMYlx5#I}WTG3`dv19*<@@r|n6tM|43P{$A zlEW&(1c3-=-||V8hS_uyzD4Html!IPP8Jkab{QLetYI!?*n@{hGn|bW}onX0z)4`Fu9Le+gZK=g(jnVN`35 z<*UPk-;md_SjvJNYES#x&1|omeN|$RYMLNjIHOCU?xyK+L;168GaIU>>{epII$P>t zH?xP`>`0ea*3@|+YLp!$o2pN$GjY3J+tX=^BmXhhwwUQTf{B9bbW{sn(f90eMqMYR zxj62=8biBS9K9W*c#iI)E@t3zRvbB|d4pAadC6zyC1x?1_b{e_+wUW)q9Ci~ z@B4&eVkAO6_KQPa~1Nk@j+ztqj?2r2N zy-=kLxwQMIvb4L!aM5?MJbe&Q!dUOfjnWZ5qZKvF9d>ML9M_SJ5m)#&|)*>PvZDJKen2Ez+PW%c;VaoGqt!l*~R z>2_v(Y4-1b`*-%|Cj+QD-n4DPYI&xfQ$7uyojlVIe7AW zNfYB)^x@a$w=h0|MK$gs&>klnDC(pdi`&J-86&G3DjR+w{e^QV$FTQBeG~r|8$C(m4&bi}5*p^e6;+T$e zZIhp@;h>s)tBN{~5yxQ1i1;$Wv^p*@Y+YA(K7}Vh1Vv6C>p#l8^MH=t@q?3YUB0a? zizz49qnONwct_xZTZH!n(;&9bX?0*^!x3IxMgf4uO&UlJ+NZT%&VS;J-Nx~&z zyhTs#2uW>bKkZtl%U#UHd^#EZPWAy!DZVKgKV*v;>~9&#l7yNj#WOQoR2O~iTq!%0 z7q)hLDzj8?XD#(@uKQVmAhPaw0!}6-ndG!9`={z7`s&_fsl;E zwi2#h7ei)OhinDy8j!t^-B1}ugRtE;Lzr!U41(}w!Fjlrq-U_SoK$j3ijh73v7!5& z#;QjMySWAZW_FV(HH=Rsn3`g3Ae)5*#qV^-!;R9ogq|^uxdQu_8gRf2Q}U3L<~HH> zr!dyCZQaZyu=U}{1a_xDPA$lIB2=Z%kz~LK+Xmf`2?0bhld>E(LNg?L7c-MRXMiTW zwD;zQTWQTyiH3Y+-bC%!v(I%PftcOiGgFoQ_6@d9A9we5_qsp*TvK7LP6+4l`y{nGaubK`os$US%+{aKfZkS-G;wrLo9 z+H}4IqC2wTM~wDl_`1L$%DG9;N6a%8?P~ahutU>9FN*p?qpuxK=;Rd^i2kmYN!&l| zch0%J&;NZav#WWYbHs(Se#M3GINP%k-*dk_pz;l^!oaXBaIS;(l2{ocG^dBwZkeg7*~=zNa|SeMC4y;LeTPTsf$`b{4L@}UzW0~DvON{la?w2 za(jtYH@`wqX-`yIgrfyu$VZ$yZ3=CVyrD&_aEg~AbZCQMV-=i8ZBX5!2bZC#U5!9j z3|_)%KmYfuOKP2!s@WgF=q6slt}(kNQZp0T*JS+T?AWPXxTEtrlbawQ_{7Xw9()wZ zJA7bMZC;4sQ+iJg!y{UrdDGbJ3XRhqB9Ytq-DuW{#)6Q+82K#>6w3z^qu6F9dL z$8-~Nz-^76D0zd<-o7fofA-%0h|Ru62^HS~GZwrd*SH=J#>M67uxJg^CKSRJE_(oS z(P7UHh9H>+zUWb|=}~fNxOpFCd(7NkYv0^NtIxGsCR5F9fX$y%%;u1w9bDQ;RvMuM zmQ8Ot!68rCU-EdB?D>p;aypACwn@btW}`&Z1aAh>?D@IThE%4}Oiqi|YFz4}Zoh;x zo`c{sR5mp9BA9<+Mdlewv~oODQ}0+3Tr!KG83YdF(?knIs0d689`CYA_d%-7bk=7y z-g+7^9sC&UFN|+Edp3;(a!!MDNh!HcOcrcEJAN@8ygD>Vn$FhADy>X8MyH5}HmnjI zR0YnaIM=!~d&cYa(2)oSvqaZ&yT)S!B=c(Z`g?{?0WGIkXXD~EvRPP=jGR*~F3$QW z2`TFex+fOxEWbEIjmkTD?fXWTU}A&;Gi=erce!Qf2`X;B#Th|9=BxM0-A|hRM8Im* z-sRl9jQf%Wu!{sqoxDQAYFtt)A3k{WSwJE9aiw<1OC$l{v;h__r%`hsAoN0(%%>yv zaWrPRO&YAo)mj(tbt4G!MLmLi6VG_DWY^G+fJh+!+$%`l5*F8{v88+mqz(xVHH7AG zOJ&yZ2uLzH*;rvQrR+Ymn|q#MPs)&N+nI={4m=+50pPv@7^Ao|lG4!2X2ct5erjNYn5RF3sG`EVr`C`z9-{ zZryJct*t$N(8Zd2(0zCk$=`1dR%v&5_7LP%bi*tganTuQq@1nTx|W1WyO%rN4v3`n z1p<|j-v#K9-f`|X@LUV=7CyJ$Q#7o#PQ2I8Tc;@D%%oj4C9lJ@Nn#r1pM_Rg${10a z;~lzNY5dR;$fl=%g`-)KU6d>0LK8{x;IZFl-mTxqhGu+b3QjmM{wgO($AwKiGS?=_xo)>BRZ@0ShPT+{bTXC4bkw(!1$#4{zk*lPdLEv8%9)!f2Y4n` zv5u&EBt&~d&Flp^M#(yIA2ujv#VP6x>rq*OrMs*V`1az;g@qY37gZ1ygtwyF z&Q%D!3~C5pe9ctMya>*KiqTm82}Fwmfr6V>mez>bh6hCb=taSI4M)t9@OQf&G?R*r zF(M91!q%6vtY(Ok)zKJ|ldYSiTd9gXeug%NEc6E#+&@f^c?&Fqs9&fXZy*bpQ&$d& z#3+)K^#zWv!8u>XqPn3R+nZUOj#(l$s^imYGt($`M(#9ZBzx)@$?&u{;cQZ$O^*XD ziputQWN&40XmixRW>y(J*ue^jXOasPPcRwFb>Ji1-0b!`w!G!Z^DS5o-+*Lt8~)4I z_M^rM{U|VY(pvJ`Ad-L0U8>}9G?UeAXy%JeU!bgb#EMjgeWL*i0hj=*W*AJ+D|Ak3 zZMAGL>zFF61n+X9A&je4sno~?RAZ)J;B1nmktSRyn+*e|nX4q1HeD4o4LOD4V~dNt z-b>cQCs;JqNW!EoA~Z)>0m%lN*i0}($}E{lJ~i70oPH71fV)`0dykM?lb^{tzcm%7 z+2_UNRf21nVku4K^nDa2D3&WkoZLlXcLmG8#QC4ifSzJOy%`O$ZEEq1P?3V6wonXW zDCpR}$AX)r7eJR4{%(!4w=S%+{GB85e%~x1OB_W_I~j{ydB7v|+hjsgS@sTH-eW0A zr}qIleOr1>KVK4nKmRBwG(R93^xM3R;_V(ZZ{%Af%{Z%Dux+WpLV6FF2+UlQI;a3(M$&>r?Px$N@NUUq2?+3N)NI%dKzP@`5OM6fbw zcKW;+ykhbX5rS!gf2&#XhF;zq5J==Jl!KRNi!lVo+G5>TH`*NOVKsxt(FFE78!k{z zidfK^LL2>X0d8zu7xmm}(1E*mV|_o{fl4ZLUN;guk=_m|Yv$9Q9y8 z*Yo^vDbEPbV6*KxJP**si@~_86fQMv*M)1D27Id}d>CY5*AR2#SSL`8r`X-#2nYeg zFsPwtbRg;A)=&dg7PgHKUZD{|uxYHr4oof2Eg2NX%eNgKs;5SjY5GPnrhS;f6RP(_ z=%}=hKeCJaqmy@BbVZ$j%V;wL-$EC`v3R zZcBtUP^Z(n#xA=Yw1YwyTEUxQ8Z`n&638BY#9<0tt|@wdtlKf{*8eBcV2TzV8W7jJo;_qbDUse!(hpoz_)8~+*?Z&y0Y z&{y)-m?;diHC^=gq~nYXf_`V;S*6z~KeGgQIl#8V#S^_##mG+k&@jL|ebTx!W?mn@ zg?ER;E35Wgxly`r)t@o>;bJ1!j^DXAzGrI1N#%%)$DWf_5JYBH3yYH6(i{?NRtEc2;6*`c$aO2p5F#A*EH z5=qJ~)AqlVQjnm%)`n~es)3`0+MPCAB3cwu2O z53=umQ1;bBaOqxVZ<)}eIabIf^SV?F_u+c?NRAUnAQ7vMo}anshE9t!B$S(%cG#7F zQd6fvE+>N54;sVrI%lh;I=IV>;=Z%;jnqq{HkQ{GO$i_}-S!3}&60l|k{qJezj&M> zRYYaRbmRWNrCOXJE<|iOScs0(_*b}V7sNx67_O`B$9*MI*0X)5zfb{lklZ4M+wQ0j z1#Cixq&?phnpX&C%D+YRIb7sR)>UEukfpt^{B7}khiu%fgdeq_W8rT9a&T+|Y| z+6w>7mCCxE_kF7^sQNaQ$z105)$n22P&T-Bx4qp+L6IjU<3>ol^FFjowvgtsv*jx-$-ACr|^s5E%MohDfg9!EM-p~Kz zzyIW$d-wSFPp?OP8qKaIr#wl_2PN+6A`Cqie_;<7<>S6BM&U%b)IwgqEeDd4M>1?N zVcG6vF&^6>HNPxtW<-l9A%nWD@fi2Sab&oUSwgXR@hco&t59P9$zZy=F(RHr~`SskSUfi3&8LbDD)tmGt zAp+L1L%=%eT!VnM#l8Uo7JsGrT8+iD9tM_D0?eQyhd6_9Yw1W5(>d#+_E2nalqO%=IH1uKiy^1mJn%dv42**D;@9F1)` zTyEoG1r0vG2_RO`Z#_gTN+k|U6m#efFtK`*|20so-eh@f-rZjL|*>4NFPN4oso7WYpX-AlL*3m@Q+G(9_njxZJNV(5^VMAB2TP0ykJW zKdjIKVhUiqdI(1Wz;i8*7l&+mO4%cS?_rO#1k6iqZ7-Mq#IvJm}sQ<;%a)!EQO8~%nX#}tv7{3qMj38{d z1|E!qHKv>@g_1YK?YDydVyFivdI!i~znaIg(qMm<4@gVd+Cj%FhSWJfq;?X4HV~x0tTmX zD%A^azV{MfLoh}(_sE|m(ad`GkOzH{6w{-Tp1F#Q&!}GPztS%^M_`trJw!k+dgHp3#(ZZ*mwc{y@Y$;z_YM*Bd_2p6Q&7wyxzEj+I9*?lsZP zv|A9K#4&_*q3tn*rpXWE?h($EL3h~;B9dqoNKChZI8$mokG2q|X_*^z1q%F3@1?cG z>{z-b%vnqFE|l`VzbW=F*F-u~+P3idj-bwzXv+jdhyN5<679?{ksITk>3g+!J7I!a z?^i>z2IQF%PGX+vBd(^#En}#QassQg4(ORDls*}Sxzo!Wl$`vy0{6@ck$in9@|kC> zHKPGXE5OfI5b$R88vL_xK*15J{n&iO>&pDd>`rm=c`a-o&sbnkd2honb$p1zjIDj; zTs`$|pwL1eZ&-{rH`YIJ4`P3gHYqFiFCo9$Kj7bsyeG(LvIWAB>;~rsvDgO&@%H*+ z3pb6k6lP-X0vWA0iE01`0J&dzdb$i4jlA7JL_gGU6{(9Pq+6)g`m-x1kS@2wbyG?b zoZ2G58vwhtd9o1#N*JPucs074(uttUBqB<~yDe`WISUJTb{NzPuj(4rs`<`gB(%J{ zF{-Yc-<7E~m!usJUq1}uEx`cOUF&1KagmTCIZuGM>ou$KVEv?l2b@8xJ=<*)C-@Et z5$EXke8PO&3kJ9f>P;z|A_P|Jsc&jvy=f~Io7u2Xmyk7H0`}%RB1e!Ln+xt@jd1kYei|SetaT zC|vnw$gKnD2+L6v^vyRjj5_p=DoM)cI#k14EsNc1r1*oQKVVC{7;{SV{YbU~y4@ZD{f0v@tIOBpT5WD`=cH6HQ38bI5UQ zCeU$SfAn=`vMqX?fBe3oROSkXz7LgQ(c2J2O3sB(9SE1FrLRH}k08^>jeO>Y!VT|g z;|=&kONRBek9xUDy_2Hl5FhcsmS$XQMk6&j5I9%sWYmPl&~odmh|+3E2q|`4#^FF( z$`Jw0C0blFRea1p5LN1sfbuupq-|E;HUhzXD>>WW}B|0J?MXjG^xM@gBu&!0zPtm2z!6FytD& z^h6oj{>lPCSll~5iD?JuZq#_n*)`yIt0&0EZOPf;E+5D3NP2!HSwG;R^x|j1 zrp^v%xd!@83jh_Aj;c|3a>hM@O|AwTyPN16-}6Mz)dGRX%7)NE*l}AO%I&xpVC?A{ zLxL;-%11Ka4pWshwFIMgwvP4u{D@g=&Y3+^wIXN&?nM?U0lF88bv@ucD_QNCK;P2` z9B&A}mnf8KG}NIa0rlV>6oIeN#eHcJQbc+jXfgX71fSise&G>2w2YA1Q(q)w6WK`| zKB>-hmyditYDor`hcYXW`20(sg_nlp8_KZ9SRacIY~3J@{ehYO)kuAm^2-3b?$$7T zINNc^b9S-Z1ci@`+(0Nltla)u_<_0v>?-6k3sw6=Km^;AZ3!7K&p(f^H#MLHA$QhB zD+$}};Cehk>BT|V$!;ME`X%2>VG7bdHnuB0ccr-E(_Y>J7L1|_z8#wfq8n;+4~L>j z(?n~YI~nHV~3tL1iQk?-F zTnSp3=qxJ=CS`rx!UT~up$i*5Tl26Czc6X!QRw>92cN9QF>JiBM_4|OY?p#EO!VN{ zSnCuk(XI_>m}u0*G)y#E1#8%7-jbjZ3T{-F#4O-Vy(8-~(AcjOib5044RFF02#1ZA zi!^Tma+n}v%=-E`hYcFbX4J!a-Bqa1 zHwZ~TFiuCzWl;-_##wMg@)?xm+y?!SByKR2A6|6rF`Jg)HtO08qw6DrRPb+Ow?@C+ z{7y#tVnhpO0=jKt)@{+kCT0p(sbmMJSM<@S7nxiWDg-c&-ub%V*qqKLBN-c@^5JJe z-huXrNhM;tX_RwgK*uOZvGkg!ElpI%sFRIsv~!r_S$9Z4ck}|WJoquKh42`$WNAd> zKu{Od6F5-6sAD`98!ek8Q)0ssPRyf~gGj}>s2)i`K1K{BN8&d+7WXmg6A!{h%lPR20f7^9+~+J5Mzmct>7Ypi*y~Vi3N~1 zzvRKlODjc}BSt2=A6h)_WuTFXKG)+$COWMN9VtEOfv^FkB93aGyy6q_BZ|*$x>(8P`m^aFe(k{IX5Jt!@S#>qB5h zk;mFFnEHCN=$I)Fa&vKooI@(pI(`;KA)jteMibShwiZu~sg?)PgllQ2sVCEqUGZgt zj38oq(-_>lXpj2 zm80`ByrN=2GPw>ZaYnXsi4ovAEtpI%PN!=}4WLY8yhw_F?f@&(zf`N&T$Qj@KAg@` zsT6q?Cvo`Q*0~XI5a)ZYqz%7&a&8RQfMz;N&H)9Tmg8o&gkbS2-3g}W0?tsrC@ef} zG9aM7pp;|sB}+Ogb3*74G^tv0wH_i!kJ56HM$t?#ZGkjBx00bw&44*Jl~P=tgTx7E zgKaXyoySPWUcF4;8)bj~Nsn8%7zjXwdks1KyF(SVX7@(Wnx52$V{3X^ z9M1Ju$RIAUU#8T^>pSMaLTt8aHZV5h2BC&qd3@bK*=*>Fc4@?INHx1qfq84j4>!Zw zbPF%IDpm^ii(s7H1aQ-z#U|aXh1`@k>S%f&g1QMh|Chm;#pI+BkKwL@5b~xdoJe*V zSj|-+oP6b-VQ`8f-VB11@4qtyPI*6d^}_);$=99XZwg;7CbR{?xGb!xs`cSF$<^JU zZwgB*z&H6O!g0W=e=zo@0)l-w^d^v!*kylb>#9$13XOK$|C> zO&CBlToYx}wO7JNW~~t*!aWKyhd8>jXg+nZGLMlX8qN#2fG= z-Jr(h04kMu)_x)M0lpCnH$6V~$dhA$r7r+Oh+g?jp899Dmwx`S+B^eKncI%cmruiY zGF^N*n$Pi->n!FA9s%jok_7fAIdSj8BcL=Bc^K_KHtc&O%rksz7{GGm>qhjeYPz#4 z*Ezo&0YKAyX)V7c^Zwi227UM9*5{|lTj13-v3P*HC9wJ}NzAU!<+~B_J=kxxg&9B~ zNzZ@&ad!`2Li?E!7)B!_$tS0XzX2SIAa-#$J5=YRdT^M84M?;ijDSU%PN{nKCbUwjcm2)o?J3jj)QVVWVNCb=d?tEKo) gfBm0$!8POrvGw3rfAq)f_51kmAMx+=f8cNb17h3yE&u=k diff --git a/docs/source/tutorials/sg_execution_times.rst b/docs/source/tutorials/sg_execution_times.rst deleted file mode 100644 index 0bb42ae9..00000000 --- a/docs/source/tutorials/sg_execution_times.rst +++ /dev/null @@ -1,40 +0,0 @@ - -:orphan: - -.. _sphx_glr_tutorials_sg_execution_times: - - -Computation times -================= -**00:01.107** total execution time for 2 files **from tutorials**: - -.. container:: - - .. raw:: html - - - - - - - - .. list-table:: - :header-rows: 1 - :class: table table-striped sg-datatable - - * - Example - - Time - - Mem (MB) - * - :ref:`sphx_glr_tutorials_template_tutorial.py` (``template_tutorial.py``) - - 00:01.101 - - 0.0 - * - :ref:`sphx_glr_tutorials_rl_fundamentals_forge_tutorial.py` (``rl_fundamentals_forge_tutorial.py``) - - 00:00.006 - - 0.0 diff --git a/docs/source/tutorials/template_tutorial.codeobj.json b/docs/source/tutorials/template_tutorial.codeobj.json deleted file mode 100644 index 21bdbcd5..00000000 --- a/docs/source/tutorials/template_tutorial.codeobj.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "torch.rand": [ - { - "is_class": false, - "is_explicit": false, - "module": "builtins", - "module_short": "builtins", - "name": "builtin_function_or_method" - }, - { - "is_class": false, - "is_explicit": false, - "module": "torch", - "module_short": "torch", - "name": "rand" - } - ], - "x": [ - { - "is_class": false, - "is_explicit": false, - "module": "torch", - "module_short": "torch", - "name": "Tensor" - } - ] -} \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.ipynb b/docs/source/tutorials/template_tutorial.ipynb deleted file mode 100644 index ae808fec..00000000 --- a/docs/source/tutorials/template_tutorial.ipynb +++ /dev/null @@ -1,75 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n# Template Tutorial\n\n**Author:** [FirstName LastName](https://github.com/username)\n\n.. grid:: 2\n\n .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn\n :class-card: card-prerequisites\n\n * Item 1\n * Item 2\n * Item 3\n\n .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites\n :class-card: card-prerequisites\n\n * PyTorch v2.0.0\n * GPU ???\n * Other items 3\n\n\nTo test your tutorial locally, you can do one of the following:\n\n* You can control specific files that generate the results by using\n ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable\n respects regular expressions.\n For example to run only ``neural_style_transfer_tutorial.py``,\n use the following command:\n\n```sh\nGALLERY_PATTERN=\"neural_style_transfer_tutorial.py\" make html\n```\n or\n\n```sh\nGALLERY_PATTERN=\"neural_style_transfer_tutorial.py\" sphinx-build . _build\n```\n* Make a copy of this repository and add only your\n tutorial to the `beginner_source` directory removing all other tutorials.\n Then run ``make html``.\n\nVerify that all outputs were generated correctly in the created HTML.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Overview\n\nDescribe Why is this topic important? Add Links to relevant research papers.\n\nThis tutorial walks you through the process of....\n\n## Steps\n\nExample code (the output below is generated automatically):\n\n\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "import torch\n\nx = torch.rand(5, 3)\nprint(x)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## (Optional) Additional Exercises\n\nAdd additional practice exercises for users to test their knowledge.\nExample: [NLP from Scratch](https://pytorch.org/tutorials/intermediate/char_rnn_generation_tutorial.html#exercises)_.\n\n\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Conclusion\n\nSummarize the steps and concepts covered. Highlight key takeaways.\n\n## Further Reading\n\n* Link1\n* Link2\n\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.18" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.py b/docs/source/tutorials/template_tutorial.py deleted file mode 100644 index 4018aa1b..00000000 --- a/docs/source/tutorials/template_tutorial.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -""" -Template Tutorial -================= - -**Author:** `FirstName LastName `_ - -.. grid:: 2 - - .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn - :class-card: card-prerequisites - - * Item 1 - * Item 2 - * Item 3 - - .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - :class-card: card-prerequisites - - * PyTorch v2.0.0 - * GPU ??? - * Other items 3 - - -To test your tutorial locally, you can do one of the following: - -* You can control specific files that generate the results by using - ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable - respects regular expressions. - For example to run only ``neural_style_transfer_tutorial.py``, - use the following command: - - .. code-block:: sh - - GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html - - or - - .. code-block:: sh - - GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build - -* Make a copy of this repository and add only your - tutorial to the `beginner_source` directory removing all other tutorials. - Then run ``make html``. - -Verify that all outputs were generated correctly in the created HTML. -""" - -######################################################################### -# Overview -# -------- -# -# Describe Why is this topic important? Add Links to relevant research papers. -# -# This tutorial walks you through the process of.... -# -# Steps -# ----- -# -# Example code (the output below is generated automatically): -# -import torch - -x = torch.rand(5, 3) -print(x) - -###################################################################### -# (Optional) Additional Exercises -# ------------------------------- -# -# Add additional practice exercises for users to test their knowledge. -# Example: `NLP from Scratch `__. -# - -###################################################################### -# Conclusion -# ---------- -# -# Summarize the steps and concepts covered. Highlight key takeaways. -# -# Further Reading -# --------------- -# -# * Link1 -# * Link2 diff --git a/docs/source/tutorials/template_tutorial.py.md5 b/docs/source/tutorials/template_tutorial.py.md5 deleted file mode 100644 index 60b15697..00000000 --- a/docs/source/tutorials/template_tutorial.py.md5 +++ /dev/null @@ -1 +0,0 @@ -1d0dbb374fded9aaf6cf60085291fe43 \ No newline at end of file diff --git a/docs/source/tutorials/template_tutorial.rst b/docs/source/tutorials/template_tutorial.rst deleted file mode 100644 index e4768c07..00000000 --- a/docs/source/tutorials/template_tutorial.rst +++ /dev/null @@ -1,152 +0,0 @@ - -.. DO NOT EDIT. -.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY. -.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE: -.. "tutorials/template_tutorial.py" -.. LINE NUMBERS ARE GIVEN BELOW. - -.. only:: html - - .. note:: - :class: sphx-glr-download-link-note - - :ref:`Go to the end ` - to download the full example code. - -.. rst-class:: sphx-glr-example-title - -.. _sphx_glr_tutorials_template_tutorial.py: - - -Template Tutorial -================= - -**Author:** `FirstName LastName `_ - -.. grid:: 2 - - .. grid-item-card:: :octicon:`mortar-board;1em;` What you will learn - :class-card: card-prerequisites - - * Item 1 - * Item 2 - * Item 3 - - .. grid-item-card:: :octicon:`list-unordered;1em;` Prerequisites - :class-card: card-prerequisites - - * PyTorch v2.0.0 - * GPU ??? - * Other items 3 - - -To test your tutorial locally, you can do one of the following: - -* You can control specific files that generate the results by using - ``GALLERY_PATTERN`` environment variable. The GALLERY_PATTERN variable - respects regular expressions. - For example to run only ``neural_style_transfer_tutorial.py``, - use the following command: - - .. code-block:: sh - - GALLERY_PATTERN="neural_style_transfer_tutorial.py" make html - - or - - .. code-block:: sh - - GALLERY_PATTERN="neural_style_transfer_tutorial.py" sphinx-build . _build - -* Make a copy of this repository and add only your - tutorial to the `beginner_source` directory removing all other tutorials. - Then run ``make html``. - -Verify that all outputs were generated correctly in the created HTML. - -.. GENERATED FROM PYTHON SOURCE LINES 56-68 - -Overview --------- - -Describe Why is this topic important? Add Links to relevant research papers. - -This tutorial walks you through the process of.... - -Steps ------ - -Example code (the output below is generated automatically): - - -.. GENERATED FROM PYTHON SOURCE LINES 68-73 - -.. code-block:: Python - - import torch - - x = torch.rand(5, 3) - print(x) - - - - - -.. rst-class:: sphx-glr-script-out - - .. code-block:: none - - tensor([[0.0394, 0.0994, 0.5448], - [0.9781, 0.6839, 0.4422], - [0.9133, 0.0060, 0.0412], - [0.7065, 0.5727, 0.8577], - [0.4415, 0.9282, 0.3205]]) - - - - -.. GENERATED FROM PYTHON SOURCE LINES 74-80 - -(Optional) Additional Exercises -------------------------------- - -Add additional practice exercises for users to test their knowledge. -Example: `NLP from Scratch `__. - - -.. GENERATED FROM PYTHON SOURCE LINES 82-92 - -Conclusion ----------- - -Summarize the steps and concepts covered. Highlight key takeaways. - -Further Reading ---------------- - -* Link1 -* Link2 - - -.. rst-class:: sphx-glr-timing - - **Total running time of the script:** (0 minutes 1.101 seconds) - - -.. _sphx_glr_download_tutorials_template_tutorial.py: - -.. only:: html - - .. container:: sphx-glr-footer sphx-glr-footer-example - - .. container:: sphx-glr-download sphx-glr-download-jupyter - - :download:`Download Jupyter notebook: template_tutorial.ipynb ` - - .. container:: sphx-glr-download sphx-glr-download-python - - :download:`Download Python source code: template_tutorial.py ` - - .. container:: sphx-glr-download sphx-glr-download-zip - - :download:`Download zipped: template_tutorial.zip ` diff --git a/docs/source/tutorials/template_tutorial.zip b/docs/source/tutorials/template_tutorial.zip deleted file mode 100644 index 321b881c49affa7b268465506f2f059b02457417..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5757 zcmd5=-EJGl6;_kBKo`AC(c6O+3P^%TSyBtMOdLmb9?n!cPc;CvYn)n#Vjeq)8KRR^zreF(_?~A@Ca0L)_Tg7lNr>Xk{w~~LYj9{cB3ec z_igtF6YCo~Ax3fd(6Xh!ZLKKotMDy^;#QU=9?d^?~pVY%{W{bTV zrW&mJG)Rso zo~E{@O9g*wDVzz6z^j*OQB^Z8#JoDClTQArRu$M+Gl5vQLr)=%_S=2U?}L@!hwq}H zR+aD6#yUhvwmSB&iDfe}-d(%?^vt?!LKg?!2i*q?t-k30o*q4Vv>?1h+#Rv&DuTAr znI+g6+=G;9;z-(NLhI>n@LDDe<(90Gvs{o!BCYMEGUFcNOY{|7%ZuQD@2sY(l$jc- zEM!%M?1B5o(nyE24rz&8uC=ddIHkG*8&)q(zc@NMIezt3(my&oJAU;dO-Y&y<*X^B z@pK_jN<%HX$bCsGYv;4f3EZIo*gH9{wQwY_5Y(zt)}W}c;E83LK$SurORhFBPEVnN zk+l;#2_=(wCrmYxZn=)rbeB0GL4zT4N5J-?K>flYm_~+jQFey#<2kaknzV_uqW8o2 zPKhxU;#|_i7dkMp?tdVnDksWZb%wRlIdv%s4;odqe10c|LO= zD$K$9c@n|3+=J4g94mw9C$oN)F7*~C3wwbCAY>cakecoG5GsV)g!rWC+zV-n_VWkn z)Mz?Ot+2QDWsQ(uqNL`zmP2pHZ49w(dRyisk}#()&z_%jLlcWO*EP^iUtXX`D|v~> zPBTKg{#;g>Q$vYX1XZ{s_`{Z{5ml7jv5bF2NATo{GUq&nG)V^ z`+w^yx)d6lxWoC0v-Nlq9A7${A?;DLU5v2p)XTDJl?#;PrVhd(vc>VT!IduNh%lcS;i*2~hP zn1$YEm#Nt^9bZY8sR~Vd;ruoTE8oF#_O@7TFP-2cM4~e^e9-0`SGo)F2wf8XQI5`y zz0@)v;~d zmsun|a0z`0bfW{pIhQC+bPjPTrcJ^=sa+`VS5oBM@o&T^2-^wyzmM73(?QgqL#=-U zYW?#cr~b2_Zfx-X&u@iVs+^kPU(kl)Oln=lJ$lU@nV7^QkP|;GC8lD|`Z#oSo`Q$7 znzJd%fRa2Gm|E-kbOFqk3ok%#xiLSSg`x!P0(m5KB^yCs&-cv=DyZ^iPMq~Z?3t+- zRQm;rtJKzOHAKJC%E-83_9lEcEL-8%O+k>(b9-qP<}B;G(B?M50|P|43g!l8V3xt# z>zm;1&DLAM+cq;;0&o12&0Gh3n{fiOI}lPU6DwN;H{aSqxK+YoyXozU-i_ksjNeDe z>p*VKVuudQ?E$D4vzNhZE zxxjU-{5r85^9!%?bI-pEl;d4cDu6kDz6XZm?eFEt|F08s8Go`Nw;|AhX(8zMmoIl9 zR91Bz@N*t|z=&1!xj38Gi$1rL?t3TAeXt1DW)JtIFw^FM_H> z^P3Q_Xbf2Ly~`1TwA#9i8r``%>^x4pRVJ|5qGbB Type[T]: """ @@ -89,7 +119,7 @@ def options( actor = await MyForgeActor.options(procs=1, hosts=1).as_actor(...) await actor.shutdown() - + * Default usage without calling options .. code-block:: python @@ -100,14 +130,13 @@ def options( attrs = { "procs": procs, - "hosts": hosts, "with_gpus": with_gpus, "num_replicas": num_replicas, + "mesh_name": mesh_name, "_extra_config": kwargs, } - return type(cls.__name__, (cls,), attrs) @classmethod @@ -130,11 +159,12 @@ async def as_service( "hosts": cls.hosts, "with_gpus": cls.with_gpus, "num_replicas": cls.num_replicas, + "mesh_name": cls.mesh_name, **cls._extra_config, # all extra fields } cfg = ServiceConfig(**cfg_kwargs) - logger.info("Spawning Service Actor for %s", cls.__name__) + logger.info("Spawning Service for %s", cls.__name__) service = Service(cfg, cls, actor_args, actor_kwargs) await service.__initialize__() return ServiceInterface(service, cls) @@ -154,28 +184,6 @@ async def setup(self): """ pass - @endpoint - async def set_env(self, addr: str, port: str): - """A temporary workaround to set master addr/port. - - TODO - issues/144. This should be done in proc_mesh creation. - The ideal path: - - Create a host mesh - - Grab a host from host mesh, from proc 0 spawn an actor that - gets addr/port - - Spawn procs on the HostMesh with addr/port, setting the - addr/port in bootstrap. - - We can't currently do this because HostMesh only supports single - proc_mesh creation at the moment. This will be possible once - we have "proper HostMesh support". - - """ - import os - - os.environ["MASTER_ADDR"] = addr - os.environ["MASTER_PORT"] = port - @classmethod async def launch(cls, *args, **kwargs) -> "ForgeActor": """Provisions and deploys a new actor. @@ -195,17 +203,14 @@ async def launch(cls, *args, **kwargs) -> "ForgeActor": procs=cls.procs, hosts=cls.hosts, with_gpus=cls.with_gpus, + mesh_name=cls.mesh_name, ) proc_mesh = await get_proc_mesh(process_config=cfg) actor_name = kwargs.pop("name", cls.__name__) - actor = await proc_mesh.spawn(actor_name, cls, *args, **kwargs) + actor = proc_mesh.spawn(actor_name, cls, *args, **kwargs) actor._proc_mesh = proc_mesh - - if hasattr(proc_mesh, "_hostname") and hasattr(proc_mesh, "_port"): - host, port = proc_mesh._hostname, proc_mesh._port - await actor.set_env.call(addr=host, port=port) await actor.setup.call() return actor diff --git a/src/forge/controller/service/service.py b/src/forge/controller/service/service.py index 402b9a41..a1fe327e 100644 --- a/src/forge/controller/service/service.py +++ b/src/forge/controller/service/service.py @@ -486,6 +486,10 @@ async def _get_replica(self, sess_id: str | None) -> "Replica": ) async def stop(self): + """ + Stops the service and all managed replicas. + This method should be called when the service is no longer needed. + """ logger.debug("Stopping service...") # Signal shutdown to health loop self._shutdown_requested = True From 76434e33838cf04b58a341174f50bf96eedee617 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 12:30:08 -0700 Subject: [PATCH 05/26] Update --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 78fd0963..c6636dcf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,7 +39,7 @@ jobs: run: python -m pip install torch==2.9.0 --index-url https://download.pytorch.org/whl/test/cu130 - name: Install monarch shell: bash -l {0} - run: python -m pip install monarch-no-torch==0.1.0.dev20250826 --find-links assets/ci + run: pip install torchmonarch - name: Install torchforge shell: bash -l {0} env: From d9542a37ce26e015080baa562037b78a6f741f96 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 12:46:35 -0700 Subject: [PATCH 06/26] Update --- .github/workflows/docs.yml | 15 ++++++++++++ docs/source/conf.py | 4 ++-- src/forge/controller/actor.py | 45 ++++++++++++++++------------------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c6636dcf..2cbec1a4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -52,6 +52,21 @@ jobs: shell: bash -l {0} working-directory: docs run: | + # Set up library paths to ensure all dependencies are available + # This is critical for monarch and other native dependencies that need libpython3.10.so.1.0 + export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH:-}" + + # Also set CUDA paths if needed + if [ -d "/usr/local/cuda-12.9" ]; then + export LD_LIBRARY_PATH="/usr/local/cuda-12.9/compat:${LD_LIBRARY_PATH}" + export CUDA_HOME=/usr/local/cuda-12.9 + fi + + # Verify dependencies can be imported before building docs + echo "Verifying dependencies..." + python -c "import forge; print('✓ forge imported successfully')" + python -c "import monarch; print('✓ monarch imported successfully')" + set +e # Don't exit on error make html SPHINXOPTS="-WT --keep-going" || echo "Build completed with warnings/errors" set -e # Re-enable exit on error for subsequent commands diff --git a/docs/source/conf.py b/docs/source/conf.py index 77ee1338..d34e2c92 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -124,6 +124,7 @@ def get_version_path(): "navbar_center": "navbar-nav", "canonical_url": "https://meta-pytorch.org/forge/", "header_links_before_dropdown": 7, + "show_nav_level": 2, } theme_variables = pytorch_sphinx_theme2.get_theme_variables() @@ -176,7 +177,6 @@ def get_version_path(): "plot_gallery": "True", "promote_jupyter_magic": True, "backreferences_dir": None, - "write_computation_times": True, "show_signature": False, - "write_computation_times": False + "write_computation_times": False, } diff --git a/src/forge/controller/actor.py b/src/forge/controller/actor.py index baa1bd0d..4a5cbf17 100644 --- a/src/forge/controller/actor.py +++ b/src/forge/controller/actor.py @@ -24,40 +24,37 @@ class ForgeActor(Actor): """ Base class for Forge actors with configurable resource attributes. + + The initialization sets up logging configuration with rank/size information and + initializes the actor's process mesh reference. The rank and size are automatically + determined from the current execution context. + + Args: + *args: Variable length argument list passed to the parent Actor class. + **kwargs: Arbitrary keyword arguments passed to the parent Actor class. """ - + procs: int = 1 """Number of processes to use for this actor. Defaults to 1.""" - + hosts: int | None = None """Number of hosts to distribute the actor across. If None, uses as many hosts as needed to accommodate the requested processes. Defaults to None.""" - + with_gpus: bool = False """Whether to allocate GPU resources for this actor. Defaults to False.""" - + num_replicas: int = 1 """Number of replicas to create when spawning as a service. Only applies when using as_service(). Defaults to 1.""" - + mesh_name: str | None = None """Optional name for the process mesh used by this actor. If None, a default name will be generated. Defaults to None.""" - + _extra_config: dict[str, Any] = {} def __init__(self, *args, **kwargs): - """ - Initialize the ForgeActor instance. - - Sets up logging configuration with rank/size information and initializes - the actor's process mesh reference. The rank and size are automatically - determined from the current execution context. - - Args: - *args: Variable length argument list passed to the parent Actor class. - **kwargs: Arbitrary keyword arguments passed to the parent Actor class. - """ if not hasattr(self, "_rank"): self._rank = current_rank().rank if not hasattr(self, "_size"): @@ -100,28 +97,28 @@ def options( Examples: * Pre-configure a service with multiple replicas: - + .. code-block:: python - + service = await MyForgeActor.options(num_replicas=2, procs=2).as_service(...) await service.shutdown() * Default usage without calling options: .. code-block:: python - + service = await MyForgeActor.as_service(...) await service.shutdown() * Pre-configure a single actor - + .. code-block:: python - + actor = await MyForgeActor.options(procs=1, hosts=1).as_actor(...) await actor.shutdown() - + * Default usage without calling options - + .. code-block:: python actor = await MyForgeActor.as_actor(...) From f101a804bd818902347fefede2738870dfaf483a Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 13:16:53 -0700 Subject: [PATCH 07/26] Update --- .github/workflows/docs.yml | 17 ++++++++++++++--- docs/source/api_actors.md | 2 +- docs/source/api_service.md | 4 ++-- docs/source/conf.py | 1 + 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2cbec1a4..73b655c9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -67,9 +67,20 @@ jobs: python -c "import forge; print('✓ forge imported successfully')" python -c "import monarch; print('✓ monarch imported successfully')" - set +e # Don't exit on error - make html SPHINXOPTS="-WT --keep-going" || echo "Build completed with warnings/errors" - set -e # Re-enable exit on error for subsequent commands + # Build docs with -WT (warnings as errors) and --keep-going to see all issues + # Capture exit code but continue to see all errors + set +e + make html SPHINXOPTS="-WT --keep-going" + BUILD_EXIT_CODE=$? + set -e + + # Report results + if [ $BUILD_EXIT_CODE -ne 0 ]; then + echo "❌ Documentation build failed with warnings or errors (exit code: $BUILD_EXIT_CODE)" + exit $BUILD_EXIT_CODE + else + echo "✅ Documentation build completed successfully with no warnings or errors" + fi - name: Upload docs artifact uses: actions/upload-artifact@v4 with: diff --git a/docs/source/api_actors.md b/docs/source/api_actors.md index f086591d..73eae122 100644 --- a/docs/source/api_actors.md +++ b/docs/source/api_actors.md @@ -16,5 +16,5 @@ as building blocks for complex distributed training systems. :members: :undoc-members: :show-inheritance: - :exclude-members: logger, setup, set_env + :exclude-members: logger, setup, set_env, __init__ ``` diff --git a/docs/source/api_service.md b/docs/source/api_service.md index 878d4332..f6a72d45 100644 --- a/docs/source/api_service.md +++ b/docs/source/api_service.md @@ -8,7 +8,7 @@ .. autoclass:: Service :members: call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop :show-inheritance: - :special-members: __init__ + :exclude-members: __init__ ``` ## ServiceActor @@ -17,5 +17,5 @@ .. autoclass:: ServiceActor :members: call, call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop :show-inheritance: - :special-members: __init__ + :exclude-members: __init__ ``` diff --git a/docs/source/conf.py b/docs/source/conf.py index d34e2c92..1979a42c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -125,6 +125,7 @@ def get_version_path(): "canonical_url": "https://meta-pytorch.org/forge/", "header_links_before_dropdown": 7, "show_nav_level": 2, + "show_toc_level": 2 } theme_variables = pytorch_sphinx_theme2.get_theme_variables() From 067ded40bf9a386fecf18f548184a5f5f7092767 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Tue, 7 Oct 2025 13:18:05 -0700 Subject: [PATCH 08/26] Update --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 1979a42c..af4f0983 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -125,7 +125,7 @@ def get_version_path(): "canonical_url": "https://meta-pytorch.org/forge/", "header_links_before_dropdown": 7, "show_nav_level": 2, - "show_toc_level": 2 + "show_toc_level": 2, } theme_variables = pytorch_sphinx_theme2.get_theme_variables() From 499c9199c6bd019a2180fa6f68cad46dbe92570a Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 8 Oct 2025 16:10:15 -0700 Subject: [PATCH 09/26] Update --- docs/source/api.md | 3 ++ docs/source/api_generator.md | 43 ++++++++++++++++ docs/source/api_model.md | 29 +++++++++++ docs/source/api_service.md | 11 +--- docs/source/api_trainer.md | 68 +++++++++++++++++++++++++ docs/source/conf.py | 8 ++- src/forge/actors/reference_model.py | 14 +++-- src/forge/controller/service/service.py | 28 +--------- 8 files changed, 161 insertions(+), 43 deletions(-) create mode 100644 docs/source/api_generator.md create mode 100644 docs/source/api_model.md create mode 100644 docs/source/api_trainer.md diff --git a/docs/source/api.md b/docs/source/api.md index f4ecd01d..1235f9d4 100644 --- a/docs/source/api.md +++ b/docs/source/api.md @@ -29,4 +29,7 @@ APIs provide direct access to the underlying distributed components. :maxdepth: 1 api_actors api_service +api_generator +api_model +api_trainer ``` diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md new file mode 100644 index 00000000..8d17ce7f --- /dev/null +++ b/docs/source/api_generator.md @@ -0,0 +1,43 @@ +# Generator + +```{eval-rst} +.. currentmodule:: forge.actors.policy +``` + +The Generator (Policy) is the core inference engine in TorchForge, built on top of vLLM. +It manages model serving, text generation, and weight updates for reinforcement learning workflows. + +## Policy + +```{eval-rst} +.. autoclass:: Policy + :members: generate, update_weights, get_version, stop + :exclude-members: __init__ +``` + +## Configuration + +### EngineConfig + +```{eval-rst} +.. autoclass:: EngineConfig + :members: + :undoc-members: +``` + +### SamplingConfig + +```{eval-rst} +.. autoclass:: SamplingConfig + :members: + :undoc-members: +``` + +## PolicyWorker + +```{eval-rst} +.. autoclass:: PolicyWorker + :members: execute_model, update, setup_kv_cache + :show-inheritance: + :exclude-members: __init__ +``` diff --git a/docs/source/api_model.md b/docs/source/api_model.md new file mode 100644 index 00000000..94e51478 --- /dev/null +++ b/docs/source/api_model.md @@ -0,0 +1,29 @@ +# Model + +```{eval-rst} +.. currentmodule:: forge.actors.reference_model +``` + +The {class}`forge.actors.reference_model.ReferenceModel` provides a frozen +copy of the policy model used for computing advantages in reinforcement +learning. It performs inference on input sequences and returns logits or +log probabilities for computing KL divergence and other RL metrics. + +## ReferenceModel + +```{eval-rst} +.. autoclass:: forge.actors.reference_model.ReferenceModel + :members: + :undoc-members: + :show-inheritance: +``` + +The ReferenceModel uses a subset of TorchTitan's configuration system: + +- **model**: Model architecture settings (Model dataclass) +- **parallelism**: Parallelism configuration for distributed inference (Parallelism dataclass) +- **checkpoint**: Checkpoint loading settings (Checkpoint dataclass) +- **compile**: Model compilation settings (Compile dataclass) +- **training**: Training configuration for dtype and other settings (Training dataclass) + +For detailed configuration options, refer to the [TorchTitan documentation](https://github.com/pytorch/torchtitan). diff --git a/docs/source/api_service.md b/docs/source/api_service.md index f6a72d45..df2bf3dc 100644 --- a/docs/source/api_service.md +++ b/docs/source/api_service.md @@ -6,16 +6,7 @@ ```{eval-rst} .. autoclass:: Service - :members: call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop - :show-inheritance: - :exclude-members: __init__ -``` -## ServiceActor - -```{eval-rst} -.. autoclass:: ServiceActor - :members: call, call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop + :members: call_all, start_session, get_metrics, get_metrics_summary, terminate_session, stop :show-inheritance: - :exclude-members: __init__ ``` diff --git a/docs/source/api_trainer.md b/docs/source/api_trainer.md new file mode 100644 index 00000000..75aba94f --- /dev/null +++ b/docs/source/api_trainer.md @@ -0,0 +1,68 @@ +# Trainer + +```{eval-rst} +.. currentmodule:: forge.actors.trainer +``` + +The Trainer manages model training in TorchForge, built on top of TorchTitan. +It handles forward/backward passes, weight updates, and checkpoint management for reinforcement learning workflows. + +## RLTrainer + +```{eval-rst} +.. autoclass:: RLTrainer + :members: train_step, push_weights, cleanup + :exclude-members: __init__ +``` + +## Configuration + +The RLTrainer uses TorchTitan's configuration system with the following components: + +### Job Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Job + :members: + :undoc-members: +``` + +### Model Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Model + :members: + :undoc-members: +``` + +### Optimizer Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Optimizer + :members: + :undoc-members: +``` + +### Training Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Training + :members: + :undoc-members: +``` + +### Parallelism Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Parallelism + :members: + :undoc-members: +``` + +### Checkpoint Configuration + +```{eval-rst} +.. autoclass:: torchtitan.config.job_config.Checkpoint + :members: + :undoc-members: +``` diff --git a/docs/source/conf.py b/docs/source/conf.py index af4f0983..da7c0014 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -162,11 +162,15 @@ def get_version_path(): autodoc_default_options = { "members": True, "member-order": "bysource", - "special-members": "__init__", - "undoc-members": True, "exclude-members": "__weakref__", + "private-members": False, } +# Suppress warnings from third-party library docstrings +suppress_warnings = [ + "docutils", # Suppress docstring formatting issues from third-party libraries +] + # -- Sphinx Gallery configuration ------------------------------------------- sphinx_gallery_conf = { diff --git a/src/forge/actors/reference_model.py b/src/forge/actors/reference_model.py index cc57e524..969c686a 100644 --- a/src/forge/actors/reference_model.py +++ b/src/forge/actors/reference_model.py @@ -13,6 +13,11 @@ from dataclasses import dataclass, field, fields import torch + +from forge.controller import ForgeActor +from forge.observability.metrics import record_metric, Reduce +from forge.observability.perf_tracker import Tracer +from forge.util.ops import compute_logprobs from monarch.actor import current_rank, current_size, endpoint from torch.distributed.tensor import DTensor @@ -26,17 +31,16 @@ from torchtitan.experiments.forge.engine import ForgeEngine from torchtitan.experiments.forge.job_config import ForgeJobConfig -from forge.controller import ForgeActor -from forge.observability.metrics import record_metric, Reduce -from forge.observability.perf_tracker import Tracer -from forge.util.ops import compute_logprobs - logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @dataclass class ReferenceModel(ForgeActor): + """ + Reference model implementation for the TorchForge service. + """ + # Refer to titan JobConfig for enabling more ForgeEngine configuration model: Model = field(default_factory=Model) parallelism: Parallelism = field(default_factory=Parallelism) diff --git a/src/forge/controller/service/service.py b/src/forge/controller/service/service.py index a1fe327e..94638c07 100644 --- a/src/forge/controller/service/service.py +++ b/src/forge/controller/service/service.py @@ -38,8 +38,6 @@ import uuid from typing import Dict, List -from monarch.actor import Actor, endpoint - from forge.controller.service.interface import _session_context, Session from forge.controller.service.metrics import ServiceMetrics @@ -52,6 +50,8 @@ ) from forge.types import ServiceConfig +from monarch.actor import Actor, endpoint + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -68,13 +68,6 @@ class Service: actor_def: Actor class definition to instantiate on each replica *actor_args: Positional arguments passed to actor constructor **actor_kwargs: Keyword arguments passed to actor constructor - - - Attributes: - _cfg: Service configuration - _replicas: List of managed replica instances - _active_sessions: Currently active sessions - _metrics: Aggregated service and replica metrics """ def __init__( @@ -609,12 +602,6 @@ class ServiceActor(Actor): actor_def: Actor class definition to instantiate on each replica *actor_args: Positional arguments passed to actor constructor **actor_kwargs: Keyword arguments passed to actor constructor - - Attributes: - _cfg: Service configuration - _replicas: List of managed replica instances - _active_sessions: Currently active sessions - _metrics: Aggregated service and replica metrics """ def __init__(self, cfg: ServiceConfig, actor_def, actor_kwargs: dict): @@ -1163,14 +1150,3 @@ def _get_internal_state(self) -> dict: def __repr__(self): return f"Service(actor={self._actor_def.__name__})" - - -# Copy docstrings from Service to ServiceActor methods for Sphinx autodoc -# This ensures ServiceActor methods have complete docstrings while avoiding duplication -ServiceActor.call.__doc__ = Service._call.__doc__ -ServiceActor.call_all.__doc__ = Service.call_all.__doc__ -ServiceActor.start_session.__doc__ = Service.start_session.__doc__ -ServiceActor.get_metrics.__doc__ = Service.get_metrics.__doc__ -ServiceActor.get_metrics_summary.__doc__ = Service.get_metrics_summary.__doc__ -ServiceActor.terminate_session.__doc__ = Service.terminate_session.__doc__ -ServiceActor.stop.__doc__ = Service.stop.__doc__ From ef3524477f378478ca65fd55b6bc41a73985bae3 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Wed, 8 Oct 2025 16:15:29 -0700 Subject: [PATCH 10/26] precommit --- src/forge/actors/reference_model.py | 10 +++++----- src/forge/controller/service/service.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/forge/actors/reference_model.py b/src/forge/actors/reference_model.py index 969c686a..bfe9f949 100644 --- a/src/forge/actors/reference_model.py +++ b/src/forge/actors/reference_model.py @@ -13,11 +13,6 @@ from dataclasses import dataclass, field, fields import torch - -from forge.controller import ForgeActor -from forge.observability.metrics import record_metric, Reduce -from forge.observability.perf_tracker import Tracer -from forge.util.ops import compute_logprobs from monarch.actor import current_rank, current_size, endpoint from torch.distributed.tensor import DTensor @@ -31,6 +26,11 @@ from torchtitan.experiments.forge.engine import ForgeEngine from torchtitan.experiments.forge.job_config import ForgeJobConfig +from forge.controller import ForgeActor +from forge.observability.metrics import record_metric, Reduce +from forge.observability.perf_tracker import Tracer +from forge.util.ops import compute_logprobs + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) diff --git a/src/forge/controller/service/service.py b/src/forge/controller/service/service.py index 94638c07..1413cbba 100644 --- a/src/forge/controller/service/service.py +++ b/src/forge/controller/service/service.py @@ -38,6 +38,8 @@ import uuid from typing import Dict, List +from monarch.actor import Actor, endpoint + from forge.controller.service.interface import _session_context, Session from forge.controller.service.metrics import ServiceMetrics @@ -50,8 +52,6 @@ ) from forge.types import ServiceConfig -from monarch.actor import Actor, endpoint - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) From bbd8d8fa758df9dc9e7c51021c30ee611a7856bf Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 08:56:50 -0700 Subject: [PATCH 11/26] Fixes --- .github/workflows/docs.yml | 2 +- docs/source/conf.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 73b655c9..9b16e58b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -70,7 +70,7 @@ jobs: # Build docs with -WT (warnings as errors) and --keep-going to see all issues # Capture exit code but continue to see all errors set +e - make html SPHINXOPTS="-WT --keep-going" + make html SPHINXOPTS="--keep-going" BUILD_EXIT_CODE=$? set -e diff --git a/docs/source/conf.py b/docs/source/conf.py index da7c0014..78d274f9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -169,8 +169,13 @@ def get_version_path(): # Suppress warnings from third-party library docstrings suppress_warnings = [ "docutils", # Suppress docstring formatting issues from third-party libraries + "app.add_node", # Suppress node warnings + "app.add_directive", # Suppress directive warnings ] +# Treat warnings as non-fatal - continue build even if there are warnings +keep_warnings = True + # -- Sphinx Gallery configuration ------------------------------------------- sphinx_gallery_conf = { From cba68df8ebb6c7694d9ffe775231be398f175a77 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 10:24:16 -0700 Subject: [PATCH 12/26] Update --- docs/source/api_generator.md | 3 ++- docs/source/conf.py | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index 8d17ce7f..6f563058 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -11,8 +11,9 @@ It manages model serving, text generation, and weight updates for reinforcement ```{eval-rst} .. autoclass:: Policy - :members: generate, update_weights, get_version, stop + :members: launch, generate, update_weights, get_version, stop :exclude-members: __init__ + :no-inherited-members: ``` ## Configuration diff --git a/docs/source/conf.py b/docs/source/conf.py index 78d274f9..7d7fa197 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,6 +58,7 @@ def get_version_path(): "myst_parser", "sphinx.ext.autodoc", "sphinx.ext.autosummary", + "sphinx-autodoc-typehints", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", @@ -166,6 +167,13 @@ def get_version_path(): "private-members": False, } +# Autodoc configuration for cleaner signatures +autodoc_preserve_defaults = True # Preserves default values without expansion +autodoc_typehints = "description" # Move type hints to description instead of signature +autodoc_typehints_description_target = ( + "documented" # Only add types to documented params +) + # Suppress warnings from third-party library docstrings suppress_warnings = [ "docutils", # Suppress docstring formatting issues from third-party libraries From f828bc17f74dd2cb6b962457e939600031b9a95f Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 10:49:33 -0700 Subject: [PATCH 13/26] Update --- docs/requirements.txt | 1 + src/forge/actors/trainer.py | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 8846bc62..78dddcdf 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,3 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 +sphinx-autodoc-typehints==3.4.0 diff --git a/src/forge/actors/trainer.py b/src/forge/actors/trainer.py index 4ffc6300..b9882c6d 100644 --- a/src/forge/actors/trainer.py +++ b/src/forge/actors/trainer.py @@ -18,6 +18,17 @@ import torch.distributed.checkpoint as dcp import torchstore as ts +from forge.actors._torchstore_utils import ( + DcpHandle, + get_dcp_whole_state_dict_key, + get_param_key, +) + +from forge.controller import ForgeActor +from forge.data.utils import batch_to_device +from forge.observability.metrics import record_metric, Reduce +from forge.observability.perf_tracker import Tracer + from monarch.actor import current_rank, current_size, endpoint from torch import Tensor from torch.distributed.checkpoint._nested_dict import flatten_state_dict @@ -39,17 +50,6 @@ from torchtitan.experiments.forge.engine import ForgeEngine from torchtitan.experiments.forge.job_config import ForgeJobConfig -from forge.actors._torchstore_utils import ( - DcpHandle, - get_dcp_whole_state_dict_key, - get_param_key, -) - -from forge.controller import ForgeActor -from forge.data.utils import batch_to_device -from forge.observability.metrics import record_metric, Reduce -from forge.observability.perf_tracker import Tracer - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -95,6 +95,10 @@ def cleanup_old_weight_versions( @dataclass class RLTrainer(ForgeActor): + """ + RL Trainer implementation for the TorchForge service. + """ + job: Job = field(default_factory=Job) model: Model = field(default_factory=Model) optimizer: Optimizer = field(default_factory=Optimizer) From 60ec7e781ec5f968a2bd9dd59d52dcfa8dbab4f5 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 10:53:38 -0700 Subject: [PATCH 14/26] Precommit --- src/forge/actors/trainer.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/forge/actors/trainer.py b/src/forge/actors/trainer.py index b9882c6d..48628668 100644 --- a/src/forge/actors/trainer.py +++ b/src/forge/actors/trainer.py @@ -18,17 +18,6 @@ import torch.distributed.checkpoint as dcp import torchstore as ts -from forge.actors._torchstore_utils import ( - DcpHandle, - get_dcp_whole_state_dict_key, - get_param_key, -) - -from forge.controller import ForgeActor -from forge.data.utils import batch_to_device -from forge.observability.metrics import record_metric, Reduce -from forge.observability.perf_tracker import Tracer - from monarch.actor import current_rank, current_size, endpoint from torch import Tensor from torch.distributed.checkpoint._nested_dict import flatten_state_dict @@ -50,6 +39,17 @@ from torchtitan.experiments.forge.engine import ForgeEngine from torchtitan.experiments.forge.job_config import ForgeJobConfig +from forge.actors._torchstore_utils import ( + DcpHandle, + get_dcp_whole_state_dict_key, + get_param_key, +) + +from forge.controller import ForgeActor +from forge.data.utils import batch_to_device +from forge.observability.metrics import record_metric, Reduce +from forge.observability.perf_tracker import Tracer + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) From 754f7f7a945a7a10b1887d1f9a6ea8cb804948ea Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 11:27:20 -0700 Subject: [PATCH 15/26] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 78dddcdf..601236d5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.4.0 +sphinx-autodoc-typehints==3.2.0 From cf66872bae7b8a6fda310cf92a6c6043a0ae32ce Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 11:37:48 -0700 Subject: [PATCH 16/26] Update --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 7d7fa197..3037f066 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -58,7 +58,7 @@ def get_version_path(): "myst_parser", "sphinx.ext.autodoc", "sphinx.ext.autosummary", - "sphinx-autodoc-typehints", + "sphinx_autodoc_typehints", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", From b7b39c1015586bcbacaab2dafaddc83876dbc03d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 11:51:02 -0700 Subject: [PATCH 17/26] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 601236d5..f783fde7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.2.0 +sphinx-autodoc-typehints==3.0.0 From dfb2b01dea6918ffc36edd6a08f77e12f0173ea0 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 12:14:07 -0700 Subject: [PATCH 18/26] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f783fde7..78dddcdf 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.0.0 +sphinx-autodoc-typehints==3.4.0 From 678c82c4ab0664afbed5cc54a854a363502b8170 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 12:22:33 -0700 Subject: [PATCH 19/26] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 78dddcdf..5b35c89c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.4.0 +sphinx-autodoc-typehints==3.0.1 From 8162490538dbc42496b3d64dc3f855dbd30beb66 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 13:08:17 -0700 Subject: [PATCH 20/26] Update --- docs/requirements.txt | 2 +- docs/source/api_generator.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 5b35c89c..f783fde7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.0.1 +sphinx-autodoc-typehints==3.0.0 diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index 6f563058..c4c448ec 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -4,7 +4,8 @@ .. currentmodule:: forge.actors.policy ``` -The Generator (Policy) is the core inference engine in TorchForge, built on top of vLLM. +The Generator (Policy) is the core inference engine in TorchForge, +built on top of [vLLM](https://docs.vllm.ai/en/latest/). It manages model serving, text generation, and weight updates for reinforcement learning workflows. ## Policy From 5d1c5c9b4ae32ea5847a61efda62f802fcb66fe8 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 13:46:45 -0700 Subject: [PATCH 21/26] Update --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index f783fde7..525ca1e8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -6,4 +6,4 @@ sphinxcontrib-mermaid==1.0.0 sphinx-gallery==0.19.0 myst-parser #==0.18.1 # if want to contribute in markdown sphinx-sitemap==2.7.1 -sphinx-autodoc-typehints==3.0.0 +sphinx-autodoc-typehints==1.25.3 From 197d24d233be2531204dbfe4ef760072b766d576 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Thu, 9 Oct 2025 14:35:55 -0700 Subject: [PATCH 22/26] Update --- docs/source/api_generator.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/api_generator.md b/docs/source/api_generator.md index c4c448ec..a0bb67f3 100644 --- a/docs/source/api_generator.md +++ b/docs/source/api_generator.md @@ -25,6 +25,7 @@ It manages model serving, text generation, and weight updates for reinforcement .. autoclass:: EngineConfig :members: :undoc-members: + :no-inherited-members: ``` ### SamplingConfig From bb6b58524608da79604295a0a0a1ff3b21a29c16 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 10 Oct 2025 10:18:32 -0700 Subject: [PATCH 23/26] Update --- docs/source/_static/custom.css | 108 +++++++++++++++++++++++++++++++++ docs/source/_static/custom.js | 93 ++++++++++++++++++++++++++++ docs/source/conf.py | 7 +++ 3 files changed, 208 insertions(+) create mode 100644 docs/source/_static/custom.css create mode 100644 docs/source/_static/custom.js diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 00000000..db41acaa --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,108 @@ +/* Custom CSS for collapsible parameter lists */ + +/* Hide parameters in signatures */ +.sig-param-hidden { + display: none !important; +} + +/* Inline toggle button for signatures */ +.params-toggle-btn-inline { + display: inline; + padding: 0.2rem 0.5rem; + margin: 0 0.25rem; + background-color: var(--pst-color-background); + border: var(--pst-color-border); + border-radius: 3px; + cursor: pointer; + font-size: 0.85em; + font-family: var(--pst-font-family-base); + color: #495057; + transition: all 0.2s ease; + vertical-align: middle; +} + +.params-toggle-btn-inline:hover { + background-color: #e9ecef; + border-color: #adb5bd; +} + +.params-toggle-btn-inline:focus { + outline: none; +} + +.params-toggle-btn-inline:focus-visible { + outline: 2px solid #007bff; + outline-offset: 2px; +} + +.toggle-icon { + display: inline-block; + font-size: 0.8em; + transition: transform 0.2s ease; +} + +/* Wrapper for the button */ +.sig-params-wrapper { + display: inline; +} + +/* Old styles for field-list collapsing (kept for backward compatibility) */ +.collapsible-params { + margin: 1rem 0; +} + +.params-toggle-btn { + display: inline-block; + padding: 0.5rem 1rem; + margin-bottom: 0.5rem; + background-color: #f8f9fa; + border: 1px solid #dee2e6; + border-radius: 4px; + cursor: pointer; + font-size: 0.9rem; + color: #495057; + transition: all 0.3s ease; +} + +.params-toggle-btn:hover { + background-color: #e9ecef; + border-color: #adb5bd; +} + +.params-toggle-btn:focus { + outline: 2px solid #007bff; + outline-offset: 2px; +} + +.params-content { + max-height: 10000px; + overflow: hidden; + transition: max-height 0.5s ease, opacity 0.3s ease; + opacity: 1; +} + +.params-content.collapsed { + max-height: 0; + opacity: 0; +} + +/* Ensure the collapsed parameters look good */ +.params-content dl.field-list { + margin-top: 0; +} + +.params-content > dt { + margin-top: 0.5rem; +} + +.params-content > dt:first-child { + margin-top: 0; +} + +/* Responsive adjustments */ +@media (max-width: 768px) { + .params-toggle-btn { + width: 100%; + text-align: left; + } +} diff --git a/docs/source/_static/custom.js b/docs/source/_static/custom.js new file mode 100644 index 00000000..415592d3 --- /dev/null +++ b/docs/source/_static/custom.js @@ -0,0 +1,93 @@ +// Custom JavaScript to make long parameter lists in class signatures collapsible +document.addEventListener('DOMContentLoaded', function() { + console.log('Collapsible parameters script loaded'); + + // Find all class/function signatures + const signatures = document.querySelectorAll('dl.py.class > dt, dl.py.function > dt, dl.py.method > dt'); + + signatures.forEach(function(signature) { + // Find all parameter elements in the signature + const params = signature.querySelectorAll('em.sig-param, .sig-param'); + + console.log(`Found signature with ${params.length} parameters`); + + // Only make it collapsible if there are more than 10 parameters + if (params.length > 10) { + console.log('Creating collapsible structure for signature with', params.length, 'parameters'); + + const visibleCount = 5; + const hiddenCount = params.length - visibleCount; + + // Create a wrapper div for the toggle button + const wrapper = document.createElement('span'); + wrapper.className = 'sig-params-wrapper'; + wrapper.style.display = 'inline'; + + // Create toggle button + const toggleBtn = document.createElement('button'); + toggleBtn.className = 'params-toggle-btn-inline'; + toggleBtn.innerHTML = ` Show More`; + toggleBtn.setAttribute('aria-expanded', 'false'); + toggleBtn.title = `Show ${hiddenCount} more parameters`; + + // Collect all nodes to hide (params and text nodes between them) + const nodesToHide = []; + + // Hide parameters after the first 3 + let insertedButton = false; + params.forEach(function(param, index) { + if (index >= visibleCount) { + // Add 'hidden' class to hide the parameter + param.classList.add('sig-param-hidden'); + nodesToHide.push(param); + + // Also hide the text node (comma/space) that follows this parameter + let nextNode = param.nextSibling; + while (nextNode && nextNode.nodeType === Node.TEXT_NODE) { + const textSpan = document.createElement('span'); + textSpan.className = 'sig-param-hidden'; + textSpan.textContent = nextNode.textContent; + nextNode.parentNode.replaceChild(textSpan, nextNode); + nodesToHide.push(textSpan); + break; + } + + // Insert the toggle button before the first hidden parameter + if (!insertedButton) { + param.parentNode.insertBefore(wrapper, param); + wrapper.appendChild(toggleBtn); + insertedButton = true; + } + } + }); + + // Add click handler to toggle + toggleBtn.addEventListener('click', function(e) { + e.preventDefault(); + e.stopPropagation(); + + const isExpanded = toggleBtn.getAttribute('aria-expanded') === 'true'; + + if (isExpanded) { + // Collapse: hide parameters again + nodesToHide.forEach(function(node) { + node.classList.add('sig-param-hidden'); + }); + toggleBtn.setAttribute('aria-expanded', 'false'); + toggleBtn.innerHTML = ` Show More`; + toggleBtn.title = `Show ${hiddenCount} more parameters`; + } else { + // Expand: show all parameters + nodesToHide.forEach(function(node) { + node.classList.remove('sig-param-hidden'); + }); + toggleBtn.setAttribute('aria-expanded', 'true'); + toggleBtn.innerHTML = ` Hide`; + toggleBtn.title = `Hide ${hiddenCount} parameters`; + } + }); + + console.log('Collapsible structure created successfully'); + } + }); +}); diff --git a/docs/source/conf.py b/docs/source/conf.py index 3037f066..c5cdb862 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -75,12 +75,19 @@ def get_version_path(): ] sitemap_url_scheme = "{link}" +# Ensure static files use relative paths +html_static_path = ["_static"] + templates_path = [ "_templates", os.path.join(os.path.dirname(pytorch_sphinx_theme2.__file__), "templates"), ] exclude_patterns = ["tutorials/index.rst", "tutorials/template_tutorial.rst"] +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_js_files = ["custom.js"] + sys.path.insert(0, os.path.abspath(".")) sys.path.insert(0, os.path.abspath("../../src")) html_theme = "pytorch_sphinx_theme2" From 8c186b110e765a712aae097ed0dac461b0791456 Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 10 Oct 2025 11:14:08 -0700 Subject: [PATCH 24/26] Update --- docs/source/_static/custom.css | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css index db41acaa..89854cc8 100644 --- a/docs/source/_static/custom.css +++ b/docs/source/_static/custom.css @@ -11,30 +11,25 @@ padding: 0.2rem 0.5rem; margin: 0 0.25rem; background-color: var(--pst-color-background); - border: var(--pst-color-border); + border: 1px solid var(--pst-color-border); border-radius: 3px; cursor: pointer; font-size: 0.85em; font-family: var(--pst-font-family-base); - color: #495057; + color: var(--pst-color-primary); transition: all 0.2s ease; vertical-align: middle; } .params-toggle-btn-inline:hover { - background-color: #e9ecef; - border-color: #adb5bd; + background-color: var(--pst-color-background); + border-color: var(--pst-color-border); } .params-toggle-btn-inline:focus { outline: none; } -.params-toggle-btn-inline:focus-visible { - outline: 2px solid #007bff; - outline-offset: 2px; -} - .toggle-icon { display: inline-block; font-size: 0.8em; @@ -55,23 +50,18 @@ display: inline-block; padding: 0.5rem 1rem; margin-bottom: 0.5rem; - background-color: #f8f9fa; - border: 1px solid #dee2e6; + background-color: var(--pst-color-background); + border: 1px solid var(--pst-color-border); border-radius: 4px; cursor: pointer; font-size: 0.9rem; - color: #495057; + color: var(--pst-color-primary); transition: all 0.3s ease; } .params-toggle-btn:hover { - background-color: #e9ecef; - border-color: #adb5bd; -} - -.params-toggle-btn:focus { - outline: 2px solid #007bff; - outline-offset: 2px; + background-color: var(--pst-color-background); + border-color: var(--pst-color-border); } .params-content { From 25221e71c2c5bd9fbec807a4f051ae28748e2f2c Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 10 Oct 2025 14:22:52 -0700 Subject: [PATCH 25/26] Update --- docs/source/conf.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index c5cdb862..55d57680 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -191,6 +191,13 @@ def get_version_path(): # Treat warnings as non-fatal - continue build even if there are warnings keep_warnings = True +# Napoleon settings for Google-style docstrings (from torchtitan and other dependencies) +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_preprocess_types = False + # -- Sphinx Gallery configuration ------------------------------------------- sphinx_gallery_conf = { From f3c79f68644aff00f820cb039d6a88cfc99a134d Mon Sep 17 00:00:00 2001 From: Svetlana Karslioglu Date: Fri, 10 Oct 2025 15:58:34 -0700 Subject: [PATCH 26/26] Update --- docs/source/conf.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 55d57680..ee9d6214 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -186,17 +186,25 @@ def get_version_path(): "docutils", # Suppress docstring formatting issues from third-party libraries "app.add_node", # Suppress node warnings "app.add_directive", # Suppress directive warnings + "ref.class", # Suppress missing reference warnings + "ref.func", # Suppress missing function reference warnings + "ref.meth", # Suppress missing method reference warnings ] # Treat warnings as non-fatal - continue build even if there are warnings keep_warnings = True +# Don't fail the build on warnings - important for handling third-party library docstrings +# This is especially important when dependencies (like torchtitan) have RST formatting +# that may not be perfect but works with Napoleon extension +nitpicky = False # Don't be overly strict about references + # Napoleon settings for Google-style docstrings (from torchtitan and other dependencies) napoleon_google_docstring = True napoleon_numpy_docstring = True napoleon_use_param = True napoleon_use_rtype = True -napoleon_preprocess_types = False +napoleon_use_ivar = True # -- Sphinx Gallery configuration -------------------------------------------