|
| 1 | +from abc import ABC, abstractmethod |
| 2 | +from typing import Any, Dict, List, Tuple |
| 3 | + |
| 4 | +from trinity.algorithm.sample_strategy.utils import representative_sample, to_data_proto |
| 5 | +from trinity.buffer import get_buffer_reader |
| 6 | +from trinity.common.config import BufferConfig |
| 7 | +from trinity.common.experience import Experiences |
| 8 | +from trinity.utils.registry import Registry |
| 9 | +from trinity.utils.timer import Timer |
| 10 | + |
| 11 | +SAMPLE_STRATEGY = Registry("sample_strategy") |
| 12 | + |
| 13 | + |
| 14 | +class SampleStrategy(ABC): |
| 15 | + def __init__(self, buffer_config: BufferConfig, trainer_type: str, **kwargs): |
| 16 | + self.pad_token_id = buffer_config.pad_token_id |
| 17 | + self.trainer_type = trainer_type |
| 18 | + |
| 19 | + @abstractmethod |
| 20 | + def sample(self, step: int) -> Tuple[Any, Dict, List]: |
| 21 | + """Sample experiences from buffer. |
| 22 | +
|
| 23 | + Args: |
| 24 | + step (`int`): The step number of current step. |
| 25 | +
|
| 26 | + Returns: |
| 27 | + `Any`: The sampled experiences. |
| 28 | + `Dict`: Metrics for logging. |
| 29 | + `List`: Representative experiences for logging. |
| 30 | + """ |
| 31 | + |
| 32 | + @classmethod |
| 33 | + def default_args(cls) -> dict: |
| 34 | + return {} |
| 35 | + |
| 36 | + |
| 37 | +@SAMPLE_STRATEGY.register_module("warmup") |
| 38 | +class WarmupSampleStrategy(SampleStrategy): |
| 39 | + """The default sample strategy.""" |
| 40 | + |
| 41 | + def __init__(self, buffer_config: BufferConfig, trainer_type: str, **kwargs): |
| 42 | + super().__init__(buffer_config, trainer_type) |
| 43 | + self.exp_buffer = get_buffer_reader( |
| 44 | + buffer_config.trainer_input.experience_buffer, buffer_config # type: ignore |
| 45 | + ) |
| 46 | + self.sft_warmup_steps = buffer_config.trainer_input.sft_warmup_steps |
| 47 | + if self.sft_warmup_steps > 0 and buffer_config.trainer_input.sft_warmup_dataset is None: |
| 48 | + raise ValueError("sft_warmup_dataset is required when sft_warmup_steps > 0") |
| 49 | + if buffer_config.trainer_input.sft_warmup_dataset is not None: |
| 50 | + self.sft_buffer = get_buffer_reader( |
| 51 | + buffer_config.trainer_input.sft_warmup_dataset, buffer_config |
| 52 | + ) |
| 53 | + else: |
| 54 | + self.sft_buffer = None |
| 55 | + |
| 56 | + def sample(self, step: int, **kwargs) -> Tuple[Any, Dict, List]: |
| 57 | + metrics = {} |
| 58 | + with Timer(metrics, "read_time"): |
| 59 | + if step <= self.sft_warmup_steps: |
| 60 | + exp_list = self.sft_buffer.read() |
| 61 | + else: |
| 62 | + exp_list = self.exp_buffer.read() |
| 63 | + repr_samples = representative_sample(exp_list) |
| 64 | + with Timer(metrics, "gather_time"): |
| 65 | + exps = Experiences.gather_experiences(exp_list, self.pad_token_id) # type: ignore |
| 66 | + if self.trainer_type == "verl": |
| 67 | + with Timer(metrics, "convert_time"): |
| 68 | + data = to_data_proto(exps) |
| 69 | + return data, metrics, repr_samples |
| 70 | + else: |
| 71 | + raise NotImplementedError(f"backend {self.trainer_type} is not supported") |
| 72 | + |
| 73 | + |
| 74 | +@SAMPLE_STRATEGY.register_module("default") |
| 75 | +class DefaultSampleStrategy(SampleStrategy): |
| 76 | + def __init__(self, buffer_config: BufferConfig, trainer_type: str, **kwargs): |
| 77 | + super().__init__(buffer_config, trainer_type) |
| 78 | + self.exp_buffer = get_buffer_reader( |
| 79 | + buffer_config.trainer_input.experience_buffer, buffer_config # type: ignore |
| 80 | + ) |
| 81 | + |
| 82 | + def sample(self, step: int, **kwargs) -> Tuple[Any, Dict, List]: |
| 83 | + metrics = {} |
| 84 | + with Timer(metrics, "read_time"): |
| 85 | + exp_list = self.exp_buffer.read() |
| 86 | + repr_samples = representative_sample(exp_list) |
| 87 | + with Timer(metrics, "gather_time"): |
| 88 | + exps = Experiences.gather_experiences(exp_list, self.pad_token_id) # type: ignore |
| 89 | + if self.trainer_type == "verl": |
| 90 | + with Timer(metrics, "convert_time"): |
| 91 | + data = to_data_proto(exps) |
| 92 | + return data, metrics, repr_samples |
| 93 | + else: |
| 94 | + raise NotImplementedError(f"backend {self.trainer_type} is not supported") |
| 95 | + |
| 96 | + |
| 97 | +@SAMPLE_STRATEGY.register_module("dpo") |
| 98 | +class DPOSampleStrategy(WarmupSampleStrategy): |
| 99 | + def sample(self, step: int, **kwargs) -> Tuple[Any, Dict, List]: |
| 100 | + metrics = {} |
| 101 | + with Timer(metrics, "read_time"): |
| 102 | + if step <= self.sft_warmup_steps: |
| 103 | + exp_list = self.sft_buffer.read() |
| 104 | + else: |
| 105 | + exp_list = self.exp_buffer.read() |
| 106 | + repr_samples = representative_sample(exp_list) |
| 107 | + with Timer(metrics, "gather_time"): |
| 108 | + exps = Experiences.gather_dpo_experiences(exp_list, pad_token_id=self.pad_token_id) # type: ignore |
| 109 | + if self.trainer_type == "verl": |
| 110 | + with Timer(metrics, "convert_time"): |
| 111 | + data = to_data_proto(exps) |
| 112 | + return data, metrics, repr_samples |
| 113 | + else: |
| 114 | + raise NotImplementedError(f"backend {self.trainer_type} is not supported") |
0 commit comments