fix: preserve special tokens during decode for prompt block separators (#737)

Signed-off-by: Anthony Casagrande <acasagrande@nvidia.com>

Author: ajcasagrande

src/aiperf/common/tokenizer.py

@@ -152,7 +152,10 @@ def __init__(self) -> None:
         self._resolved_name: str | None = None
         self._call_args = {"add_special_tokens": False}
         self._encode_args = {"add_special_tokens": False}
-        self._decode_args = {"skip_special_tokens": True}
+        # Prompt generation inserts BOS/EOS tokens as block separators
+        # (see PromptGenerator._build_token_sequence). Skipping special tokens
+        # during decode would silently strip those separators.
+        self._decode_args = {"skip_special_tokens": False}
 
     def _require_init(self) -> None:
         """Raise NotInitializedError if tokenizer is not initialized."""

tests/unit/common/test_tokenizer_kwarg_overrides.py

@@ -189,7 +189,7 @@ def test_standard_tokenizer_keeps_defaults(self):
         tok = self._make_tokenizer(StandardTokenizerBackend())
         assert tok._encode_args == {"add_special_tokens": False}
         assert tok._call_args == {"add_special_tokens": False}
-        assert tok._decode_args == {"skip_special_tokens": True}
+        assert tok._decode_args == {"skip_special_tokens": False}
 
     def test_kimi_like_overrides_encode_and_call_args(self):
         tok = self._make_tokenizer(KimiLikeTokenizerBackend())
@@ -210,14 +210,14 @@ def test_mismatched_call_encode_sets_args_independently(self):
         tok = self._make_tokenizer(MismatchedCallEncodeBackend())
         assert tok._encode_args == {"allow_special_tokens": False}
         assert tok._call_args == {"add_special_tokens": False}
-        assert tok._decode_args == {"skip_special_tokens": True}
+        assert tok._decode_args == {"skip_special_tokens": False}
 
     def test_none_tokenizer_is_noop(self):
         tok = Tokenizer()
         tok._apply_kwarg_overrides()
         assert tok._encode_args == {"add_special_tokens": False}
         assert tok._call_args == {"add_special_tokens": False}
-        assert tok._decode_args == {"skip_special_tokens": True}
+        assert tok._decode_args == {"skip_special_tokens": False}
 
 
 # -- End-to-end: encode/decode through Tokenizer wrapper --