Set add special tokens to false (#672)

* Set add special tokens to false

* formatting

* Fix tests

* Ignore special tokens and add test

* Add a wrapper around huggingface tokenizer

---------

Co-authored-by: David Yastremsky <[email protected]>
Co-authored-by: Hyunjae Woo <[email protected]>

Author: 3 people

src/c++/perf_analyzer/genai-perf/genai_perf/tokenizer.py

@@ -14,43 +14,65 @@
 
 import contextlib
 import io
-from typing import Union
+from typing import List
 
 from genai_perf.exceptions import GenAIPerfException
 
 # Silence tokenizer warning on import
 with contextlib.redirect_stdout(io.StringIO()) as stdout, contextlib.redirect_stderr(
     io.StringIO()
 ) as stderr:
-    from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
+    from transformers import AutoTokenizer, BatchEncoding
     from transformers import logging as token_logger
 
     token_logger.set_verbosity_error()
 
-Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast]
 DEFAULT_TOKENIZER = "hf-internal-testing/llama-tokenizer"
 
 
-def get_tokenizer(
-    tokenizer_model: str,
-) -> Tokenizer:
+class Tokenizer:
     """
-    Download the tokenizer from Huggingface.co
+    A small wrapper class around Huggingface Tokenizer
     """
-    try:
-        # Silence tokenizer warning on first use
-        with contextlib.redirect_stdout(
-            io.StringIO()
-        ) as stdout, contextlib.redirect_stderr(io.StringIO()) as stderr:
-            tokenizer = AutoTokenizer.from_pretrained(tokenizer_model)
-    except Exception as e:
-        raise GenAIPerfException(e)
-
-    # Disable add_bos_token so that llama tokenizer does not add bos token
-    # (aka. beginning-of-sentence) to the beginning of every response
-    # outputs, increasing the token count by 1 for each output response.
-    # Note: The type is being ignored here, because not all tokenizers have
-    # an add_bos_token variable.
-    tokenizer.add_bos_token = False  # type: ignore
-
-    return tokenizer
+
+    def __init__(self, name: str) -> None:
+        """
+        Initialize by downloading the tokenizer from Huggingface.co
+        """
+        try:
+            # Silence tokenizer warning on first use
+            with contextlib.redirect_stdout(
+                io.StringIO()
+            ) as stdout, contextlib.redirect_stderr(io.StringIO()) as stderr:
+                tokenizer = AutoTokenizer.from_pretrained(name)
+        except Exception as e:
+            raise GenAIPerfException(e)
+
+        self._tokenizer = tokenizer
+
+        # default tokenizer parameters for __call__, encode, decode methods
+        self._call_args = {"add_special_tokens": False}
+        self._encode_args = {"add_special_tokens": False}
+        self._decode_args = {"skip_special_tokens": True}
+
+    def __call__(self, text, **kwargs) -> BatchEncoding:
+        self._call_args.update(kwargs)
+        return self._tokenizer(text, **self._call_args)
+
+    def encode(self, text, **kwargs) -> List[int]:
+        self._encode_args.update(kwargs)
+        return self._tokenizer.encode(text, **self._encode_args)
+
+    def decode(self, token_ids, **kwargs) -> str:
+        self._decode_args.update(kwargs)
+        return self._tokenizer.decode(token_ids, **self._decode_args)
+
+    def __repr__(self) -> str:
+        return self._tokenizer.__repr__()
+
+
+def get_tokenizer(tokenizer_model: str) -> Tokenizer:
+    """
+    Return tokenizer for the given model name
+    """
+    return Tokenizer(tokenizer_model)

src/c++/perf_analyzer/genai-perf/tests/test_llm_metrics.py

@@ -33,10 +33,8 @@
 
 import numpy as np
 import pytest
-from genai_perf.llm_inputs.llm_inputs import OutputFormat
-from genai_perf.llm_metrics import LLMMetrics, LLMProfileDataParser
+from genai_perf.llm_metrics import LLMMetrics, LLMProfileDataParser, ResponseFormat
 from genai_perf.tokenizer import DEFAULT_TOKENIZER, get_tokenizer
-from transformers import AutoTokenizer
 
 
 def ns_to_sec(ns: int) -> Union[int, float]:
@@ -380,6 +378,42 @@ def test_merged_sse_response(self, mock_read_write: pytest.MonkeyPatch) -> None:
         pd._preprocess_response(res_timestamps, res_outputs)
         assert res_outputs[1]["response"] == expected_response
 
+    def test_no_special_tokens(self, mock_read_write: pytest.MonkeyPatch) -> None:
+        """Test special tokens are not included when counting input/output tokens."""
+        tokenizer = get_tokenizer(DEFAULT_TOKENIZER)
+        pd = LLMProfileDataParser(
+            filename=Path("openai_profile_export.json"),
+            tokenizer=tokenizer,
+        )
+
+        # There are 3 special tokens in the default tokenizer
+        #  - <unk>: 0  (unknown)
+        #  - <s>: 1  (beginning of sentence)
+        #  - </s>: 2  (end of sentence)
+        special_token_ids = list(tokenizer._tokenizer.added_tokens_encoder.values())
+
+        # Check if special tokens are present in request input
+        req_input = {"text_input": "This is test input."}
+        tokens = pd._tokenize_triton_request_input(req_input)
+        assert all([s not in tokens for s in special_token_ids])
+
+        pd._response_format = ResponseFormat.OPENAI_COMPLETIONS
+        req_input = {"payload": '{"prompt":"This is test input."}'}
+        tokens = pd._tokenize_openai_request_input(req_input)
+        assert all([s not in tokens for s in special_token_ids])
+
+        pd._response_format = ResponseFormat.OPENAI_CHAT_COMPLETIONS
+        req_input = {"payload": '{"messages":[{"content":"This is test input."}]}'}
+        tokens = pd._tokenize_openai_request_input(req_input)
+        assert all([s not in tokens for s in special_token_ids])
+
+        # Check if special tokens are present in the responses
+        res_outputs = ["This", "is", "test", "input."]
+        tokens = []
+        for t in pd._run_tokenizer(res_outputs):
+            tokens += t
+        assert all([s not in tokens for s in special_token_ids])
+
     def test_llm_metrics_get_base_name(self) -> None:
         """Test get_base_name method in LLMMetrics class."""
         # initialize with dummy values

src/c++/perf_analyzer/genai-perf/tests/test_tokenizer.py

@@ -41,3 +41,36 @@ def test_non_default_tokenizer(self):
     def test_bad_tokenizer(self):
         with pytest.raises(GenAIPerfException):
             get_tokenizer("bad_tokenizer")
+
+    def test_default_args(self):
+        tokenizer_model = DEFAULT_TOKENIZER
+        tokenizer = get_tokenizer(tokenizer_model)
+
+        # There are 3 special tokens in the default tokenizer
+        #  - <unk>: 0  (unknown)
+        #  - <s>: 1  (beginning of sentence)
+        #  - </s>: 2  (end of sentence)
+        special_tokens = list(tokenizer._tokenizer.added_tokens_encoder.keys())
+        special_token_ids = list(tokenizer._tokenizer.added_tokens_encoder.values())
+
+        # special tokens are disabled by default
+        text = "This is test."
+        tokens = tokenizer(text)["input_ids"]
+        assert all([s not in tokens for s in special_token_ids])
+
+        tokens = tokenizer.encode(text)
+        assert all([s not in tokens for s in special_token_ids])
+
+        output = tokenizer.decode(tokens)
+        assert all([s not in output for s in special_tokens])
+
+        # check special tokens is enabled
+        text = "This is test."
+        tokens = tokenizer(text, add_special_tokens=True)["input_ids"]
+        assert any([s in tokens for s in special_token_ids])
+
+        tokens = tokenizer.encode(text, add_special_tokens=True)
+        assert any([s in tokens for s in special_token_ids])
+
+        output = tokenizer.decode(tokens, skip_special_tokens=False)
+        assert any([s in output for s in special_tokens])