Leverage vllm's tokenizer_info endpoint to avoid manual duplication (#3185)

* ✨ added an approach to use tokenizer_info endpoint from vllm

Signed-off-by: m-misiura <[email protected]>

* 🚧 removed all auto-detection and tokenization logic from `LocalChatCompletion`

* pacify pre-commit

---------

Signed-off-by: m-misiura <[email protected]>
Co-authored-by: Baber <[email protected]>

Author: m-misiura

lm_eval/models/api_models.py

@@ -114,7 +114,7 @@ def __init__(
         # however the requests can be sent as a string if the API doesn't support token inputs.
         # use tokenized_requests=False
         tokenizer_backend: Optional[
-            Literal["tiktoken", "huggingface", "None", "none"]
+            Literal["tiktoken", "huggingface", "remote", "None", "none"]
         ] = "huggingface",
         truncate: bool = False,
         # number of concurrent requests. More useful if not batching
@@ -132,6 +132,8 @@ def __init__(
         revision: Optional[str] = "main",
         use_fast_tokenizer: bool = True,
         verify_certificate: bool = True,
+        ca_cert_path: Optional[str] = None,
+        auth_token: Optional[str] = None,
         eos_string: str = None,
         # timeout in seconds
         timeout: int = 300,
@@ -182,6 +184,8 @@ def __init__(
         self.tokenized_requests = tokenized_requests
         self.max_retries = int(max_retries)
         self.verify_certificate = verify_certificate
+        self.ca_cert_path = ca_cert_path
+        self.auth_token = auth_token
         self._eos_string = eos_string
         self.timeout = int(timeout)
         self.max_images = int(max_images)
@@ -218,6 +222,21 @@ def __init__(
                             f"Passed `base_url={self.base_url}` but using (OpenAI) Tiktoken tokenizer backend. "
                             "Pass `tokenizer_backend=huggingface` and provide the HF tokenizer name if your model does not use Tiktoken."
                         )
+                elif self.tokenizer_backend == "remote":
+                    from lm_eval.utils import RemoteTokenizer
+
+                    if not self.base_url:
+                        raise ValueError(
+                            "base_url is required for remote tokenizer backend"
+                        )
+                    self.tokenizer = RemoteTokenizer(
+                        self.base_url,
+                        self.timeout,
+                        self.verify_certificate,
+                        self.ca_cert_path,
+                        self.auth_token,
+                    )
+                    eval_logger.info(f"Using remote tokenizer from {self.base_url}")
             else:
                 import transformers
 
@@ -310,7 +329,7 @@ def tokenizer_name(self) -> str:
 
     def apply_chat_template(
         self, chat_history: List[Dict[str, str]], add_generation_prompt: bool = True
-    ) -> Union[str, JsonChatStr]:
+    ) -> Union[str, JsonChatStr, List[Dict]]:
         """Applies a chat template to a list of chat history between user and model."""
         if self.tokenizer_backend == "huggingface" and self.tokenized_requests:
             return self.tokenizer.apply_chat_template(
@@ -319,6 +338,8 @@ def apply_chat_template(
                 add_generation_prompt=add_generation_prompt,
                 continue_final_message=not add_generation_prompt,
             )
+        elif self.tokenizer_backend == "remote" and self.tokenized_requests:
+            return chat_history
         else:
             # bit of a hack. We'll load back before sending to the API
             return JsonChatStr(
@@ -337,6 +358,8 @@ def eot_token_id(self) -> Optional[int]:
                 return self.tokenizer.eos_token_id
             elif self.tokenizer_backend == "tiktoken":
                 return self.tokenizer.eot_token
+            elif self.tokenizer_backend == "remote":
+                return self.tokenizer.eos_token_id
 
     @cached_property
     def eos_string(self) -> Optional[str]:
@@ -347,6 +370,8 @@ def eos_string(self) -> Optional[str]:
                 return self.tokenizer.eos_token
             elif self.tokenizer_backend == "tiktoken":
                 return self.tokenizer.decode([self.tokenizer.eot_token])
+            elif self.tokenizer_backend == "remote":
+                return self.tokenizer.eos_token
         else:
             eval_logger.warning(
                 "Cannot determine EOS string to pass to stop sequence. Manually set by passing `eos_string` to model_args."
@@ -364,6 +389,8 @@ def prefix_token_id(self) -> Optional[int]:
                 if self.tokenizer.bos_token_id is not None:
                     return self.tokenizer.bos_token_id
                 return self.tokenizer.eos_token_id
+            elif self.tokenizer_backend == "remote":
+                return self.tokenizer.bos_token_id or self.tokenizer.eos_token_id
             else:
                 return self.tokenizer.eot_token
 
@@ -396,7 +423,19 @@ def tok_encode(
                     encoding = encoding[-left_truncate_len:]
 
             return encoding
+        elif self.tokenizer_backend == "remote":
+            if isinstance(string, str):
+                encoding = self.tokenizer.encode(string)
+            else:
+                encoding = [self.tokenizer.encode(s) for s in string]
 
+            if left_truncate_len:
+                if isinstance(string, str):
+                    encoding = encoding[-left_truncate_len:]
+                else:
+                    encoding = [enc[-left_truncate_len:] for enc in encoding]
+
+            return encoding
         else:
             try:
                 encoding = self.tokenizer.encode(string)
@@ -409,6 +448,8 @@ def decode_batch(self, tokens: List[List[int]]) -> List[str]:
             return self.tokenizer.batch_decode(tokens)
         elif self.tokenizer_backend == "tiktoken":
             return self.tokenizer.decode_batch(tokens)
+        elif self.tokenizer_backend == "remote":
+            return self.tokenizer.batch_decode(tokens)
 
     def model_call(
         self,

lm_eval/models/openai_completions.py

@@ -16,12 +16,46 @@
 class LocalCompletionsAPI(TemplateAPI):
     def __init__(
         self,
-        base_url: str = None,
-        tokenizer_backend: str = "huggingface",
+        base_url=None,
+        tokenizer_backend="auto",
+        verify_certificate=True,
+        ca_cert_path=None,
+        auth_token=None,
         **kwargs,
     ):
+        # Auto-detect tokenizer backend
+        if tokenizer_backend == "auto":
+            if base_url:
+                from lm_eval.utils import check_remote_tokenizer_support
+
+                if check_remote_tokenizer_support(
+                    base_url,
+                    verify_certificate=verify_certificate,
+                    ca_cert_path=ca_cert_path,
+                    auth_token=auth_token,
+                ):
+                    eval_logger.info(
+                        "Auto-detected remote tokenizer support. Using remote tokenizer backend."
+                    )
+                    tokenizer_backend = "remote"
+                else:
+                    eval_logger.info(
+                        "Remote tokenizer not supported. Using huggingface tokenizer backend."
+                    )
+                    tokenizer_backend = "huggingface"
+            else:
+                eval_logger.warning(
+                    "No base_url provided. Using huggingface tokenizer backend."
+                )
+                tokenizer_backend = "huggingface"
+
         super().__init__(
-            base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs
+            base_url=base_url,
+            tokenizer_backend=tokenizer_backend,
+            verify_certificate=verify_certificate,
+            ca_cert_path=ca_cert_path,
+            auth_token=auth_token,
+            **kwargs,
         )
 
     def _create_payload(
@@ -106,20 +140,28 @@ def api_key(self):
 
 @register_model("local-chat-completions")
 class LocalChatCompletion(LocalCompletionsAPI):
+    """
+    Minimal chat-completions wrapper.
+    - Only accepts messages as list[dict].
+    - No tokenization or template logic.
+    - Use with --apply_chat_template or ensure upstream formats messages correctly.
+    """
+
     def __init__(
         self,
-        base_url: str = None,
-        tokenizer_backend: str = None,
-        tokenized_requests: bool = False,
+        base_url=None,
+        verify_certificate=True,
+        ca_cert_path=None,
+        auth_token=None,
         **kwargs,
     ):
-        eval_logger.warning(
-            "chat-completions endpoint requires the `--apply_chat_template` flag."
-        )
         super().__init__(
             base_url=base_url,
-            tokenizer_backend=tokenizer_backend,
-            tokenized_requests=tokenized_requests,
+            tokenizer_backend=None,
+            tokenized_requests=None,
+            verify_certificate=verify_certificate,
+            ca_cert_path=ca_cert_path,
+            auth_token=auth_token,
             **kwargs,
         )
         if self._batch_size > 1:
@@ -137,9 +179,13 @@ def _create_payload(
         eos=None,
         **kwargs,
     ) -> dict:
-        assert type(messages) is not str, (
-            "chat-completions require the --apply_chat_template flag."
+        assert isinstance(messages, list) and all(
+            isinstance(m, dict) for m in messages
+        ), (
+            "LocalChatCompletion expects messages as list[dict]. "
+            "If you see this error, ensure --apply_chat_template is set or upstream code formats messages correctly."
         )
+        gen_kwargs = gen_kwargs or {}
         gen_kwargs.pop("do_sample", False)
         if "max_tokens" in gen_kwargs:
             max_tokens = gen_kwargs.pop("max_tokens")