feat: handle the deprecated default chat template

Author: baixiac

app/api/utils.py

@@ -352,7 +352,7 @@ async def generate_text(
             chat_template="{% for message in messages %}\n{% if message['role'] == 'user' %}\nUser: {{ message['content'] }}\n{% elif message['role'] == 'assistant' %}\nAssistant: {{ message['content'] }}\n{% endif %}\n{% endfor %}\nAssistant:",
             tokenize=True,
         )
-        prompt_obj = TokensPrompt(prompt_token_ids=prompt_tokens)
+        prompt_obj = TokensPrompt(prompt_token_ids=prompt_tokens)   # type: ignore
 
         async def _stream() -> AsyncGenerator[bytes, None]:
             start = 0

app/utils.py

@@ -756,6 +756,14 @@ def get_prompt_from_messages(tokenizer: PreTrainedTokenizer, messages: List[Prom
             tokenize=False,
             add_generation_prompt=True,
         )
+    elif hasattr(tokenizer, "default_chat_template") and tokenizer.default_chat_template:
+        # This largely depends on how older versions of HF tokenizers behave and may not work universally
+        tokenizer.chat_template = tokenizer.default_chat_template
+        prompt = tokenizer.apply_chat_template(
+            [dump_pydantic_object_to_dict(message) for message in messages],
+            tokenize=False,
+            add_generation_prompt=True,
+        )
     else:
         system_content = ""
         prompt_parts: List[str] = []
@@ -835,4 +843,3 @@ def get_prompt_from_messages(tokenizer: PreTrainedTokenizer, messages: List[Prom
     "25624495": '© 2002-2020 International Health Terminology Standards Development Organisation (IHTSDO). All rights reserved. SNOMED CT®, was originally created by The College of American Pathologists. "SNOMED" and "SNOMED CT" are registered trademarks of the IHTSDO.',
     "55540447": "linkage concept"
 }
-

tests/app/test_utils.py

@@ -412,10 +412,27 @@ def test_get_prompt_with_chat_template():
         assert prompt == "Mock chat template applied"
 
 
+def test_get_prompt_with_default_chat_template():
+    with patch('transformers.PreTrainedTokenizer') as tok:
+        mock_tokenizer = tok.return_value
+        mock_tokenizer.chat_template = None
+        mock_tokenizer.default_chat_template = "Mock default chat template"
+        mock_tokenizer.apply_chat_template.return_value = "Mock default chat template applied"
+        messages = [
+            PromptMessage(content="Alright?", role=PromptRole.USER.value),
+            PromptMessage(content="Yeah.", role=PromptRole.ASSISTANT.value),
+        ]
+
+        prompt = get_prompt_from_messages(mock_tokenizer, messages)
+
+        assert prompt == "Mock default chat template applied"
+
+
 def test_get_prompt_without_chat_template():
     with patch('transformers.PreTrainedTokenizer') as tok:
         mock_tokenizer = tok.return_value
         mock_tokenizer.chat_template = None
+        mock_tokenizer.default_chat_template = None
         messages = [
             PromptMessage(content="You are a helpful assistant.", role=PromptRole.SYSTEM.value),
             PromptMessage(content="Alright?", role=PromptRole.USER.value),
@@ -432,9 +449,9 @@ def test_get_prompt_with_no_messages():
     with patch('transformers.PreTrainedTokenizer') as tok:
         mock_tokenizer = tok.return_value
         mock_tokenizer.chat_template = None
+        mock_tokenizer.default_chat_template = None
         messages = []
 
         prompt = get_prompt_from_messages(mock_tokenizer, messages)
 
-        expected_prompt = "\n<|assistant|>\n"
-        assert prompt == expected_prompt
+        assert prompt == "\n<|assistant|>\n"