Update <think> tag logic for DeepSeek chat template (#956)

* update <think> tag logic in deepseek chat template

* resolve comment

* remove hf token req for deepseek python test

---------

Co-authored-by: Sayan Shaw <[email protected]>

Author: sayanshaw24

shared/api/chat_template.cc

@@ -559,6 +559,14 @@ OrtxStatus TokenizerImpl::DeepSeekChatTemplate(std::string& output, bool add_gen
 
     // Handle assistant message without tool calls
     if (role == "assistant" && !content.empty()) {
+      constexpr std::string_view think_token = "</think>";
+      
+      // Remove content before and including "</think>" if present
+      std::size_t think_pos = content.find(think_token);
+      if (think_pos != std::string::npos) {
+        content = content.substr(think_pos + think_token.size());
+      }
+
       if (is_tool) {
         output += "<｜tool_outputs_end｜>" + content;
         output += eos_token;

test/test_pp_api.py

@@ -485,10 +485,9 @@ def test_llama3_3_chat_template(self):
             ortx_inputs = tokenizer.tokenize(prompt)
             np.testing.assert_array_equal(ortx_inputs, hf_tok(prompt, return_tensors="np")["input_ids"][0])
 
-    @unittest.skipIf(hf_token_id is None, "HF_TOKEN is not available")
     def test_deepseek_chat_template(self):
         ckpt = "deepseek-ai/DeepSeek-R1-Distill-Llama-70B"
-        hf_tok = AutoTokenizer.from_pretrained(ckpt, token=hf_token_id)
+        hf_tok = AutoTokenizer.from_pretrained(ckpt)
 
         for messages in self.messages_list:
             inputs = hf_tok.apply_chat_template(