Update Tokenizer and Detokenizer for Chat Template Tokenization (#957)

* update tokenizer and detokenizer for chat template tokenization

* undo certain decoder changes

---------

Co-authored-by: Sayan Shaw <[email protected]>

Author: sayanshaw24

operators/tokenizer/bpe_streaming.hpp

@@ -192,6 +192,7 @@ class BpeStreamingDecoder : public KernelBpeDecoder {
       for (size_t tok_idx = 0; tok_idx < seq_len; ++tok_idx) {
         const auto id = ort_extensions::narrow<extTokenId_t>(*(p_ids + tok_idx));
         std::string decoded_token;
+
         auto status = spm_model_
                       ? SpmId2Token(id, decoded_token, f_special_last)
                       : Id2Token(id, decoded_token, true, f_special_last);

operators/tokenizer/bpe_tokenizer_model.hpp

@@ -360,7 +360,6 @@ class BpeModel {
                   pos--;
                 }
                 auto stripped_token = token.substr(0, pos);
-                final_result.back().first = stripped_token;
                 final_result.emplace_back(token, id);
                 continue;
               }

shared/api/tokenizer_impl.cc

@@ -120,6 +120,9 @@ OrtxStatus TokenizerImpl::BatchDecode(const std::vector<span<extTokenId_t const>
     std::transform(s.begin(), s.end(), ids.begin(), [](extTokenId_t v) { return static_cast<int64_t>(v); });
     ortc::Tensor<int64_t> ts_input(std::vector<int64_t>{1, static_cast<int64_t>(ids.size())}, (void*)ids.data());
     ortc::Tensor<std::string> ts_output;
+
+    // Note: currently the detokenizer Compute is called with skip_special_tokens = true
+    // by default, but this parameter should be exposed to GenAI in the future.
     OrtxStatus status = std::visit([&](auto& detokenizer) {
       return detokenizer->Compute(ts_input, ts_output); }, detokenizer_);