Revert "server : use text_to_send instead of detokenized token"

This reverts commit 0f66d99.

Author: danbev

examples/server/server.cpp

@@ -541,16 +541,12 @@ struct completion_token_output {
     json to_json(bool post_sampling_probs) const {
         json probs_for_token = json::array();
         for (const auto & p : probs) {
-            // If the predicted token id is the same as this.tok, then we use the text_to_send instead
-            // of the detokenized token. This is to avoid a mismatch between the text tokens where
-            // the text_to_send token may include a leading whitespace character but the detokenized
-            // token would not.
-            std::string txt = tok == p.tok ? text_to_send : p.txt;
+            std::string txt(p.txt);
             txt.resize(validate_utf8(txt));
             probs_for_token.push_back(json {
                 {"id",      p.tok},
                 {"token",   txt},
-                {"bytes",   str_to_bytes(txt)},
+                {"bytes",   str_to_bytes(p.txt)},
                 {
                     post_sampling_probs ? "prob" : "logprob",
                     post_sampling_probs ? p.prob : logarithm(p.prob)