Add sentencepiece tokenizer support to llm runner (#11645)

Summary:
X-link: meta-pytorch/tokenizers#85

Fixes #11618

Add sentencepiece tokenizer support

Pull Request resolved: #11645

Reviewed By: guangy10

Differential Revision: D76789606

Pulled By: larryliu0820

Author: larryliu0820

extension/llm/runner/targets.bzl

@@ -103,7 +103,7 @@ def define_common_targets():
                 ":text_token_generator" + aten_suffix,
                 "//pytorch/tokenizers:hf_tokenizer",
                 "//pytorch/tokenizers:llama2c_tokenizer",
-                # "//pytorch/tokenizers:sentencepiece", # TODO(larryliu0820) Make sure this compiles in xplat.
+                "//pytorch/tokenizers:sentencepiece",
                 "//pytorch/tokenizers:tiktoken",
             ],
         )

extension/llm/runner/text_llm_runner.cpp

@@ -14,6 +14,7 @@
 #include <executorch/extension/llm/runner/util.h>
 #include <pytorch/tokenizers/hf_tokenizer.h>
 #include <pytorch/tokenizers/llama2c_tokenizer.h>
+#include <pytorch/tokenizers/sentencepiece.h>
 #include <pytorch/tokenizers/tiktoken.h>
 
 namespace executorch::extension::llm {
@@ -278,6 +279,12 @@ std::unique_ptr<tokenizers::Tokenizer> load_tokenizer(
     return tiktoken_tokenizer;
   }
 
+  auto sp_tokenizer = std::make_unique<::tokenizers::SPTokenizer>();
+  if (sp_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
+    ET_LOG(Info, "Loaded Sentencepiece tokenizer");
+    return sp_tokenizer;
+  }
+
   auto bpe_tokenizer = std::make_unique<::tokenizers::Llama2cTokenizer>();
   if (bpe_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
     ET_LOG(Info, "Loaded BPE tokenizer");