[llm] Add sentencepiece tokenizer support to llm runner

Add sentencepiece tokenizer support

Author: larryliu0820

extension/llm/runner/text_llm_runner.cpp

@@ -14,6 +14,7 @@
 #include <executorch/extension/llm/runner/util.h>
 #include <pytorch/tokenizers/hf_tokenizer.h>
 #include <pytorch/tokenizers/llama2c_tokenizer.h>
+#include <pytorch/tokenizers/sentencepiece_tokenizer.h>
 #include <pytorch/tokenizers/tiktoken.h>
 
 namespace executorch::extension::llm {
@@ -252,6 +253,12 @@ std::unique_ptr<tokenizers::Tokenizer> load_tokenizer(
     return tiktoken_tokenizer;
   }
 
+  auto sp_tokenizer = std::make_unique<::tokenizers::SPTokenizer>();
+  if (sp_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
+    ET_LOG(Info, "Loaded Sentencepiece tokenizer");
+    return sp_tokenizer;
+  }
+
   auto bpe_tokenizer = std::make_unique<::tokenizers::Llama2cTokenizer>();
   if (bpe_tokenizer->load(tokenizer_path) == ::tokenizers::Error::Ok) {
     ET_LOG(Info, "Loaded BPE tokenizer");