Incorporate tokenzers changes

larryliu0820 · larryliu0820 · commit 9cf4b6463436 · 2025-06-16T20:46:58.000-07:00
diff --git a/extension/llm/runner/text_llm_runner.cpp b/extension/llm/runner/text_llm_runner.cpp
@@ -14,7 +14,7 @@
 #include <executorch/extension/llm/runner/util.h>
 #include <pytorch/tokenizers/hf_tokenizer.h>
 #include <pytorch/tokenizers/llama2c_tokenizer.h>
-#include <pytorch/tokenizers/sentencepiece_tokenizer.h>
+#include <pytorch/tokenizers/sentencepiece.h>
 #include <pytorch/tokenizers/tiktoken.h>
 
 namespace executorch::extension::llm {
diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers
@@ -1 +1 @@
-Subproject commit fc32028858020c4fcafe37aaaeaf5d1b480336a2
+Subproject commit 4da2344c0403edcbfa2fd3b81157ab069706758b
