Add sentencepiece tokenizer support to llm runner (#85)

Summary:

Fixes pytorch/executorch#11618

Add sentencepiece tokenizer support

X-link: pytorch/executorch#11645

Reviewed By: guangy10

Differential Revision: D76789606

Pulled By: larryliu0820

Author: larryliu0820

CMakeLists.txt

@@ -92,19 +92,18 @@ if(SUPPORT_REGEX_LOOKAHEAD)
     pcre2-8-static
     PROPERTIES
       INTERFACE_INCLUDE_DIRECTORIES
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2/src>
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/third-party/pcre2>
   )
   add_library(
     regex_lookahead STATIC
     ${CMAKE_CURRENT_SOURCE_DIR}/src/pcre2_regex.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/regex_lookahead.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/src/std_regex.cpp
   )
-  target_link_libraries(regex_lookahead PUBLIC pcre2-8)
+  target_link_libraries(regex_lookahead PUBLIC pcre2-8-static)
   target_include_directories(
     regex_lookahead
     PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
-           $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third-party/pcre2/src>
   )
   target_link_options_shared_lib(regex_lookahead)
   target_link_libraries(tokenizers PUBLIC regex_lookahead)

targets.bzl

@@ -101,7 +101,7 @@ def define_common_targets():
             "@EXECUTORCH_CLIENTS",
             "//pytorch/tokenizers/...",
         ],
-        external_deps = [
+        exported_external_deps = [
             "sentencepiece",
             "abseil-cpp",
         ],