Target link tokenizers library

Author: jackzhxng

examples/models/llama/runner/CMakeLists.txt

@@ -41,14 +41,6 @@ target_include_directories(
   extension_module INTERFACE ${_common_include_directories}
 )
 
-list(
-  APPEND _llama_runner__srcs
-  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/src/tiktoken.cpp
-)
-list(APPEND _llama_runner__srcs
-     ${CMAKE_CURRENT_SOURCE_DIR}/../tokenizer/llama_tiktoken.cpp
-)
-
 if(CMAKE_TOOLCHAIN_IOS
    OR ANDROID
    OR APPLE
@@ -60,32 +52,8 @@ else()
   add_library(llama_runner SHARED ${_llama_runner__srcs})
 endif()
 
-# find RE2 for tokenizer, build tiktoken
-set(ABSL_ENABLE_INSTALL ON)
-set(ABSL_PROPAGATE_CXX_STD ON)
-set(_pic_flag ${CMAKE_POSITION_INDEPENDENT_CODE})
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-add_subdirectory(
-  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/abseil-cpp
-  ${CMAKE_CURRENT_BINARY_DIR}/abseil-cpp
-)
-add_subdirectory(
-  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/re2
-  ${CMAKE_CURRENT_BINARY_DIR}/re2
-)
-add_subdirectory(
-  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/json
-  ${CMAKE_CURRENT_BINARY_DIR}/json
-)
-target_include_directories(llama_runner
-  PRIVATE ${CMAKE_INSTALL_PREFIX}/include
-  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/include
-  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/src
-)
-set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag})
-
 set(llama_runner_deps executorch extension_data_loader extension_module
-                      extension_tensor re2::re2 nlohmann_json::nlohmann_json
+                      extension_tensor
 )
 
 target_link_libraries(llama_runner PUBLIC ${llama_runner_deps})
@@ -94,6 +62,17 @@ target_include_directories(
   llama_runner
   INTERFACE ${_common_include_directories}
 )
+
+# Include tokenizers dependency
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+add_subdirectory(
+  ${EXECUTORCH_ROOT}/extension/llm/tokenizers
+  ${CMAKE_CURRENT_BINARY_DIR}/tokenizers
+)
+target_link_libraries(
+  llama_runner PUBLIC tokenizers
+)
+
 target_include_directories(
   llama_runner
   PUBLIC ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include

extension/llm/runner/CMakeLists.txt

@@ -59,4 +59,4 @@ target_include_directories(
 target_include_directories(
   extension_llm_runner INTERFACE ${_common_include_directories}
                                  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include
-)
+			       )