Use ExternalProject_Add to build tokenizers

This way it avoids the CMake target name collision between XNNPACK's
`memory` and Abseil's `memory`

Author: larryliu0820

extension/llm/runner/CMakeLists.txt

@@ -44,18 +44,23 @@ target_include_directories(
 add_library(extension_llm_runner STATIC ${_extension_llm_runner__srcs})
 
 # add tokenizers
-add_subdirectory(
-  ${EXECUTORCH_ROOT}/extension/llm/tokenizers
-  ${CMAKE_CURRENT_BINARY_DIR}/../../../extension/llm/tokenizers
+ExternalProject_Add(
+  tokenizers_external_project
+  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/../tokenizers
+  SOURCE_DIR ${EXECUTORCH_ROOT}/extension/llm/tokenizers
+  CMAKE_ARGS -DSUPPORT_REGEX_LOOKAHEAD=ON -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/../tokenizers
+  INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/../tokenizers
 )
-
-set(runner_deps executorch_core extension_module extension_tensor tokenizers)
+add_dependencies(extension_llm_runner tokenizers_external_project)
+find_package(tokenizers CONFIG HINTS ${CMAKE_INSTALL_PREFIX} ${CMAKE_CURRENT_BINARY_DIR}/../tokenizers)
+set(runner_deps executorch extension_module extension_tensor ${TOKENIZERS_LIBRARIES})
 
 target_link_libraries(extension_llm_runner PUBLIC ${runner_deps})
 
 target_include_directories(
-  extension_llm_runner INTERFACE ${_common_include_directories}
-                                 ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include
+  extension_llm_runner PUBLIC ${_common_include_directories}
+                                 ${TOKENIZERS_INCLUDE_DIRS}
+                                 ${CMAKE_CURRENT_BINARY_DIR}/../tokenizers/include
 )
 
 if(BUILD_TESTING)