Update on "[llm] Add a generic text only LLM runner"

Introducing `text_llm_runner`. This can be used to run all text only decoder only LLM models supported by ExecuTorch.

* Metadata is being read out from the .pte file and being used to construct the runner object.
* examples/models/llama/runner.h[.cpp] only contains a simple wrapper around `text_llm_runner.h[.cpp]`.


In next PRs I will move examples/models/phi-3-mini/runner to use the generic runner.

Will look into QNN and MediaTek runners as well.

Differential Revision: [D75910889](https://our.internmc.facebook.com/intern/diff/D75910889/)

[ghstack-poisoned]

Author: larryliu0820

.ci/scripts/test_llama.sh

@@ -169,6 +169,7 @@ cmake_build_llama_runner() {
     popd
     dir="examples/models/llama"
     retry cmake \
+        -DBUILD_TESTING=OFF \
         -DCMAKE_INSTALL_PREFIX=cmake-out \
         -DCMAKE_BUILD_TYPE="$CMAKE_BUILD_TYPE" \
         -Bcmake-out/${dir} \

.ci/scripts/test_llava.sh

@@ -64,9 +64,10 @@ cmake_install_executorch_libraries_for_android() {
 
 
 LLAVA_COMMON_CMAKE_ARGS="                        \
+        -DBUILD_TESTING=OFF                      \
         -DPYTHON_EXECUTABLE="$PYTHON_EXECUTABLE" \
         -DCMAKE_INSTALL_PREFIX=${BUILD_DIR}      \
-        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}         \
+        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}   \
         -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON     \
         -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON  \
         -DEXECUTORCH_BUILD_XNNPACK=ON"

examples/models/llama/CMakeLists.txt

@@ -220,7 +220,6 @@ endif()
 target_include_directories(
   llama_main
   PUBLIC ${_common_include_directories}
-         ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include
 )
 target_link_libraries(llama_main PUBLIC llama_runner ${link_libraries})
 target_compile_options(llama_main PUBLIC ${_common_compile_options})

examples/models/llama/runner/CMakeLists.txt

@@ -52,23 +52,18 @@ else()
   add_library(llama_runner SHARED ${_llama_runner__srcs})
 endif()
 
+# For extension_llm_runner
+add_subdirectory(
+  ${EXECUTORCH_ROOT}/extension/llm/runner
+  ${CMAKE_CURRENT_BINARY_DIR}/../../../../extension/llm/runner
+)
+
 set(llama_runner_deps executorch_core extension_data_loader extension_module
-                      extension_tensor extension_flat_tensor
+                      extension_tensor extension_flat_tensor extension_llm_runner
 )
 
 target_link_libraries(llama_runner PUBLIC ${llama_runner_deps})
 
-target_include_directories(
-  llama_runner
-  INTERFACE ${_common_include_directories}
-)
-
-# Include tokenizers dependency
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-add_subdirectory(
-  ${EXECUTORCH_ROOT}/extension/llm/tokenizers
-  ${CMAKE_CURRENT_BINARY_DIR}/tokenizers
-)
 target_link_libraries(
   llama_runner PUBLIC tokenizers
 )

examples/models/llava/runner/CMakeLists.txt

@@ -28,8 +28,6 @@ set(_common_include_directories ${EXECUTORCH_ROOT}/..)
 # build llava_runner library
 set(_llava_runner__srcs
     "${CMAKE_CURRENT_SOURCE_DIR}/llava_runner.cpp"
-    "${EXECUTORCH_ROOT}/extension/llm/sampler/sampler.cpp"
-    "${EXECUTORCH_ROOT}/extension/llm/tokenizers/src/llama2c_tokenizer.cpp"
 )
 
 # extension llm runner lib
@@ -45,8 +43,3 @@ set(llava_runner_deps executorch_core extension_data_loader extension_llm_runner
 )
 
 target_link_libraries(llava_runner PUBLIC ${llava_runner_deps})
-
-target_include_directories(
-  llava_runner INTERFACE ${_common_include_directories}
-                         ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include
-)

extension/llm/runner/CMakeLists.txt

@@ -43,16 +43,15 @@ target_include_directories(
 
 add_library(extension_llm_runner STATIC ${_extension_llm_runner__srcs})
 
-set(runner_deps executorch_core extension_module extension_tensor)
+# add tokenizers
+add_subdirectory(
+  ${EXECUTORCH_ROOT}/extension/llm/tokenizers
+  ${CMAKE_CURRENT_BINARY_DIR}/../../../extension/llm/tokenizers
+)
 
-target_link_libraries(extension_llm_runner PUBLIC ${runner_deps})
+set(runner_deps executorch_core extension_module extension_tensor tokenizers)
 
-target_include_directories(
-  extension_llm_runner
-  PUBLIC
-    ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/include
-    ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/llama.cpp-unicode/src
-)
+target_link_libraries(extension_llm_runner PUBLIC ${runner_deps})
 
 target_include_directories(
   extension_llm_runner INTERFACE ${_common_include_directories}

tools/cmake/cmake_deps.toml

@@ -241,6 +241,17 @@ deps = [
   "executorch_core",
 ]
 
+[targets.extension_tokenizers]
+buck_targets = [
+  "//extension/llm/tokenizers:sentencepiece",
+  "//extension/llm/tokenizers:tiktoken",
+  "//extension/llm/tokenizers:hf_tokenizer",
+  "//extension/llm/tokenizers:llama2c_tokenizer",
+]
+filters = [
+  ".cpp$",
+]
+
 [targets.extension_llm_runner]
 buck_targets = [
   "//extension/llm/runner:runner_lib",
@@ -257,6 +268,7 @@ deps = [
   "extension_flat_tensor",
   "extension_runner_util",
   "extension_tensor",
+  "extension_tokenizers",
 ]
 
 [targets.extension_tensor]