Update

Author: larryliu0820

CMakeLists.txt

@@ -507,7 +507,7 @@ install(
   INCLUDES
   DESTINATION ${_common_include_directories}
 )
-install(FILES tools/cmake/executorch-config.cmake
+install(FILES tools/cmake/executorch-config.cmake tools/cmake/Utils.cmake
         DESTINATION lib/cmake/ExecuTorch
 )
 

examples/models/llama/CMakeLists.txt

@@ -80,12 +80,11 @@ find_package(gflags REQUIRED)
 # find `executorch` libraries Same as for gflags
 set(executorch_DIR ${CMAKE_CURRENT_BINARY_DIR}/../../../lib/cmake/ExecuTorch)
 find_package(executorch CONFIG REQUIRED)
-target_link_options_shared_lib(executorch)
 
 # llama_runner library
 add_subdirectory(runner)
 
-set(link_libraries executorch gflags)
+set(link_libraries executorch_core gflags)
 set(_srcs main.cpp)
 
 if(TARGET optimized_native_cpu_ops_lib)

examples/models/llama/runner/CMakeLists.txt

@@ -60,7 +60,7 @@ if(NOT TARGET extension_llm_runner)
   )
 endif()
 
-set(llama_runner_deps executorch_core extension_data_loader extension_module
+set(llama_runner_deps executorch_core extension_data_loader extension_module_static
                       extension_tensor extension_flat_tensor extension_llm_runner
 )
 

examples/models/phi-3-mini/CMakeLists.txt

@@ -20,18 +20,12 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 set(CMAKE_BUILD_TYPE Release)
 
-# Set options for executorch build.
-option(EXECUTORCH_BUILD_EXECUTOR_RUNNER "" OFF)
-option(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER "" ON)
-option(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER "" ON)
-option(EXECUTORCH_BUILD_EXTENSION_MODULE "" ON)
-option(EXECUTORCH_BUILD_EXTENSION_TENSOR "" ON)
-option(EXECUTORCH_BUILD_KERNELS_OPTIMIZED "" ON)
-option(EXECUTORCH_BUILD_XNNPACK "" ON)
-
-add_subdirectory(
-  ${CMAKE_CURRENT_SOURCE_DIR}/../../.. ${CMAKE_BINARY_DIR}/../../..
-)
+set(EXECUTORCH_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../..")
+find_package(executorch CONFIG REQUIRED)
+
+set(BUILD_TESTING OFF)
+add_subdirectory(${EXECUTORCH_ROOT}/extension/llm/runner ${CMAKE_BINARY_DIR}/../../../extension/llm/runner)
+
 if(NOT TARGET gflags)
   add_subdirectory(
     ${CMAKE_CURRENT_SOURCE_DIR}/../../../third-party/gflags
@@ -43,11 +37,7 @@ add_executable(
   phi_3_mini_runner
   main.cpp
 )
-# target_include_directories(
-#   phi_3_mini_runner
-#   PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../../third-party/gflags/src
-#          ${CMAKE_CURRENT_SOURCE_DIR}/../../../extension/llm/tokenizers/include
-# )
+
 target_link_libraries(
-  phi_3_mini_runner PUBLIC optimized_native_cpu_ops_lib xnnpack_backend gflags extension_llm_runner
+  phi_3_mini_runner PUBLIC executorch optimized_native_cpu_ops_lib xnnpack_backend gflags extension_llm_runner
 )

examples/models/phi-3-mini/README.md

@@ -21,22 +21,17 @@ python -m examples.models.phi-3-mini.export_phi-3-mini -c "4k" -s 128 -o phi-3-m
 ```
 3. Build and run the model.
 - Build executorch with optimized CPU performance as follows. Build options available [here](https://github.com/pytorch/executorch/blob/main/CMakeLists.txt#L59).
- ```
- cmake --preset llm -DCMAKE_INSTALL_PREFIX=cmake-out
+```
+cmake --preset llm -DCMAKE_INSTALL_PREFIX=cmake-out
 
- cmake --build cmake-out -j16 --target install --config Release
- ```
+cmake --build cmake-out -j16 --target install --config Release
+```
 - Build Phi-3-mini runner.
 ```
-cmake -DPYTHON_EXECUTABLE=python \
-    -DCMAKE_INSTALL_PREFIX=cmake-out \
-    -DCMAKE_BUILD_TYPE=Release \
-    -DEXECUTORCH_BUILD_KERNELS_CUSTOM=ON \
-    -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
-    -DEXECUTORCH_BUILD_XNNPACK=ON \
-    -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
-    -Bcmake-out/examples/models/phi-3-mini \
-    examples/models/phi-3-mini
+cmake -DCMAKE_INSTALL_PREFIX=cmake-out \
+      -DCMAKE_BUILD_TYPE=Release \
+      -Bcmake-out/examples/models/phi-3-mini \
+      examples/models/phi-3-mini
 
 cmake --build cmake-out/examples/models/phi-3-mini -j16 --config Release
 ```

extension/llm/runner/CMakeLists.txt

@@ -37,30 +37,33 @@ include(${EXECUTORCH_SRCS_FILE})
 # build llm runner library
 list(TRANSFORM _extension_llm_runner__srcs PREPEND "${EXECUTORCH_ROOT}/")
 
-target_include_directories(
-  extension_module INTERFACE ${_common_include_directories}
-)
+if(NOT _common_include_directories)
+  set(_common_include_directories
+    ${EXECUTORCH_ROOT}/..
+    ${EXECUTORCH_ROOT}/runtime/core/portable_type/c10
+  )
+endif()
 
 add_library(extension_llm_runner STATIC ${_extension_llm_runner__srcs})
 
 # add tokenizers
-ExternalProject_Add(
-  tokenizers_external_project
-  PREFIX ${CMAKE_CURRENT_BINARY_DIR}/../tokenizers
-  SOURCE_DIR ${EXECUTORCH_ROOT}/extension/llm/tokenizers
-  CMAKE_ARGS -DSUPPORT_REGEX_LOOKAHEAD=ON -DCMAKE_INSTALL_PREFIX=${CMAKE_CURRENT_BINARY_DIR}/../tokenizers
-  INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/../tokenizers
+set(SUPPORT_REGEX_LOOKAHEAD ON)
+# llama/runner/CMakeLists.txt builds a shared library libllama_runner.so that
+# transitively depends on tokenizers. Need to build tokenizers with -fPIC.
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+add_subdirectory(
+  ${EXECUTORCH_ROOT}/extension/llm/tokenizers
+  ${CMAKE_CURRENT_BINARY_DIR}/../../../extension/llm/tokenizers
 )
-add_dependencies(extension_llm_runner tokenizers_external_project)
-find_package(tokenizers CONFIG HINTS ${CMAKE_INSTALL_PREFIX} ${CMAKE_CURRENT_BINARY_DIR}/../tokenizers)
-set(runner_deps executorch extension_module extension_tensor ${TOKENIZERS_LIBRARIES})
 
-target_link_libraries(extension_llm_runner PUBLIC ${runner_deps})
+set(runner_deps executorch_core extension_module extension_tensor tokenizers)
 
+target_link_libraries(extension_llm_runner PUBLIC ${runner_deps})
+target_compile_definitions(extension_llm_runner PUBLIC C10_USING_CUSTOM_GENERATED_MACROS)
 target_include_directories(
-  extension_llm_runner PUBLIC ${_common_include_directories}
-                                 ${TOKENIZERS_INCLUDE_DIRS}
-                                 ${CMAKE_CURRENT_BINARY_DIR}/../tokenizers/include
+  extension_llm_runner
+  PUBLIC ${_common_include_directories}
+            ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include
 )
 
 if(BUILD_TESTING)

extension/llm/runner/text_llm_runner.cpp

@@ -318,8 +318,14 @@ std::unordered_map<std::string, int64_t> get_llm_metadata(
   }
   // Check if we are using cache positions instead of input pos.
   auto second_input_info = method_meta.input_tensor_meta(1).get();
-  // For input_pos, size is [1], for cache_positions, size is [1, max_seq_len]
-  metadata[llm::kUseCachePositions] = second_input_info.sizes().size() == 2;
+  // For input_pos, numel is 1, for cache_positions, numel is max_seq_len
+  auto sizes = second_input_info.sizes();
+  int64_t total_size = 1;
+  for (const auto& size : sizes) {
+    total_size *= size;
+  }
+  metadata[llm::kUseCachePositions] = total_size > 1;
+  return metadata;
 }
 
 std::unordered_set<uint64_t> get_eos_ids(
@@ -386,6 +392,7 @@ std::unique_ptr<TextLLMRunner> create_text_llm_runner(
   auto text_prefiller = std::make_unique<TextPrefiller>(
       text_decoder_runner.get(),
       metadata.at(kUseKVCache),
+      metadata.at(kUseCachePositions),
       metadata.at(kEnableDynamicShape),
       metadata.at(kMaxSeqLen));
 

tools/cmake/executorch-config.cmake

@@ -25,6 +25,8 @@
 
 cmake_minimum_required(VERSION 3.19)
 
+include(${CMAKE_CURRENT_LIST_DIR}/Utils.cmake)
+
 set(_root "${CMAKE_CURRENT_LIST_DIR}/../../..")
 set(required_lib_list executorch executorch_core portable_kernels)
 set(EXECUTORCH_LIBRARIES)
@@ -163,6 +165,7 @@ if(TARGET optimized_native_cpu_ops_lib)
     PROPERTIES INTERFACE_LINK_LIBRARIES
                "optimized_kernels;${_maybe_optimized_portable_kernels_lib}"
   )
+  target_link_options_shared_lib(optimized_native_cpu_ops_lib)
 endif()
 if(TARGET extension_threadpool)
   target_compile_definitions(extension_threadpool INTERFACE ET_USE_THREADPOOL)
@@ -171,3 +174,14 @@ if(TARGET extension_threadpool)
                                     "cpuinfo;pthreadpool"
   )
 endif()
+if(TARGET xnnpack_backend)
+  if(TARGET kleidiai)
+    set(_deps "XNNPACK;microkernels-prod;kleidiai")
+  else()
+    set(_deps "XNNPACK;microkernels-prod")
+  endif()
+  set_target_properties(
+    xnnpack_backend PROPERTIES INTERFACE_LINK_LIBRARIES "${_deps}"
+  )
+  target_link_options_shared_lib(xnnpack_backend)
+endif()