tokenizer

Author: lucylq

CMakeLists.txt

@@ -186,6 +186,10 @@ option(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR "Build the Flat Tensor extension"
        OFF
 )
 
+option(EXECUTORCH_BUILD_EXTENSION_LLM "Build the LLM extension"
+       OFF
+)
+
 option(EXECUTORCH_BUILD_EXTENSION_MODULE "Build the Module extension" OFF)
 
 option(EXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL "Build the Runner Util extension"
@@ -717,6 +721,10 @@ if(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/flat_tensor/serialize)
 endif()
 
+if(EXECUTORCH_BUILD_EXTENSION_LLM)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizer)
+endif()
+
 if(EXECUTORCH_BUILD_EXTENSION_MODULE)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/module)
 endif()

build/cmake_deps.toml

@@ -402,6 +402,22 @@ deps = [
   "xnnpack_backend",
 ]
 
+[targets.extension_llm_tokenizer]
+buck_targets = [
+  "//extension/llm/tokenizer:bpe_tokenizer",
+  "//extension/llm/tokenizer:tiktoken",
+]
+filters = [
+  ".cpp$",
+]
+excludes = [
+  "^codegen",
+]
+deps = [
+  "executorch",
+  "executorch_core",
+]
+
 [targets.llama_runner]
 buck_targets = [
   "//examples/models/llama/runner:runner",

extension/llm/tokenizer/CMakeLists.txt

@@ -0,0 +1,58 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Please this file formatted by running:
+# ~~~
+# cmake-format -i CMakeLists.txt
+# ~~~
+
+cmake_minimum_required(VERSION 3.19)
+
+# Source root directory for executorch.
+if(NOT EXECUTORCH_ROOT)
+  set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../..)
+endif()
+
+set(ABSL_ENABLE_INSTALL ON)
+set(ABSL_PROPAGATE_CXX_STD ON)
+set(_pic_flag ${CMAKE_POSITION_INDEPENDENT_CODE})
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+add_subdirectory(
+  ${CMAKE_CURRENT_SOURCE_DIR}/../third-party/abseil-cpp
+  ${CMAKE_CURRENT_BINARY_DIR}/abseil-cpp
+)
+add_subdirectory(
+  ${CMAKE_CURRENT_SOURCE_DIR}/../third-party/re2
+  ${CMAKE_CURRENT_BINARY_DIR}/re2
+)
+set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag})
+
+list(TRANSFORM _extension_llm_tokenizer__srcs PREPEND "${EXECUTORCH_ROOT}/")
+add_library(extension_llm_tokenizer ${_extension_llm_tokenizer__srcs})
+target_include_directories(extension_llm_tokenizer PUBLIC
+  ${EXECUTORCH_ROOT}/..
+  ${_common_include_directories})
+
+target_link_libraries(extension_llm_tokenizer re2::re2)
+target_compile_options(extension_llm_tokenizer PUBLIC ${_common_compile_options})
+
+# Install libraries
+install(
+  TARGETS extension_llm_tokenizer
+  DESTINATION lib
+  INCLUDES
+  DESTINATION ${_common_include_directories}
+)
+
+target_include_directories(
+  extension_llm_tokenizer
+  PRIVATE ${CMAKE_INSTALL_PREFIX}/include
+          ${CMAKE_CURRENT_SOURCE_DIR}/../third-party/abseil-cpp
+)
+
+if(BUILD_TESTING)
+  add_subdirectory(test)
+endif()

extension/llm/tokenizer/test/CMakeLists.txt

@@ -4,6 +4,8 @@
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree.
 
+# @generated by test/utils/generate_gtest_cmakelists.py
+#
 # This file should be formatted with
 # ~~~
 # cmake-format -i CMakeLists.txt
@@ -12,39 +14,42 @@
 #
 
 cmake_minimum_required(VERSION 3.19)
-project(tokenizer_test)
-
-# Use C++17 for test.
-set(CMAKE_CXX_STANDARD 17)
 
 set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/../../../..)
 
 include(${EXECUTORCH_ROOT}/build/Test.cmake)
 
-set(_tokenizer_test_srcs
-    test_tiktoken.cpp test_bpe_tokenizer.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/../tiktoken.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/../bpe_tokenizer.cpp
-)
-
-set(ENV{RESOURCES_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/resources)
-set(ABSL_ENABLE_INSTALL ON)
-set(ABSL_PROPAGATE_CXX_STD ON)
-set(_pic_flag ${CMAKE_POSITION_INDEPENDENT_CODE})
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-add_subdirectory(
-  ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/abseil-cpp
-  ${CMAKE_CURRENT_BINARY_DIR}/abseil-cpp
-)
-add_subdirectory(
-  ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/re2
-  ${CMAKE_CURRENT_BINARY_DIR}/re2
+set(test_env
+    "TEST_BPE_TOKENIZER=${EXECUTORCH_ROOT}/extension/llm/tokenizer/test/resources/test_bpe_tokenizer.bin"
+    "TEST_TIKTOKEN_INVALID_BASE64=${EXECUTORCH_ROOT}/extension/llm/tokenizer/test/resources/test_tiktoken_invalid_base64.model"
+    "TEST_TIKTOKEN_INVALID_RANK=${EXECUTORCH_ROOT}/extension/llm/tokenizer/test/resources/test_tiktoken_invalid_rank.model"
+    "TEST_TIKTOKEN_NO_SPACE=${EXECUTORCH_ROOT}/extension/llm/tokenizer/test/resources/test_tiktoken_no_space.model"
+    "TEST_TIKTOKEN_TOKENIZER=${EXECUTORCH_ROOT}/extension/llm/tokenizer/test/resources/test_tiktoken_tokenizer.model"
 )
-set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag})
 
-et_cxx_test(tokenizer_test SOURCES ${_tokenizer_test_srcs} EXTRA_LIBS re2::re2)
-target_include_directories(
-  tokenizer_test
-  PRIVATE ${CMAKE_INSTALL_PREFIX}/include
-          ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/abseil-cpp
-)
+# set(ABSL_ENABLE_INSTALL ON)
+# set(ABSL_PROPAGATE_CXX_STD ON)
+# set(_pic_flag ${CMAKE_POSITION_INDEPENDENT_CODE})
+# set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+# add_subdirectory(
+#   ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/abseil-cpp
+#   ${CMAKE_CURRENT_BINARY_DIR}/abseil-cpp
+# )
+# add_subdirectory(
+#   ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/re2
+#   ${CMAKE_CURRENT_BINARY_DIR}/re2
+# )
+# set(CMAKE_POSITION_INDEPENDENT_CODE ${_pic_flag})
+
+set(_test_srcs test_bpe_tokenizer.cpp test_tiktoken.cpp)
+
+et_cxx_test(extension_llm_tokenizer_test SOURCES ${_test_srcs} EXTRA_LIBS extension_llm_tokenizer)
+
+set_property(TEST extension_llm_tokenizer_test PROPERTY ENVIRONMENT ${test_env})
+
+# target_include_directories(extension_llm_tokenizer PUBLIC)
+# target_include_directories(
+#   extension_llm_tokenizer_test
+#   PRIVATE ${CMAKE_INSTALL_PREFIX}/include
+#           ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/abseil-cpp
+# )

extension/llm/tokenizer/test/targets.bzl

@@ -7,6 +7,14 @@ def define_common_targets():
     TARGETS and BUCK files that call this function.
     """
 
+    test_env = {
+        "TEST_BPE_TOKENIZER": "$(location //executorch/extension/llm/tokenizer/test/resources/test_bpe_tokenizer.bin"
+        "TEST_TIKTOKEN_INVALID_BASE64": "$(location //executorch/extension/llm/tokenizer/test/resources/test_tiktoken_invalid_base64.model"
+        "TEST_TIKTOKEN_INVALID_RANK": "$(location //executorch/extension/llm/tokenizer/test/resources/test_tiktoken_invalid_rank.model"
+        "TEST_TIKTOKEN_NO_SPACE": "$(location //executorch/extension/llm/tokenizer/test/resources/test_tiktoken_no_space.model"
+        "TEST_TIKTOKEN_TOKENIZER": "$(location //executorch/extension/llm/tokenizer/test/resources/test_tiktoken_tokenizer.model"
+    },
+
     runtime.python_test(
         name = "test_tokenizer_py",
         srcs = [
@@ -25,9 +33,7 @@ def define_common_targets():
         deps = [
             "//executorch/extension/llm/tokenizer:bpe_tokenizer",
         ],
-        env = {
-            "RESOURCES_PATH": "$(location :resources)/resources",
-        },
+        env = test_env,
     )
 
     runtime.cxx_test(
@@ -45,10 +51,3 @@ def define_common_targets():
             "re2",
         ],
     )
-
-    runtime.filegroup(
-        name = "resources",
-        srcs = native.glob([
-            "resources/**",
-        ]),
-    )

extension/llm/tokenizer/test/test_bpe_tokenizer.cpp

@@ -6,9 +6,9 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-#ifdef EXECUTORCH_FB_BUCK
-#include <TestResourceUtils/TestResourceUtils.h>
-#endif
+// #ifdef EXECUTORCH_FB_BUCK
+// #include <TestResourceUtils/TestResourceUtils.h>
+// #endif
 #include <executorch/extension/llm/tokenizer/bpe_tokenizer.h>
 #include <executorch/runtime/platform/runtime.h>
 #include <gtest/gtest.h>
@@ -26,13 +26,15 @@ class TokenizerExtensionTest : public Test {
   void SetUp() override {
     executorch::runtime::runtime_init();
     tokenizer_ = std::make_unique<BPETokenizer>();
-#ifdef EXECUTORCH_FB_BUCK
-    modelPath_ = facebook::xplat::testing::getPathForTestResource(
-        "resources/test_bpe_tokenizer.bin");
-#else
-    modelPath_ =
-        std::getenv("RESOURCES_PATH") + std::string("/test_bpe_tokenizer.bin");
-#endif
+    modelPath_ = std::getenv("TEST_BPE_TOKENIZER");
+    // #ifdef EXECUTORCH_FB_BUCK
+    //     modelPath_ = facebook::xplat::testing::getPathForTestResource(
+    //         "resources/test_bpe_tokenizer.bin");
+    // #else
+    //     modelPath_ =
+    //         std::getenv("RESOURCES_PATH") +
+    //         std::string("/test_bpe_tokenizer.bin");
+    // #endif
   }
 
   std::unique_ptr<Tokenizer> tokenizer_;

extension/llm/tokenizer/test/test_tiktoken.cpp

@@ -6,15 +6,17 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-#ifdef EXECUTORCH_FB_BUCK
-#include <TestResourceUtils/TestResourceUtils.h>
-#endif
+// #ifdef EXECUTORCH_FB_BUCK
+// #include <TestResourceUtils/TestResourceUtils.h>
+// #endif
 #include <executorch/extension/llm/tokenizer/tiktoken.h>
 #include <executorch/runtime/platform/runtime.h>
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 #include <vector>
 
+#include <gtest/gtest.h>
+
 using namespace ::testing;
 using ::executorch::extension::llm::Tiktoken;
 using ::executorch::extension::llm::Tokenizer;
@@ -50,13 +52,15 @@ static inline std::unique_ptr<std::vector<std::string>> _get_special_tokens() {
   return special_tokens;
 }
 
-static inline std::string _get_resource_path(const std::string& name) {
-#ifdef EXECUTORCH_FB_BUCK
-  return facebook::xplat::testing::getPathForTestResource("resources/" + name);
-#else
-  return std::getenv("RESOURCES_PATH") + std::string("/") + name;
-#endif
-}
+// static inline std::string _get_resource_path(const std::string& name) {
+//   return std::get_env(name);
+// #ifdef EXECUTORCH_FB_BUCK
+//   return facebook::xplat::testing::getPathForTestResource("resources/" +
+//   name);
+// #else
+//   return std::getenv("RESOURCES_PATH") + std::string("/") + name;
+// #endif
+// }
 
 } // namespace
 
@@ -66,7 +70,7 @@ class TiktokenExtensionTest : public Test {
     executorch::runtime::runtime_init();
     tokenizer_ = std::make_unique<Tiktoken>(
         _get_special_tokens(), kBOSTokenIndex, kEOSTokenIndex);
-    modelPath_ = _get_resource_path("test_tiktoken_tokenizer.model");
+    modelPath_ = std::get_env("TEST_TIKTOKEN_TOKENIZER");
   }
 
   std::unique_ptr<Tokenizer> tokenizer_;
@@ -160,30 +164,28 @@ TEST_F(TiktokenExtensionTest, LoadWithInvalidPath) {
 }
 
 TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithInvalidRank) {
-  auto invalidModelPath =
-      _get_resource_path("test_tiktoken_invalid_rank.model");
+  auto invalidModelPath = std::get_env("TEST_TIKTOKEN_INVALID_RANK");
   Error res = tokenizer_->load(invalidModelPath.c_str());
 
   EXPECT_EQ(res, Error::InvalidArgument);
 }
 
 TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithInvalidBase64) {
-  auto invalidModelPath =
-      _get_resource_path("test_tiktoken_invalid_base64.model");
+  auto invalidModelPath = std::get_env("TEST_TIKTOKEN_INVALID_BASE64");
   Error res = tokenizer_->load(invalidModelPath.c_str());
 
   EXPECT_EQ(res, Error::InvalidArgument);
 }
 
 TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithNoSpace) {
-  auto invalidModelPath = _get_resource_path("test_tiktoken_no_space.model");
-  Error res = tokenizer_->load(invalidModelPath.c_str());
+  auto invalidModelPath = std::get_env("TEST_TIKTOKEN_NO_SPACE") Error res =
+      tokenizer_->load(invalidModelPath.c_str());
 
   EXPECT_EQ(res, Error::InvalidArgument);
 }
 
 TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithBPEFile) {
-  auto invalidModelPath = _get_resource_path("test_bpe_tokenizer.bin");
+  auto invalidModelPath = std::get_env("TEST_BPE_TOKENIZER");
   Error res = tokenizer_->load(invalidModelPath.c_str());
 
   EXPECT_EQ(res, Error::InvalidArgument);

runtime/executor/test/CMakeLists.txt

@@ -32,7 +32,7 @@ add_custom_command(
     "ModuleAdd,ModuleAddHalf,ModuleDynamicCatUnallocatedIO,ModuleIndex,ModuleLinear,ModuleMultipleEntry,ModuleSimpleTrain"
     --outdir "${CMAKE_BINARY_DIR}" 2> /dev/null
   COMMAND
-    python3 -m test.models.export_program --modules
+    python -m test.models.export_program --modules
     "ModuleLinear" --external-constants
     --outdir "${CMAKE_BINARY_DIR}" 2> /dev/null
   COMMAND

test/run_oss_cpp_tests.sh

@@ -35,18 +35,17 @@ build_executorch() {
   CMAKE_PREFIX_PATH="$(python3 -c 'import torch as _; print(_.__path__[0])')"
   cmake . \
     -DCMAKE_INSTALL_PREFIX=cmake-out \
-    -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" \
+    -DCMAKE_PREFIX_PATH=/home/lfq/.conda/envs/executorch/lib/python3.10/site-packages/torch\
     -DEXECUTORCH_USE_CPP_CODE_COVERAGE=ON \
     -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
     -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
     -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
     -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
+    -DEXECUTORCH_BUILD_EXTENSION_LLM=ON \
     -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
     -DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
     -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
     -DEXECUTORCH_BUILD_DEVTOOLS=ON \
-    -DEXECUTORCH_BUILD_VULKAN=$BUILD_VULKAN \
-    -DEXECUTORCH_BUILD_XNNPACK=ON \
     -DEXECUTORCH_BUILD_TESTS=ON \
     -Bcmake-out
   cmake --build cmake-out -j9 --target install

test/utils/OSSTestConfig.json

@@ -31,6 +31,13 @@
             "make_boxed_from_unboxed_functor_test.cpp"
         ]
     },
+    {
+        "directory": "extension/llm/tokenizer/test",
+        "sources": [
+            "test_bpe_tokenizer.cpp",
+            "test_tiktoken.cpp"
+        ]
+    },
     {
         "directory": "extension/memory_allocator/test",
         "sources": [