diff --git a/examples/models/llama/tokenizer/test/test_tiktoken.cpp b/examples/models/llama/tokenizer/test/test_tiktoken.cpp index b9309f99212..7c54dbe3013 100644 --- a/examples/models/llama/tokenizer/test/test_tiktoken.cpp +++ b/examples/models/llama/tokenizer/test/test_tiktoken.cpp @@ -14,6 +14,10 @@ #include +#ifdef EXECUTORCH_FB_BUCK +#include +#endif + using namespace ::testing; using ::example::Version; @@ -26,8 +30,13 @@ class MultimodalTiktokenV5ExtensionTest : public Test { void SetUp() override { executorch::runtime::runtime_init(); tokenizer_ = get_tiktoken_for_llama(Version::Multimodal); +#ifdef EXECUTORCH_FB_BUCK + modelPath_ = + facebook::xplat::testing::getPathForTestResource("resources/test_tiktoken_tokenizer.model"); +#else modelPath_ = std::getenv("RESOURCES_PATH") + std::string("/test_tiktoken_tokenizer.model"); +#endif } std::unique_ptr tokenizer_; diff --git a/extension/llm/tokenizer/test/test_bpe_tokenizer.cpp b/extension/llm/tokenizer/test/test_bpe_tokenizer.cpp index c553fe59f98..547bbe06972 100644 --- a/extension/llm/tokenizer/test/test_bpe_tokenizer.cpp +++ b/extension/llm/tokenizer/test/test_bpe_tokenizer.cpp @@ -6,6 +6,9 @@ * LICENSE file in the root directory of this source tree. */ +#ifdef EXECUTORCH_FB_BUCK +#include +#endif #include #include #include @@ -23,8 +26,13 @@ class TokenizerExtensionTest : public Test { void SetUp() override { executorch::runtime::runtime_init(); tokenizer_ = std::make_unique(); +#ifdef EXECUTORCH_FB_BUCK + modelPath_ = + facebook::xplat::testing::getPathForTestResource("resources/test_bpe_tokenizer.model"); +#else modelPath_ = std::getenv("RESOURCES_PATH") + std::string("/test_bpe_tokenizer.bin"); +#endif } std::unique_ptr tokenizer_; diff --git a/extension/llm/tokenizer/test/test_tiktoken.cpp b/extension/llm/tokenizer/test/test_tiktoken.cpp index ce2a781aa1c..e02e62e2874 100644 --- a/extension/llm/tokenizer/test/test_tiktoken.cpp +++ b/extension/llm/tokenizer/test/test_tiktoken.cpp @@ -6,6 +6,9 @@ * LICENSE file in the root directory of this source tree. */ +#ifdef EXECUTORCH_FB_BUCK +#include +#endif #include #include #include @@ -55,8 +58,13 @@ class TiktokenExtensionTest : public Test { executorch::runtime::runtime_init(); tokenizer_ = std::make_unique( _get_special_tokens(), kBOSTokenIndex, kEOSTokenIndex); +#ifdef EXECUTORCH_FB_BUCK + modelPath_ = + facebook::xplat::testing::getPathForTestResource("resources/test_tiktoken_tokenizer.model"); +#else modelPath_ = std::getenv("RESOURCES_PATH") + std::string("/test_tiktoken_tokenizer.model"); +#endif } std::unique_ptr tokenizer_; @@ -144,44 +152,58 @@ TEST_F(TiktokenExtensionTest, ConstructionWithInvalidEOSIndex) { } TEST_F(TiktokenExtensionTest, LoadWithInvalidPath) { - auto invalidModelPath = - std::getenv("RESOURCES_PATH") + std::string("/nonexistent.model"); - - Error res = tokenizer_->load(invalidModelPath.c_str()); + auto invalidModelPath = "./nonexistent.model"; + Error res = tokenizer_->load(invalidModelPath); EXPECT_EQ(res, Error::InvalidArgument); } TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithInvalidRank) { +#ifdef EXECUTORCH_FB_BUCK + auto invalidModelPath = + facebook::xplat::testing::getPathForTestResource("resources/test_tiktoken_invalid_rank.model"); +#else auto invalidModelPath = std::getenv("RESOURCES_PATH") + std::string("/test_tiktoken_invalid_rank.model"); - +#endif Error res = tokenizer_->load(invalidModelPath.c_str()); EXPECT_EQ(res, Error::InvalidArgument); } TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithInvalidBase64) { +#ifdef EXECUTORCH_FB_BUCK + auto invalidModelPath = + facebook::xplat::testing::getPathForTestResource("resources/test_tiktoken_invalid_base64.model"); +#else auto invalidModelPath = std::getenv("RESOURCES_PATH") + std::string("/test_tiktoken_invalid_base64.model"); - +#endif Error res = tokenizer_->load(invalidModelPath.c_str()); EXPECT_EQ(res, Error::InvalidArgument); } TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithNoSpace) { +#ifdef EXECUTORCH_FB_BUCK + auto invalidModelPath = + facebook::xplat::testing::getPathForTestResource("resources/test_tiktoken_no_space.model"); +#else auto invalidModelPath = std::getenv("RESOURCES_PATH") + std::string("/test_tiktoken_no_space.model"); - +#endif Error res = tokenizer_->load(invalidModelPath.c_str()); EXPECT_EQ(res, Error::InvalidArgument); } TEST_F(TiktokenExtensionTest, LoadTiktokenFileWithBPEFile) { +#ifdef EXECUTORCH_FB_BUCK + auto invalidModelPath = + facebook::xplat::testing::getPathForTestResource("resources/test_bpe_tokenizer.bin"); +#else auto invalidModelPath = std::getenv("RESOURCES_PATH") + std::string("/test_bpe_tokenizer.bin"); - +#endif Error res = tokenizer_->load(invalidModelPath.c_str()); EXPECT_EQ(res, Error::InvalidArgument);