Skip to content

Use generated files in module test #10497

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Apr 30, 2025
Merged

Use generated files in module test #10497

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
d38e044
Use generated files in module test
lucylq Apr 25, 2025
903998a
Update on "Use generated files in module test"
lucylq Apr 26, 2025
fced9de
Update on "Use generated files in module test"
lucylq Apr 28, 2025
258e588
Update on "Use generated files in module test"
lucylq Apr 28, 2025
9be5ea8
Update on "Use generated files in module test"
lucylq Apr 28, 2025
18fe905
Update on "Use generated files in module test"
lucylq Apr 29, 2025
d848551
Update on "Use generated files in module test"
lucylq Apr 29, 2025
a1e663e
Update on "Use generated files in module test"
lucylq Apr 29, 2025
21f1dd1
Update on "Use generated files in module test"
lucylq Apr 30, 2025
7063a53
Update on "Use generated files in module test"
lucylq Apr 30, 2025
d14858e
Update on "Use generated files in module test"
lucylq Apr 30, 2025
f355971
Update on "Use generated files in module test"
lucylq Apr 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion extension/module/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,31 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Test.cmake)

set(_test_srcs module_test.cpp)

add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
COMMAND
python3 -m test.models.export_program --modules "ModuleAdd"
--outdir "${CMAKE_CURRENT_BINARY_DIR}" 2> /dev/null
COMMAND
python3 -m test.models.export_program --modules "ModuleLinear"
--external-constants --outdir "${CMAKE_CURRENT_BINARY_DIR}" 2> /dev/null
)

add_custom_target(
generated_module_test_files
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
)

set(test_env
"ET_MODULE_ADD_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
"ET_MODULE_LINEAR_PROGRAM_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
"ET_MODULE_LINEAR_DATA_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
)

et_cxx_test(
extension_module_test
SOURCES
Expand All @@ -31,7 +56,8 @@ et_cxx_test(
portable_ops_lib
)

set(test_env "RESOURCES_PATH=${EXECUTORCH_ROOT}/extension/module/test/resources")
add_dependencies(extension_module_test generated_module_test_files)
set_property(TEST extension_module_test PROPERTY ENVIRONMENT ${test_env})

set_property(
TEST extension_module_test
Expand Down
2 changes: 1 addition & 1 deletion extension/module/test/TARGETS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ load(":targets.bzl", "define_common_targets")

oncall("executorch")

define_common_targets()
define_common_targets(is_fbcode = True)
144 changes: 72 additions & 72 deletions extension/module/test/module_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

#include <executorch/extension/data_loader/file_data_loader.h>
#include <executorch/extension/tensor/tensor.h>
#include <executorch/runtime/core/exec_aten/testing_util/tensor_util.h>

using namespace ::executorch::extension;
using namespace ::executorch::runtime;
Expand All @@ -26,9 +27,9 @@ class ModuleTest : public ::testing::Test {
if (const char* env = std::getenv("RESOURCES_PATH")) {
resources_path = env;
}
model_path_ = resources_path + "/add.pte";
linear_path_ = resources_path + "/linear.pte";
linear_data_path_ = resources_path + "/linear.ptd";
model_path_ = std::getenv("ET_MODULE_ADD_PATH");
linear_path_ = std::getenv("ET_MODULE_LINEAR_PROGRAM_PATH");
linear_data_path_ = std::getenv("ET_MODULE_LINEAR_DATA_PATH");
}

static inline std::string model_path_;
Expand Down Expand Up @@ -109,22 +110,29 @@ TEST_F(ModuleTest, TestMethodMeta) {
const auto meta = module.method_meta("forward");
EXPECT_EQ(meta.error(), Error::Ok);
EXPECT_STREQ(meta->name(), "forward");
EXPECT_EQ(meta->num_inputs(), 2);
// tensor, tensor, alpha
EXPECT_EQ(meta->num_inputs(), 3);
EXPECT_EQ(*(meta->input_tag(0)), Tag::Tensor);
EXPECT_EQ(meta->num_outputs(), 1);
EXPECT_EQ(*(meta->output_tag(0)), Tag::Tensor);

const auto input_meta = meta->input_tensor_meta(0);
EXPECT_EQ(input_meta.error(), Error::Ok);
EXPECT_EQ(input_meta->scalar_type(), executorch::aten::ScalarType::Float);
EXPECT_EQ(input_meta->sizes().size(), 1);
EXPECT_EQ(input_meta->sizes()[0], 1);
EXPECT_EQ(input_meta->sizes().size(), 2);
EXPECT_EQ(input_meta->sizes()[0], 2);

const auto input_meta1 = meta->input_tensor_meta(1);
EXPECT_EQ(input_meta1.error(), Error::Ok);
EXPECT_EQ(input_meta1->scalar_type(), executorch::aten::ScalarType::Float);
EXPECT_EQ(input_meta1->sizes().size(), 2);
EXPECT_EQ(input_meta1->sizes()[0], 2);

const auto output_meta = meta->output_tensor_meta(0);
EXPECT_EQ(output_meta.error(), Error::Ok);
EXPECT_EQ(output_meta->scalar_type(), executorch::aten::ScalarType::Float);
EXPECT_EQ(output_meta->sizes().size(), 1);
EXPECT_EQ(output_meta->sizes()[0], 1);
EXPECT_EQ(output_meta->sizes().size(), 2);
EXPECT_EQ(output_meta->sizes()[0], 2);
}

TEST_F(ModuleTest, TestNonExistentMethodMeta) {
Expand All @@ -136,17 +144,16 @@ TEST_F(ModuleTest, TestNonExistentMethodMeta) {

TEST_F(ModuleTest, TestExecute) {
Module module(model_path_);
auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

EXPECT_TRUE(module.is_loaded());
EXPECT_TRUE(module.is_method_loaded("forward"));

const auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestExecutePreload) {
Expand All @@ -155,14 +162,13 @@ TEST_F(ModuleTest, TestExecutePreload) {
const auto error = module.load();
EXPECT_EQ(error, Error::Ok);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestExecutePreload_method) {
Expand All @@ -171,14 +177,13 @@ TEST_F(ModuleTest, TestExecutePreload_method) {
const auto error = module.load_method("forward");
EXPECT_EQ(error, Error::Ok);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestExecutePreloadProgramAndMethod) {
Expand All @@ -190,14 +195,13 @@ TEST_F(ModuleTest, TestExecutePreloadProgramAndMethod) {
const auto load_method_error = module.load_method("forward");
EXPECT_EQ(load_method_error, Error::Ok);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestExecuteOnNonExistent) {
Expand All @@ -219,32 +223,30 @@ TEST_F(ModuleTest, TestExecuteOnCurrupted) {
TEST_F(ModuleTest, TestGet) {
Module module(model_path_);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.get("forward", {tensor, tensor});
const auto result = module.get("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);
const auto data = result->toTensor().const_data_ptr<float>();
EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestForward) {
auto module = std::make_unique<Module>(model_path_);
auto tensor = make_tensor_ptr({21.f});
auto tensor = make_tensor_ptr({2, 2}, {21.f, 22.f, 23.f, 24.f});

const auto result = module->forward({tensor, tensor});
const auto result = module->forward({tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();
const auto expected = make_tensor_ptr({2, 2}, {42.f, 44.f, 46.f, 48.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());

EXPECT_NEAR(data[0], 42, 1e-5);

auto tensor2 = make_tensor_ptr({2.f});
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});
const auto result2 = module->forward({tensor2, tensor2});
EXPECT_EQ(result2.error(), Error::Ok);

const auto data2 = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data2[0], 4, 1e-5);
const auto expected2 = make_tensor_ptr({2, 2}, {4.f, 6.f, 8.f, 10.f});
EXPECT_TENSOR_CLOSE(result2->at(0).toTensor(), *expected2.get());
}

TEST_F(ModuleTest, TestForwardWithInvalidInputs) {
Expand Down Expand Up @@ -294,20 +296,20 @@ TEST_F(ModuleTest, TestProgramSharingAndDataLoaderManagement) {
EXPECT_EQ(load_error, Error::Ok);
EXPECT_TRUE(module1->is_loaded());

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result1 = module1->execute("forward", {tensor, tensor});
const auto result1 = module1->execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result1.error(), Error::Ok);

auto module2 = std::make_unique<Module>(module1->program());

const auto result2 = module2->execute("forward", {tensor, tensor});
const auto result2 = module2->execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result2.error(), Error::Ok);

module1 = std::make_unique<Module>("/path/to/nonexistent/file.pte");
EXPECT_FALSE(module1->is_loaded());

const auto result3 = module2->execute("forward", {tensor, tensor});
const auto result3 = module2->execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result3.error(), Error::Ok);
}

Expand Down Expand Up @@ -339,14 +341,13 @@ TEST_F(ModuleTest, TestProgramPersistenceAndReuseAfterModuleDestruction) {

EXPECT_EQ(module.program(), shared_program);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
Expand All @@ -364,22 +365,22 @@ TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
EXPECT_TRUE(program != nullptr);

auto thread = [](std::shared_ptr<Program> program,
const std::array<float, 1>& input) {
const std::array<float, 4>& input) {
Module module(program);
auto tensor = from_blob((void*)input.data(), {1});
auto tensor = from_blob((void*)input.data(), {2, 2});

const auto result = module.forward({tensor, tensor});
const auto result = module.forward({tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();
EXPECT_NEAR(data[0], (input[0] * 2), 1e-5);
};

std::thread t1(thread, program, std::array<float, 1>{1});
std::thread t2(thread, program, std::array<float, 1>{2});
std::thread t3(thread, program, std::array<float, 1>{3});
std::thread t4(thread, program, std::array<float, 1>{4});
std::thread t5(thread, program, std::array<float, 1>{5});
std::thread t1(thread, program, std::array<float, 4>{1, 2, 3, 4});
std::thread t2(thread, program, std::array<float, 4>{2, 3, 4, 5});
std::thread t3(thread, program, std::array<float, 4>{3, 4, 5, 6});
std::thread t4(thread, program, std::array<float, 4>{4, 5, 6, 7});
std::thread t5(thread, program, std::array<float, 4>{5, 6, 7, 8});

t1.join();
t2.join();
Expand All @@ -391,37 +392,38 @@ TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
TEST_F(ModuleTest, TestSetInputsBeforeExecute) {
Module module(model_path_);

auto tensor1 = make_tensor_ptr({4.f});
auto tensor2 = make_tensor_ptr({5.f});
auto tensor1 = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});

EXPECT_EQ(module.set_inputs({tensor1, tensor2}), Error::Ok);
EXPECT_EQ(module.set_inputs({tensor1, tensor2, 1.0}), Error::Ok);

const auto result = module.forward();
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();
EXPECT_NEAR(data[0], 9, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {3.f, 5.f, 7.f, 9.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestSetInputCombinedWithExecute) {
Module module(model_path_);

auto tensor1 = make_tensor_ptr({2.f});
auto tensor2 = make_tensor_ptr({3.f});
auto tensor1 = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});

EXPECT_EQ(module.set_input(tensor2, 1), Error::Ok);
EXPECT_EQ(module.set_input(1.0, 2), Error::Ok); // alpha

const auto result = module.forward(tensor1);
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();
EXPECT_NEAR(data[0], 5, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {3.f, 5.f, 7.f, 9.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestPartiallySetInputs) {
Module module(model_path_);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

EXPECT_EQ(module.set_input(tensor, 0), Error::Ok);

Expand Down Expand Up @@ -455,8 +457,6 @@ TEST_F(ModuleTest, TestPTD) {

ASSERT_EQ(module.load_method("forward"), Error::Ok);

auto tensor1 =
make_tensor_ptr({3, 3}, {2.f, 3.f, 4.f, 2.f, 3.f, 4.f, 2.f, 3.f, 4.f});

ASSERT_EQ(module.forward(tensor1).error(), Error::Ok);
auto tensor = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 2.f});
ASSERT_EQ(module.forward(tensor).error(), Error::Ok);
}
23 changes: 0 additions & 23 deletions extension/module/test/resources/README.md
View file
Open in desktop

This file was deleted.

Binary file removed extension/module/test/resources/add.pte
View file
Open in desktop
Binary file not shown.
Binary file removed extension/module/test/resources/linear.ptd
View file
Open in desktop
Binary file not shown.
Binary file removed extension/module/test/resources/linear.pte
View file
Open in desktop
Binary file not shown.
Loading
Loading