Skip to content

Use generated files in module test #10497

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Apr 30, 2025
Merged

Use generated files in module test #10497

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
d38e044
Use generated files in module test
lucylq Apr 25, 2025
903998a
Update on "Use generated files in module test"
lucylq Apr 26, 2025
fced9de
Update on "Use generated files in module test"
lucylq Apr 28, 2025
258e588
Update on "Use generated files in module test"
lucylq Apr 28, 2025
9be5ea8
Update on "Use generated files in module test"
lucylq Apr 28, 2025
18fe905
Update on "Use generated files in module test"
lucylq Apr 29, 2025
d848551
Update on "Use generated files in module test"
lucylq Apr 29, 2025
a1e663e
Update on "Use generated files in module test"
lucylq Apr 29, 2025
21f1dd1
Update on "Use generated files in module test"
lucylq Apr 30, 2025
7063a53
Update on "Use generated files in module test"
lucylq Apr 30, 2025
d14858e
Update on "Use generated files in module test"
lucylq Apr 30, 2025
f355971
Update on "Use generated files in module test"
lucylq Apr 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion extension/android/executorch_android/android_test_setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ which "${PYTHON_EXECUTABLE}"
BASEDIR=$(dirname "$(realpath $0)")

prepare_add() {
cp "${BASEDIR}/../../../extension/module/test/resources/add.pte" "${BASEDIR}/src/androidTest/resources"
python3 -m test.models.export_program --modules "ModuleAdd" --outdir "${BASEDIR}/src/androidTest/resources/"
}

prepare_tinyllama() {
Expand Down
2 changes: 1 addition & 1 deletion ...utorch_android/src/androidTest/java/org/pytorch/executorch/ModuleInstrumentationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
/** Unit tests for {@link Module}. */
@RunWith(AndroidJUnit4.class)
public class ModuleInstrumentationTest {
private static String TEST_FILE_NAME = "/add.pte";
private static String TEST_FILE_NAME = "/ModuleAdd.pte";
private static String MISSING_FILE_NAME = "/missing.pte";
private static String NON_PTE_FILE_NAME = "/test.txt";
private static String FORWARD_METHOD = "forward";
Expand Down
29 changes: 28 additions & 1 deletion extension/module/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,32 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Test.cmake)

set(_test_srcs module_test.cpp)

add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
COMMAND
python3 -m test.models.export_program --modules "ModuleAdd"
--outdir "${CMAKE_CURRENT_BINARY_DIR}" 2> /dev/null
COMMAND
python3 -m test.models.export_program --modules "ModuleLinear"
--external-constants --outdir "${CMAKE_CURRENT_BINARY_DIR}" 2> /dev/null
WORKING_DIRECTORY ${EXECUTORCH_ROOT}
)

add_custom_target(
generated_module_test_files
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
"${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
)

set(test_env
"ET_MODULE_ADD_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte"
"ET_MODULE_LINEAR_PROGRAM_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte"
"ET_MODULE_LINEAR_DATA_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd"
)

et_cxx_test(
extension_module_test
SOURCES
Expand All @@ -31,7 +57,8 @@ et_cxx_test(
portable_ops_lib
)

set(test_env "RESOURCES_PATH=${EXECUTORCH_ROOT}/extension/module/test/resources")
add_dependencies(extension_module_test generated_module_test_files)
set_property(TEST extension_module_test PROPERTY ENVIRONMENT ${test_env})

set_property(
TEST extension_module_test
Expand Down
2 changes: 1 addition & 1 deletion extension/module/test/TARGETS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ load(":targets.bzl", "define_common_targets")

oncall("executorch")

define_common_targets()
define_common_targets(is_fbcode = True)
148 changes: 72 additions & 76 deletions extension/module/test/module_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,17 @@

#include <executorch/extension/data_loader/file_data_loader.h>
#include <executorch/extension/tensor/tensor.h>
#include <executorch/runtime/core/exec_aten/testing_util/tensor_util.h>

using namespace ::executorch::extension;
using namespace ::executorch::runtime;

class ModuleTest : public ::testing::Test {
protected:
static void SetUpTestSuite() {
std::string resources_path;
if (const char* env = std::getenv("RESOURCES_PATH")) {
resources_path = env;
}
model_path_ = resources_path + "/add.pte";
linear_path_ = resources_path + "/linear.pte";
linear_data_path_ = resources_path + "/linear.ptd";
model_path_ = std::getenv("ET_MODULE_ADD_PATH");
linear_path_ = std::getenv("ET_MODULE_LINEAR_PROGRAM_PATH");
linear_data_path_ = std::getenv("ET_MODULE_LINEAR_DATA_PATH");
}

static inline std::string model_path_;
Expand Down Expand Up @@ -109,22 +106,29 @@ TEST_F(ModuleTest, TestMethodMeta) {
const auto meta = module.method_meta("forward");
EXPECT_EQ(meta.error(), Error::Ok);
EXPECT_STREQ(meta->name(), "forward");
EXPECT_EQ(meta->num_inputs(), 2);
// tensor, tensor, alpha
EXPECT_EQ(meta->num_inputs(), 3);
EXPECT_EQ(*(meta->input_tag(0)), Tag::Tensor);
EXPECT_EQ(meta->num_outputs(), 1);
EXPECT_EQ(*(meta->output_tag(0)), Tag::Tensor);

const auto input_meta = meta->input_tensor_meta(0);
EXPECT_EQ(input_meta.error(), Error::Ok);
EXPECT_EQ(input_meta->scalar_type(), executorch::aten::ScalarType::Float);
EXPECT_EQ(input_meta->sizes().size(), 1);
EXPECT_EQ(input_meta->sizes()[0], 1);
EXPECT_EQ(input_meta->sizes().size(), 2);
EXPECT_EQ(input_meta->sizes()[0], 2);

const auto input_meta1 = meta->input_tensor_meta(1);
EXPECT_EQ(input_meta1.error(), Error::Ok);
EXPECT_EQ(input_meta1->scalar_type(), executorch::aten::ScalarType::Float);
EXPECT_EQ(input_meta1->sizes().size(), 2);
EXPECT_EQ(input_meta1->sizes()[0], 2);

const auto output_meta = meta->output_tensor_meta(0);
EXPECT_EQ(output_meta.error(), Error::Ok);
EXPECT_EQ(output_meta->scalar_type(), executorch::aten::ScalarType::Float);
EXPECT_EQ(output_meta->sizes().size(), 1);
EXPECT_EQ(output_meta->sizes()[0], 1);
EXPECT_EQ(output_meta->sizes().size(), 2);
EXPECT_EQ(output_meta->sizes()[0], 2);
}

TEST_F(ModuleTest, TestNonExistentMethodMeta) {
Expand All @@ -136,17 +140,16 @@ TEST_F(ModuleTest, TestNonExistentMethodMeta) {

TEST_F(ModuleTest, TestExecute) {
Module module(model_path_);
auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

EXPECT_TRUE(module.is_loaded());
EXPECT_TRUE(module.is_method_loaded("forward"));

const auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestExecutePreload) {
Expand All @@ -155,14 +158,13 @@ TEST_F(ModuleTest, TestExecutePreload) {
const auto error = module.load();
EXPECT_EQ(error, Error::Ok);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestExecutePreload_method) {
Expand All @@ -171,14 +173,13 @@ TEST_F(ModuleTest, TestExecutePreload_method) {
const auto error = module.load_method("forward");
EXPECT_EQ(error, Error::Ok);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestExecutePreloadProgramAndMethod) {
Expand All @@ -190,14 +191,13 @@ TEST_F(ModuleTest, TestExecutePreloadProgramAndMethod) {
const auto load_method_error = module.load_method("forward");
EXPECT_EQ(load_method_error, Error::Ok);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestExecuteOnNonExistent) {
Expand All @@ -219,32 +219,30 @@ TEST_F(ModuleTest, TestExecuteOnCurrupted) {
TEST_F(ModuleTest, TestGet) {
Module module(model_path_);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.get("forward", {tensor, tensor});
const auto result = module.get("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);
const auto data = result->toTensor().const_data_ptr<float>();
EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestForward) {
auto module = std::make_unique<Module>(model_path_);
auto tensor = make_tensor_ptr({21.f});
auto tensor = make_tensor_ptr({2, 2}, {21.f, 22.f, 23.f, 24.f});

const auto result = module->forward({tensor, tensor});
const auto result = module->forward({tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();
const auto expected = make_tensor_ptr({2, 2}, {42.f, 44.f, 46.f, 48.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());

EXPECT_NEAR(data[0], 42, 1e-5);

auto tensor2 = make_tensor_ptr({2.f});
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});
const auto result2 = module->forward({tensor2, tensor2});
EXPECT_EQ(result2.error(), Error::Ok);

const auto data2 = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data2[0], 4, 1e-5);
const auto expected2 = make_tensor_ptr({2, 2}, {4.f, 6.f, 8.f, 10.f});
EXPECT_TENSOR_CLOSE(result2->at(0).toTensor(), *expected2.get());
}

TEST_F(ModuleTest, TestForwardWithInvalidInputs) {
Expand Down Expand Up @@ -294,20 +292,20 @@ TEST_F(ModuleTest, TestProgramSharingAndDataLoaderManagement) {
EXPECT_EQ(load_error, Error::Ok);
EXPECT_TRUE(module1->is_loaded());

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result1 = module1->execute("forward", {tensor, tensor});
const auto result1 = module1->execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result1.error(), Error::Ok);

auto module2 = std::make_unique<Module>(module1->program());

const auto result2 = module2->execute("forward", {tensor, tensor});
const auto result2 = module2->execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result2.error(), Error::Ok);

module1 = std::make_unique<Module>("/path/to/nonexistent/file.pte");
EXPECT_FALSE(module1->is_loaded());

const auto result3 = module2->execute("forward", {tensor, tensor});
const auto result3 = module2->execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result3.error(), Error::Ok);
}

Expand Down Expand Up @@ -339,14 +337,13 @@ TEST_F(ModuleTest, TestProgramPersistenceAndReuseAfterModuleDestruction) {

EXPECT_EQ(module.program(), shared_program);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

const auto result = module.execute("forward", {tensor, tensor});
const auto result = module.execute("forward", {tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

auto data = result->at(0).toTensor().const_data_ptr<float>();

EXPECT_NEAR(data[0], 2, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
Expand All @@ -364,22 +361,22 @@ TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
EXPECT_TRUE(program != nullptr);

auto thread = [](std::shared_ptr<Program> program,
const std::array<float, 1>& input) {
const std::array<float, 4>& input) {
Module module(program);
auto tensor = from_blob((void*)input.data(), {1});
auto tensor = from_blob((void*)input.data(), {2, 2});

const auto result = module.forward({tensor, tensor});
const auto result = module.forward({tensor, tensor, 1.0});
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();
EXPECT_NEAR(data[0], (input[0] * 2), 1e-5);
};

std::thread t1(thread, program, std::array<float, 1>{1});
std::thread t2(thread, program, std::array<float, 1>{2});
std::thread t3(thread, program, std::array<float, 1>{3});
std::thread t4(thread, program, std::array<float, 1>{4});
std::thread t5(thread, program, std::array<float, 1>{5});
std::thread t1(thread, program, std::array<float, 4>{1, 2, 3, 4});
std::thread t2(thread, program, std::array<float, 4>{2, 3, 4, 5});
std::thread t3(thread, program, std::array<float, 4>{3, 4, 5, 6});
std::thread t4(thread, program, std::array<float, 4>{4, 5, 6, 7});
std::thread t5(thread, program, std::array<float, 4>{5, 6, 7, 8});

t1.join();
t2.join();
Expand All @@ -391,37 +388,38 @@ TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) {
TEST_F(ModuleTest, TestSetInputsBeforeExecute) {
Module module(model_path_);

auto tensor1 = make_tensor_ptr({4.f});
auto tensor2 = make_tensor_ptr({5.f});
auto tensor1 = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});

EXPECT_EQ(module.set_inputs({tensor1, tensor2}), Error::Ok);
EXPECT_EQ(module.set_inputs({tensor1, tensor2, 1.0}), Error::Ok);

const auto result = module.forward();
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();
EXPECT_NEAR(data[0], 9, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {3.f, 5.f, 7.f, 9.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestSetInputCombinedWithExecute) {
Module module(model_path_);

auto tensor1 = make_tensor_ptr({2.f});
auto tensor2 = make_tensor_ptr({3.f});
auto tensor1 = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});
auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f});

EXPECT_EQ(module.set_input(tensor2, 1), Error::Ok);
EXPECT_EQ(module.set_input(1.0, 2), Error::Ok); // alpha

const auto result = module.forward(tensor1);
EXPECT_EQ(result.error(), Error::Ok);

const auto data = result->at(0).toTensor().const_data_ptr<float>();
EXPECT_NEAR(data[0], 5, 1e-5);
const auto expected = make_tensor_ptr({2, 2}, {3.f, 5.f, 7.f, 9.f});
EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get());
}

TEST_F(ModuleTest, TestPartiallySetInputs) {
Module module(model_path_);

auto tensor = make_tensor_ptr({1.f});
auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f});

EXPECT_EQ(module.set_input(tensor, 0), Error::Ok);

Expand Down Expand Up @@ -455,8 +453,6 @@ TEST_F(ModuleTest, TestPTD) {

ASSERT_EQ(module.load_method("forward"), Error::Ok);

auto tensor1 =
make_tensor_ptr({3, 3}, {2.f, 3.f, 4.f, 2.f, 3.f, 4.f, 2.f, 3.f, 4.f});

ASSERT_EQ(module.forward(tensor1).error(), Error::Ok);
auto tensor = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 2.f});
ASSERT_EQ(module.forward(tensor).error(), Error::Ok);
}
Loading
Loading