diff --git a/extension/android/executorch_android/android_test_setup.sh b/extension/android/executorch_android/android_test_setup.sh index c1fb2a19386..682a1d16787 100644 --- a/extension/android/executorch_android/android_test_setup.sh +++ b/extension/android/executorch_android/android_test_setup.sh @@ -15,7 +15,7 @@ which "${PYTHON_EXECUTABLE}" BASEDIR=$(dirname "$(realpath $0)") prepare_add() { - cp "${BASEDIR}/../../../extension/module/test/resources/add.pte" "${BASEDIR}/src/androidTest/resources" + python3 -m test.models.export_program --modules "ModuleAdd" --outdir "${BASEDIR}/src/androidTest/resources/" } prepare_tinyllama() { diff --git a/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/ModuleInstrumentationTest.java b/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/ModuleInstrumentationTest.java index f71351ae6ae..be6efdd67be 100644 --- a/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/ModuleInstrumentationTest.java +++ b/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/ModuleInstrumentationTest.java @@ -38,7 +38,7 @@ /** Unit tests for {@link Module}. */ @RunWith(AndroidJUnit4.class) public class ModuleInstrumentationTest { - private static String TEST_FILE_NAME = "/add.pte"; + private static String TEST_FILE_NAME = "/ModuleAdd.pte"; private static String MISSING_FILE_NAME = "/missing.pte"; private static String NON_PTE_FILE_NAME = "/test.txt"; private static String FORWARD_METHOD = "forward"; diff --git a/extension/module/test/CMakeLists.txt b/extension/module/test/CMakeLists.txt index 0192b63e632..6ed42f63034 100644 --- a/extension/module/test/CMakeLists.txt +++ b/extension/module/test/CMakeLists.txt @@ -19,6 +19,32 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Test.cmake) set(_test_srcs module_test.cpp) +add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte" + "${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte" + "${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd" + COMMAND + python3 -m test.models.export_program --modules "ModuleAdd" + --outdir "${CMAKE_CURRENT_BINARY_DIR}" 2> /dev/null + COMMAND + python3 -m test.models.export_program --modules "ModuleLinear" + --external-constants --outdir "${CMAKE_CURRENT_BINARY_DIR}" 2> /dev/null + WORKING_DIRECTORY ${EXECUTORCH_ROOT} +) + +add_custom_target( + generated_module_test_files + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte" + "${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte" + "${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd" +) + +set(test_env + "ET_MODULE_ADD_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleAdd.pte" + "ET_MODULE_LINEAR_PROGRAM_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.pte" + "ET_MODULE_LINEAR_DATA_PATH=${CMAKE_CURRENT_BINARY_DIR}/ModuleLinearProgram.ptd" +) + et_cxx_test( extension_module_test SOURCES @@ -31,7 +57,8 @@ et_cxx_test( portable_ops_lib ) -set(test_env "RESOURCES_PATH=${EXECUTORCH_ROOT}/extension/module/test/resources") +add_dependencies(extension_module_test generated_module_test_files) +set_property(TEST extension_module_test PROPERTY ENVIRONMENT ${test_env}) set_property( TEST extension_module_test diff --git a/extension/module/test/TARGETS b/extension/module/test/TARGETS index 2341af9282f..a6c52d105f6 100644 --- a/extension/module/test/TARGETS +++ b/extension/module/test/TARGETS @@ -5,4 +5,4 @@ load(":targets.bzl", "define_common_targets") oncall("executorch") -define_common_targets() +define_common_targets(is_fbcode = True) diff --git a/extension/module/test/module_test.cpp b/extension/module/test/module_test.cpp index a82e257a703..38d739767a9 100644 --- a/extension/module/test/module_test.cpp +++ b/extension/module/test/module_test.cpp @@ -15,6 +15,7 @@ #include #include +#include using namespace ::executorch::extension; using namespace ::executorch::runtime; @@ -22,13 +23,9 @@ using namespace ::executorch::runtime; class ModuleTest : public ::testing::Test { protected: static void SetUpTestSuite() { - std::string resources_path; - if (const char* env = std::getenv("RESOURCES_PATH")) { - resources_path = env; - } - model_path_ = resources_path + "/add.pte"; - linear_path_ = resources_path + "/linear.pte"; - linear_data_path_ = resources_path + "/linear.ptd"; + model_path_ = std::getenv("ET_MODULE_ADD_PATH"); + linear_path_ = std::getenv("ET_MODULE_LINEAR_PROGRAM_PATH"); + linear_data_path_ = std::getenv("ET_MODULE_LINEAR_DATA_PATH"); } static inline std::string model_path_; @@ -109,7 +106,8 @@ TEST_F(ModuleTest, TestMethodMeta) { const auto meta = module.method_meta("forward"); EXPECT_EQ(meta.error(), Error::Ok); EXPECT_STREQ(meta->name(), "forward"); - EXPECT_EQ(meta->num_inputs(), 2); + // tensor, tensor, alpha + EXPECT_EQ(meta->num_inputs(), 3); EXPECT_EQ(*(meta->input_tag(0)), Tag::Tensor); EXPECT_EQ(meta->num_outputs(), 1); EXPECT_EQ(*(meta->output_tag(0)), Tag::Tensor); @@ -117,14 +115,20 @@ TEST_F(ModuleTest, TestMethodMeta) { const auto input_meta = meta->input_tensor_meta(0); EXPECT_EQ(input_meta.error(), Error::Ok); EXPECT_EQ(input_meta->scalar_type(), executorch::aten::ScalarType::Float); - EXPECT_EQ(input_meta->sizes().size(), 1); - EXPECT_EQ(input_meta->sizes()[0], 1); + EXPECT_EQ(input_meta->sizes().size(), 2); + EXPECT_EQ(input_meta->sizes()[0], 2); + + const auto input_meta1 = meta->input_tensor_meta(1); + EXPECT_EQ(input_meta1.error(), Error::Ok); + EXPECT_EQ(input_meta1->scalar_type(), executorch::aten::ScalarType::Float); + EXPECT_EQ(input_meta1->sizes().size(), 2); + EXPECT_EQ(input_meta1->sizes()[0], 2); const auto output_meta = meta->output_tensor_meta(0); EXPECT_EQ(output_meta.error(), Error::Ok); EXPECT_EQ(output_meta->scalar_type(), executorch::aten::ScalarType::Float); - EXPECT_EQ(output_meta->sizes().size(), 1); - EXPECT_EQ(output_meta->sizes()[0], 1); + EXPECT_EQ(output_meta->sizes().size(), 2); + EXPECT_EQ(output_meta->sizes()[0], 2); } TEST_F(ModuleTest, TestNonExistentMethodMeta) { @@ -136,17 +140,16 @@ TEST_F(ModuleTest, TestNonExistentMethodMeta) { TEST_F(ModuleTest, TestExecute) { Module module(model_path_); - auto tensor = make_tensor_ptr({1.f}); + auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); - const auto result = module.execute("forward", {tensor, tensor}); + const auto result = module.execute("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result.error(), Error::Ok); EXPECT_TRUE(module.is_loaded()); EXPECT_TRUE(module.is_method_loaded("forward")); - const auto data = result->at(0).toTensor().const_data_ptr(); - - EXPECT_NEAR(data[0], 2, 1e-5); + const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f}); + EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); } TEST_F(ModuleTest, TestExecutePreload) { @@ -155,14 +158,13 @@ TEST_F(ModuleTest, TestExecutePreload) { const auto error = module.load(); EXPECT_EQ(error, Error::Ok); - auto tensor = make_tensor_ptr({1.f}); + auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); - const auto result = module.execute("forward", {tensor, tensor}); + const auto result = module.execute("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result.error(), Error::Ok); - const auto data = result->at(0).toTensor().const_data_ptr(); - - EXPECT_NEAR(data[0], 2, 1e-5); + const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f}); + EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); } TEST_F(ModuleTest, TestExecutePreload_method) { @@ -171,14 +173,13 @@ TEST_F(ModuleTest, TestExecutePreload_method) { const auto error = module.load_method("forward"); EXPECT_EQ(error, Error::Ok); - auto tensor = make_tensor_ptr({1.f}); + auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); - const auto result = module.execute("forward", {tensor, tensor}); + const auto result = module.execute("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result.error(), Error::Ok); - const auto data = result->at(0).toTensor().const_data_ptr(); - - EXPECT_NEAR(data[0], 2, 1e-5); + const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f}); + EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); } TEST_F(ModuleTest, TestExecutePreloadProgramAndMethod) { @@ -190,14 +191,13 @@ TEST_F(ModuleTest, TestExecutePreloadProgramAndMethod) { const auto load_method_error = module.load_method("forward"); EXPECT_EQ(load_method_error, Error::Ok); - auto tensor = make_tensor_ptr({1.f}); + auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); - const auto result = module.execute("forward", {tensor, tensor}); + const auto result = module.execute("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result.error(), Error::Ok); - const auto data = result->at(0).toTensor().const_data_ptr(); - - EXPECT_NEAR(data[0], 2, 1e-5); + const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f}); + EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); } TEST_F(ModuleTest, TestExecuteOnNonExistent) { @@ -219,32 +219,30 @@ TEST_F(ModuleTest, TestExecuteOnCurrupted) { TEST_F(ModuleTest, TestGet) { Module module(model_path_); - auto tensor = make_tensor_ptr({1.f}); + auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); - const auto result = module.get("forward", {tensor, tensor}); + const auto result = module.get("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result.error(), Error::Ok); - const auto data = result->toTensor().const_data_ptr(); - EXPECT_NEAR(data[0], 2, 1e-5); + const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f}); + EXPECT_TENSOR_CLOSE(result->toTensor(), *expected.get()); } TEST_F(ModuleTest, TestForward) { auto module = std::make_unique(model_path_); - auto tensor = make_tensor_ptr({21.f}); + auto tensor = make_tensor_ptr({2, 2}, {21.f, 22.f, 23.f, 24.f}); - const auto result = module->forward({tensor, tensor}); + const auto result = module->forward({tensor, tensor, 1.0}); EXPECT_EQ(result.error(), Error::Ok); - const auto data = result->at(0).toTensor().const_data_ptr(); + const auto expected = make_tensor_ptr({2, 2}, {42.f, 44.f, 46.f, 48.f}); + EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); - EXPECT_NEAR(data[0], 42, 1e-5); - - auto tensor2 = make_tensor_ptr({2.f}); + auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f}); const auto result2 = module->forward({tensor2, tensor2}); EXPECT_EQ(result2.error(), Error::Ok); - const auto data2 = result->at(0).toTensor().const_data_ptr(); - - EXPECT_NEAR(data2[0], 4, 1e-5); + const auto expected2 = make_tensor_ptr({2, 2}, {4.f, 6.f, 8.f, 10.f}); + EXPECT_TENSOR_CLOSE(result2->at(0).toTensor(), *expected2.get()); } TEST_F(ModuleTest, TestForwardWithInvalidInputs) { @@ -294,20 +292,20 @@ TEST_F(ModuleTest, TestProgramSharingAndDataLoaderManagement) { EXPECT_EQ(load_error, Error::Ok); EXPECT_TRUE(module1->is_loaded()); - auto tensor = make_tensor_ptr({1.f}); + auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); - const auto result1 = module1->execute("forward", {tensor, tensor}); + const auto result1 = module1->execute("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result1.error(), Error::Ok); auto module2 = std::make_unique(module1->program()); - const auto result2 = module2->execute("forward", {tensor, tensor}); + const auto result2 = module2->execute("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result2.error(), Error::Ok); module1 = std::make_unique("/path/to/nonexistent/file.pte"); EXPECT_FALSE(module1->is_loaded()); - const auto result3 = module2->execute("forward", {tensor, tensor}); + const auto result3 = module2->execute("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result3.error(), Error::Ok); } @@ -339,14 +337,13 @@ TEST_F(ModuleTest, TestProgramPersistenceAndReuseAfterModuleDestruction) { EXPECT_EQ(module.program(), shared_program); - auto tensor = make_tensor_ptr({1.f}); + auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); - const auto result = module.execute("forward", {tensor, tensor}); + const auto result = module.execute("forward", {tensor, tensor, 1.0}); EXPECT_EQ(result.error(), Error::Ok); - auto data = result->at(0).toTensor().const_data_ptr(); - - EXPECT_NEAR(data[0], 2, 1e-5); + const auto expected = make_tensor_ptr({2, 2}, {2.f, 4.f, 6.f, 8.f}); + EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); } TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) { @@ -364,22 +361,22 @@ TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) { EXPECT_TRUE(program != nullptr); auto thread = [](std::shared_ptr program, - const std::array& input) { + const std::array& input) { Module module(program); - auto tensor = from_blob((void*)input.data(), {1}); + auto tensor = from_blob((void*)input.data(), {2, 2}); - const auto result = module.forward({tensor, tensor}); + const auto result = module.forward({tensor, tensor, 1.0}); EXPECT_EQ(result.error(), Error::Ok); const auto data = result->at(0).toTensor().const_data_ptr(); EXPECT_NEAR(data[0], (input[0] * 2), 1e-5); }; - std::thread t1(thread, program, std::array{1}); - std::thread t2(thread, program, std::array{2}); - std::thread t3(thread, program, std::array{3}); - std::thread t4(thread, program, std::array{4}); - std::thread t5(thread, program, std::array{5}); + std::thread t1(thread, program, std::array{1, 2, 3, 4}); + std::thread t2(thread, program, std::array{2, 3, 4, 5}); + std::thread t3(thread, program, std::array{3, 4, 5, 6}); + std::thread t4(thread, program, std::array{4, 5, 6, 7}); + std::thread t5(thread, program, std::array{5, 6, 7, 8}); t1.join(); t2.join(); @@ -391,37 +388,38 @@ TEST_F(ModuleTest, TestConcurrentExecutionWithSharedProgram) { TEST_F(ModuleTest, TestSetInputsBeforeExecute) { Module module(model_path_); - auto tensor1 = make_tensor_ptr({4.f}); - auto tensor2 = make_tensor_ptr({5.f}); + auto tensor1 = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); + auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f}); - EXPECT_EQ(module.set_inputs({tensor1, tensor2}), Error::Ok); + EXPECT_EQ(module.set_inputs({tensor1, tensor2, 1.0}), Error::Ok); const auto result = module.forward(); EXPECT_EQ(result.error(), Error::Ok); - const auto data = result->at(0).toTensor().const_data_ptr(); - EXPECT_NEAR(data[0], 9, 1e-5); + const auto expected = make_tensor_ptr({2, 2}, {3.f, 5.f, 7.f, 9.f}); + EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); } TEST_F(ModuleTest, TestSetInputCombinedWithExecute) { Module module(model_path_); - auto tensor1 = make_tensor_ptr({2.f}); - auto tensor2 = make_tensor_ptr({3.f}); + auto tensor1 = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); + auto tensor2 = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 5.f}); EXPECT_EQ(module.set_input(tensor2, 1), Error::Ok); + EXPECT_EQ(module.set_input(1.0, 2), Error::Ok); // alpha const auto result = module.forward(tensor1); EXPECT_EQ(result.error(), Error::Ok); - const auto data = result->at(0).toTensor().const_data_ptr(); - EXPECT_NEAR(data[0], 5, 1e-5); + const auto expected = make_tensor_ptr({2, 2}, {3.f, 5.f, 7.f, 9.f}); + EXPECT_TENSOR_CLOSE(result->at(0).toTensor(), *expected.get()); } TEST_F(ModuleTest, TestPartiallySetInputs) { Module module(model_path_); - auto tensor = make_tensor_ptr({1.f}); + auto tensor = make_tensor_ptr({2, 2}, {1.f, 2.f, 3.f, 4.f}); EXPECT_EQ(module.set_input(tensor, 0), Error::Ok); @@ -455,8 +453,6 @@ TEST_F(ModuleTest, TestPTD) { ASSERT_EQ(module.load_method("forward"), Error::Ok); - auto tensor1 = - make_tensor_ptr({3, 3}, {2.f, 3.f, 4.f, 2.f, 3.f, 4.f, 2.f, 3.f, 4.f}); - - ASSERT_EQ(module.forward(tensor1).error(), Error::Ok); + auto tensor = make_tensor_ptr({2, 2}, {2.f, 3.f, 4.f, 2.f}); + ASSERT_EQ(module.forward(tensor).error(), Error::Ok); } diff --git a/extension/module/test/resources/README.md b/extension/module/test/resources/README.md deleted file mode 100644 index ecbdd41c107..00000000000 --- a/extension/module/test/resources/README.md +++ /dev/null @@ -1,23 +0,0 @@ -## Resources - -### add.pte, linear.pte, linear.ptd -- Internally generated after D62209852, 2024-09-06 with: - ``` - buck2 run fbcode//executorch/examples/portable/scripts:export -- --model_name="add" - ``` - - and - - ``` - buck2 run fbcode//executorch/examples/portable/scripts:export -- --model_name="linear" -examples - ``` -- In OSS, the same file can be generated after [#5145](https://github.com/pytorch/executorch/pull/5145), 2024-09-06 with: - ``` - python -m examples.portable.scripts.export --model_name="add" - ``` - - and - - ``` - python -m examples.portable.scripts.export --model_name="linear" -e - ``` diff --git a/extension/module/test/resources/add.pte b/extension/module/test/resources/add.pte deleted file mode 100644 index 43252ca7d3d..00000000000 Binary files a/extension/module/test/resources/add.pte and /dev/null differ diff --git a/extension/module/test/resources/linear.ptd b/extension/module/test/resources/linear.ptd deleted file mode 100644 index edab857bb3f..00000000000 Binary files a/extension/module/test/resources/linear.ptd and /dev/null differ diff --git a/extension/module/test/resources/linear.pte b/extension/module/test/resources/linear.pte deleted file mode 100644 index 707815ad881..00000000000 Binary files a/extension/module/test/resources/linear.pte and /dev/null differ diff --git a/extension/module/test/targets.bzl b/extension/module/test/targets.bzl index 19ba09cf4e6..b684e8d07b6 100644 --- a/extension/module/test/targets.bzl +++ b/extension/module/test/targets.bzl @@ -5,39 +5,42 @@ load( ) load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_aten_mode_options", "runtime") -def define_common_targets(): +def define_common_targets(is_fbcode=False): """Defines targets that should be shared between fbcode and xplat. The directory containing this targets.bzl file should also contain both TARGETS and BUCK files that call this function. """ + if not runtime.is_oss and is_fbcode: + modules_env = { + # The tests use this var to find the program file to load. This uses + # an fbcode target path because the authoring/export tools + # intentionally don't work in xplat (since they're host-only tools). + "ET_MODULE_ADD_PATH": "$(location fbcode//executorch/test/models:exported_programs[ModuleAdd.pte])", + "ET_MODULE_LINEAR_PROGRAM_PATH": "$(location fbcode//executorch/test/models:exported_program_and_data[ModuleLinear.pte])", + "ET_MODULE_LINEAR_DATA_PATH": "$(location fbcode//executorch/test/models:exported_program_and_data[ModuleLinear.ptd])", + } - for aten_mode in get_aten_mode_options(): - aten_suffix = ("_aten" if aten_mode else "") + for aten_mode in get_aten_mode_options(): + aten_suffix = ("_aten" if aten_mode else "") - runtime.cxx_test( - name = "test" + aten_suffix, - srcs = [ - "module_test.cpp", - ], - deps = [ - "//executorch/kernels/portable:generated_lib" + aten_suffix, - "//executorch/extension/data_loader:file_data_loader", - "//executorch/extension/module:module" + aten_suffix, - "//executorch/extension/tensor:tensor" + aten_suffix, - ], - env = { - "RESOURCES_PATH": "$(location :resources)/resources", - }, - platforms = [CXX, ANDROID], # Cannot bundle resources on Apple platform. - compiler_flags = [ - "-Wno-error=deprecated-declarations", - ], - ) - - runtime.filegroup( - name = "resources", - srcs = native.glob([ - "resources/**", - ]), - ) + runtime.cxx_test( + name = "test" + aten_suffix, + srcs = [ + "module_test.cpp", + ], + deps = [ + "//executorch/kernels/portable:generated_lib" + aten_suffix, + "//executorch/extension/data_loader:file_data_loader", + "//executorch/extension/module:module" + aten_suffix, + "//executorch/extension/tensor:tensor" + aten_suffix, + "//executorch/runtime/core/exec_aten/testing_util:tensor_util" + aten_suffix, + ], + env = modules_env, + platforms = [CXX, ANDROID], # Cannot bundle resources on Apple platform. + compiler_flags = [ + "-Wno-error=deprecated-declarations", + ], + ) + +