diff --git a/CMakeLists.txt b/CMakeLists.txt index 91800297469..cfae0f8b74b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -309,9 +309,15 @@ set(_common_include_directories ) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # - +if(EXECUTORCH_SRCS_FILE) + message( + WARNING + "EXECUTORCH_SRCS_FILE is no longer necessary and will not affect the build." + ) +endif() +executorch_load_build_variables() if(NOT EXECUTORCH_SRCS_FILE) # A file wasn't provided. Run a script to extract the source lists from the # buck2 build system and write them to a file we can include. @@ -324,10 +330,6 @@ if(NOT EXECUTORCH_SRCS_FILE) executorch_validate_build_variables() endif() -# This file defines the `___srcs` variables used below. -message(STATUS "executorch: Using sources file ${EXECUTORCH_SRCS_FILE}") -include(${EXECUTORCH_SRCS_FILE}) - # Detect if an iOS toolchain is set. if(CMAKE_TOOLCHAIN_FILE MATCHES ".*(iOS|ios\.toolchain)\.cmake$") set(CMAKE_TOOLCHAIN_IOS ON) diff --git a/backends/apple/mps/targets.bzl b/backends/apple/mps/targets.bzl index 74d79448362..99c97d2b318 100644 --- a/backends/apple/mps/targets.bzl +++ b/backends/apple/mps/targets.bzl @@ -3,6 +3,7 @@ # Provided subject to the LICENSE file in the top level directory. # +load("@fbsource//xplat/executorch/build:build_variables.bzl", "MPS_BACKEND_BUCK_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") def define_common_targets(is_xplat = False, platforms = []): @@ -37,10 +38,7 @@ def define_common_targets(is_xplat = False, platforms = []): "runtime/*.h", "runtime/operations/*.h", ]), - "srcs": native.glob([ - "runtime/*.mm", - "runtime/operations/*.mm", - ]), + "srcs": MPS_BACKEND_BUCK_SRCS, "visibility": [ "//executorch/backends/apple/...", "//executorch/examples/...", diff --git a/backends/xnnpack/targets.bzl b/backends/xnnpack/targets.bzl index aee5104b17a..0eab89a00f9 100644 --- a/backends/xnnpack/targets.bzl +++ b/backends/xnnpack/targets.bzl @@ -1,4 +1,5 @@ load("@fbsource//xplat/executorch/backends/xnnpack/third-party:third_party_libs.bzl", "third_party_dep") +load("@fbsource//xplat/executorch/build:build_variables.bzl", "XNNPACK_BACKEND_BUCK_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_aten_mode_options", "runtime") def _get_preprocessor_flags(): @@ -37,10 +38,7 @@ def define_common_targets(): aten_suffix = "_aten" if aten_mode else "" runtime.cxx_library( name = "xnnpack_backend" + aten_suffix, - srcs = native.glob([ - "runtime/*.cpp", - "runtime/profiling/*.cpp", - ]), + srcs = XNNPACK_BACKEND_BUCK_SRCS, headers = native.glob([ "runtime/*.h", "runtime/profiling/*.h", diff --git a/examples/apple/mps/CMakeLists.txt b/examples/apple/mps/CMakeLists.txt index 3f61cedec8e..8a562dd206b 100644 --- a/examples/apple/mps/CMakeLists.txt +++ b/examples/apple/mps/CMakeLists.txt @@ -76,16 +76,10 @@ if(NOT CMAKE_TOOLCHAIN_FILE MATCHES ".*(iOS|ios\.toolchain)\.cmake$") ) # - # The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. + # The `__srcs` lists are defined by executorch_load_build_variables. # - set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../../executorch_srcs.cmake" - ) - - extract_sources(${EXECUTORCH_SRCS_FILE}) - + executorch_load_build_variables() set(_mps_schema_headers ${CMAKE_BINARY_DIR}/../../../schema/include/) - include(${EXECUTORCH_SRCS_FILE}) target_include_directories( bundled_program INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/../../../devtools/include diff --git a/examples/mediatek/CMakeLists.txt b/examples/mediatek/CMakeLists.txt index 57c4b13e5cb..2bd08de2ffb 100644 --- a/examples/mediatek/CMakeLists.txt +++ b/examples/mediatek/CMakeLists.txt @@ -36,13 +36,9 @@ set(_common_include_directories ) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../executorch_srcs.cmake" -) -extract_sources(${EXECUTORCH_SRCS_FILE}) -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # Find prebuilt libraries. executorch package should contain portable_ops_lib, # etdump, bundled_program. diff --git a/examples/portable/custom_ops/CMakeLists.txt b/examples/portable/custom_ops/CMakeLists.txt index c4a00e47991..4188554af79 100644 --- a/examples/portable/custom_ops/CMakeLists.txt +++ b/examples/portable/custom_ops/CMakeLists.txt @@ -59,15 +59,9 @@ option( # ------------------------------- OPTIONS END -------------------------------- # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../../executorch_srcs.cmake" -) - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # Generate C++ bindings to register kernels into both PyTorch (for AOT) and # Executorch (for runtime). diff --git a/examples/qualcomm/CMakeLists.txt b/examples/qualcomm/CMakeLists.txt index d33d666b9c0..19190b6f794 100644 --- a/examples/qualcomm/CMakeLists.txt +++ b/examples/qualcomm/CMakeLists.txt @@ -35,13 +35,9 @@ find_package(gflags REQUIRED) set(_common_compile_options -Wno-deprecated-declarations -fPIC) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../executorch_srcs.cmake" -) -extract_sources(${EXECUTORCH_SRCS_FILE}) -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() get_filename_component( EXECUTORCH_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE diff --git a/examples/selective_build/CMakeLists.txt b/examples/selective_build/CMakeLists.txt index 3cc5e759ac4..dbff311a39a 100644 --- a/examples/selective_build/CMakeLists.txt +++ b/examples/selective_build/CMakeLists.txt @@ -77,15 +77,9 @@ option(EXECUTORCH_DTYPE_SELECTIVE_BUILD "Enable dtype selective build." OFF) # ------------------------------- OPTIONS END -------------------------------- # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../executorch_srcs.cmake" -) - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # # select_build_lib: C++ library to register selected ops in custom kernel diff --git a/extension/llm/custom_ops/CMakeLists.txt b/extension/llm/custom_ops/CMakeLists.txt index c5eba4b7a19..1678dc80296 100644 --- a/extension/llm/custom_ops/CMakeLists.txt +++ b/extension/llm/custom_ops/CMakeLists.txt @@ -25,15 +25,9 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../../executorch_srcs.cmake" -) - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() set(_common_include_directories $ diff --git a/extension/llm/custom_ops/targets.bzl b/extension/llm/custom_ops/targets.bzl index 545f6516bb7..26198ec0854 100644 --- a/extension/llm/custom_ops/targets.bzl +++ b/extension/llm/custom_ops/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//xplat/executorch/build:build_variables.bzl", "EXTENSION_LLM_CUSTOM_OPS_BUCK_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") load( "@fbsource//xplat/executorch/kernels/optimized:lib_defs.bzl", @@ -30,12 +31,7 @@ def define_common_targets(): for mkl_dep in ["", "_mkl_noomp"]: runtime.cxx_library( name = "custom_ops" + mkl_dep, - srcs = [ - "op_fallback.cpp", - "op_fast_hadamard_transform.cpp", - "op_sdpa.cpp", - "op_update_cache.cpp", - ], + srcs = EXTENSION_LLM_CUSTOM_OPS_BUCK_SRCS, exported_headers = [ "op_fallback.h", "op_fast_hadamard_transform.h", diff --git a/extension/llm/runner/CMakeLists.txt b/extension/llm/runner/CMakeLists.txt index f5933e82e32..c9a72fe61a0 100644 --- a/extension/llm/runner/CMakeLists.txt +++ b/extension/llm/runner/CMakeLists.txt @@ -24,15 +24,9 @@ include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE - "${CMAKE_CURRENT_BINARY_DIR}/../../../executorch_srcs.cmake" -) - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # build llm runner library list(TRANSFORM _extension_llm_runner__srcs PREPEND "${EXECUTORCH_ROOT}/") diff --git a/extension/threadpool/targets.bzl b/extension/threadpool/targets.bzl index 5e7cf2c7dae..6ef55c42434 100644 --- a/extension/threadpool/targets.bzl +++ b/extension/threadpool/targets.bzl @@ -1,4 +1,5 @@ load("@fbsource//xplat/executorch/backends/xnnpack/third-party:third_party_libs.bzl", "third_party_dep") +load("@fbsource//xplat/executorch/build:build_variables.bzl", "THREADPOOL_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") def define_common_targets(): @@ -8,11 +9,9 @@ def define_common_targets(): TARGETS and BUCK files that call this function. """ - _THREADPOOL_SRCS = [ - "thread_parallel.cpp", - "threadpool.cpp", - "threadpool_guard.cpp", - ] + (["fb/threadpool_use_n_threads.cpp"] if not runtime.is_oss else []) + _THREADPOOL_SRCS = THREADPOOL_SRCS + ( + ["fb/threadpool_use_n_threads.cpp"] if not runtime.is_oss else [] + ) _THREADPOOL_HEADERS = [ "threadpool.h", diff --git a/kernels/portable/cpu/pattern/targets.bzl b/kernels/portable/cpu/pattern/targets.bzl index 9d6f1a6885d..636c5d2127b 100644 --- a/kernels/portable/cpu/pattern/targets.bzl +++ b/kernels/portable/cpu/pattern/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//xplat/executorch/build:build_variables.bzl", "PATTERN_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") def define_common_targets(): @@ -49,11 +50,7 @@ def define_common_targets(): runtime.cxx_library( name = "pattern", - srcs = [ - "unary_ufunc_realhbbf16_to_bool.cpp", - "unary_ufunc_realhbbf16_to_floathbf16.cpp", - "unary_ufunc_realhbf16.cpp", - ], + srcs = PATTERN_SRCS, exported_headers = [ "pattern.h", ], diff --git a/runtime/executor/targets.bzl b/runtime/executor/targets.bzl index ec0fb19ff96..103ea299c34 100644 --- a/runtime/executor/targets.bzl +++ b/runtime/executor/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//xplat/executorch/build:build_variables.bzl", "PROGRAM_NO_PRIM_OPS_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_aten_mode_options", "runtime") def _program_preprocessor_flags(): @@ -93,11 +94,7 @@ def define_common_targets(): runtime.cxx_library( name = "program_no_prim_ops" + aten_suffix, - srcs = [ - "method.cpp", - "method_meta.cpp", - "program.cpp", - "tensor_parser_exec_aten.cpp", + srcs = PROGRAM_NO_PRIM_OPS_SRCS + [ "tensor_parser{}.cpp".format(aten_suffix if aten_mode else "_portable"), ], headers = [ diff --git a/runtime/platform/targets.bzl b/runtime/platform/targets.bzl index eecac8ae5db..457deed531e 100644 --- a/runtime/platform/targets.bzl +++ b/runtime/platform/targets.bzl @@ -1,3 +1,4 @@ +load("@fbsource//xplat/executorch/build:build_variables.bzl", "PLATFORM_SRCS") load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") load(":log.bzl", "get_et_logging_flags") @@ -73,13 +74,7 @@ def define_common_targets(): "runtime.h", "compat_unistd.h", ], - srcs = [ - "abort.cpp", - "log.cpp", - "platform.cpp", - "profiler.cpp", - "runtime.cpp", - ], + srcs = PLATFORM_SRCS, exported_preprocessor_flags = get_profiling_flags() + get_et_logging_flags(), exported_deps = [ "//executorch/runtime/platform:pal_interface", diff --git a/shim_et/xplat/executorch/build/build_variables.bzl b/shim_et/xplat/executorch/build/build_variables.bzl index e78cc08ef27..aa8ad0d4003 100644 --- a/shim_et/xplat/executorch/build/build_variables.bzl +++ b/shim_et/xplat/executorch/build/build_variables.bzl @@ -32,7 +32,22 @@ EXECUTORCH_SRCS = [ "kernels/prim_ops/register_prim_ops.cpp", ] -EXECUTORCH_CORE_SRCS = [ +PROGRAM_NO_PRIM_OPS_SRCS = [ + "method.cpp", + "method_meta.cpp", + "program.cpp", + "tensor_parser_exec_aten.cpp", +] + +PLATFORM_SRCS = [ + "abort.cpp", + "log.cpp", + "platform.cpp", + "profiler.cpp", + "runtime.cpp", +] + +EXECUTORCH_CORE_SRCS = sorted([ "runtime/backend/interface.cpp", "runtime/core/evalue.cpp", "runtime/core/exec_aten/util/tensor_shape_to_c_string.cpp", @@ -40,19 +55,16 @@ EXECUTORCH_CORE_SRCS = [ "runtime/core/portable_type/tensor_impl.cpp", "runtime/core/tag.cpp", "runtime/core/tensor_layout.cpp", - "runtime/executor/method.cpp", - "runtime/executor/method_meta.cpp", - "runtime/executor/program.cpp", - "runtime/executor/pte_data_map.cpp", - "runtime/executor/tensor_parser_exec_aten.cpp", "runtime/executor/tensor_parser_portable.cpp", + "runtime/executor/pte_data_map.cpp", "runtime/kernel/operator_registry.cpp", - "runtime/platform/abort.cpp", - "runtime/platform/log.cpp", - "runtime/platform/platform.cpp", - "runtime/platform/profiler.cpp", - "runtime/platform/runtime.cpp", "schema/extended_header.cpp", +] + ["runtime/executor/" + x for x in PROGRAM_NO_PRIM_OPS_SRCS] + ["runtime/platform/" + x for x in PLATFORM_SRCS]) + +PATTERN_SRCS = [ + "unary_ufunc_realhbbf16_to_bool.cpp", + "unary_ufunc_realhbbf16_to_floathbf16.cpp", + "unary_ufunc_realhbf16.cpp", ] PORTABLE_KERNELS_SRCS = [ @@ -221,10 +233,7 @@ PORTABLE_KERNELS_SRCS = [ "kernels/portable/cpu/op_view_copy.cpp", "kernels/portable/cpu/op_where.cpp", "kernels/portable/cpu/op_zeros.cpp", - "kernels/portable/cpu/pattern/unary_ufunc_realhbbf16_to_bool.cpp", - "kernels/portable/cpu/pattern/unary_ufunc_realhbbf16_to_floathbf16.cpp", - "kernels/portable/cpu/pattern/unary_ufunc_realhbf16.cpp", -] +] + ["kernels/portable/cpu/pattern/" + x for x in PATTERN_SRCS] KERNELS_UTIL_ALL_DEPS_SRCS = [ "kernels/portable/cpu/util/activation_ops_util.cpp", @@ -354,12 +363,14 @@ EXTENSION_TENSOR_SRCS = [ "extension/tensor/tensor_ptr_maker.cpp", ] -EXTENSION_THREADPOOL_SRCS = [ - "extension/threadpool/thread_parallel.cpp", - "extension/threadpool/threadpool.cpp", - "extension/threadpool/threadpool_guard.cpp", +THREADPOOL_SRCS = [ + "thread_parallel.cpp", + "threadpool.cpp", + "threadpool_guard.cpp", ] +EXTENSION_THREADPOOL_SRCS = ["extension/threadpool/" + x for x in THREADPOOL_SRCS] + EXTENSION_TRAINING_SRCS = [ "extension/data_loader/file_data_loader.cpp", "extension/data_loader/mmap_data_loader.cpp", @@ -426,32 +437,34 @@ MPS_EXECUTOR_RUNNER_SRCS = [ "extension/data_loader/file_data_loader.cpp", ] -MPS_BACKEND_SRCS = [ - "backends/apple/mps/runtime/MPSBackend.mm", - "backends/apple/mps/runtime/MPSCompiler.mm", - "backends/apple/mps/runtime/MPSDelegateHeader.mm", - "backends/apple/mps/runtime/MPSDevice.mm", - "backends/apple/mps/runtime/MPSExecutor.mm", - "backends/apple/mps/runtime/MPSGraphBuilder.mm", - "backends/apple/mps/runtime/MPSStream.mm", - "backends/apple/mps/runtime/operations/ActivationOps.mm", - "backends/apple/mps/runtime/operations/BinaryOps.mm", - "backends/apple/mps/runtime/operations/ClampOps.mm", - "backends/apple/mps/runtime/operations/ConstantOps.mm", - "backends/apple/mps/runtime/operations/ConvolutionOps.mm", - "backends/apple/mps/runtime/operations/IndexingOps.mm", - "backends/apple/mps/runtime/operations/LinearAlgebra.mm", - "backends/apple/mps/runtime/operations/NormalizationOps.mm", - "backends/apple/mps/runtime/operations/OperationUtils.mm", - "backends/apple/mps/runtime/operations/PadOps.mm", - "backends/apple/mps/runtime/operations/PoolingOps.mm", - "backends/apple/mps/runtime/operations/QuantDequant.mm", - "backends/apple/mps/runtime/operations/RangeOps.mm", - "backends/apple/mps/runtime/operations/ReduceOps.mm", - "backends/apple/mps/runtime/operations/ShapeOps.mm", - "backends/apple/mps/runtime/operations/UnaryOps.mm", +MPS_BACKEND_BUCK_SRCS = [ + "runtime/MPSBackend.mm", + "runtime/MPSCompiler.mm", + "runtime/MPSDelegateHeader.mm", + "runtime/MPSDevice.mm", + "runtime/MPSExecutor.mm", + "runtime/MPSGraphBuilder.mm", + "runtime/MPSStream.mm", + "runtime/operations/ActivationOps.mm", + "runtime/operations/BinaryOps.mm", + "runtime/operations/ClampOps.mm", + "runtime/operations/ConstantOps.mm", + "runtime/operations/ConvolutionOps.mm", + "runtime/operations/IndexingOps.mm", + "runtime/operations/LinearAlgebra.mm", + "runtime/operations/NormalizationOps.mm", + "runtime/operations/OperationUtils.mm", + "runtime/operations/PadOps.mm", + "runtime/operations/PoolingOps.mm", + "runtime/operations/QuantDequant.mm", + "runtime/operations/RangeOps.mm", + "runtime/operations/ReduceOps.mm", + "runtime/operations/ShapeOps.mm", + "runtime/operations/UnaryOps.mm", ] +MPS_BACKEND_SRCS = ["backends/apple/mps/" + x for x in MPS_BACKEND_BUCK_SRCS] + MPS_SCHEMA_SRCS = [ "backends/apple/mps/serialization/schema.fbs", ] @@ -461,15 +474,17 @@ XNN_EXECUTOR_RUNNER_SRCS = [ "extension/data_loader/file_data_loader.cpp", ] -XNNPACK_BACKEND_SRCS = [ - "backends/xnnpack/runtime/XNNCompiler.cpp", - "backends/xnnpack/runtime/XNNExecutor.cpp", - "backends/xnnpack/runtime/XNNHeader.cpp", - "backends/xnnpack/runtime/XNNPACKBackend.cpp", - "backends/xnnpack/runtime/XNNWeightsCache.cpp", - "backends/xnnpack/runtime/profiling/XNNProfiler.cpp", +XNNPACK_BACKEND_BUCK_SRCS = [ + "runtime/XNNCompiler.cpp", + "runtime/XNNExecutor.cpp", + "runtime/XNNHeader.cpp", + "runtime/XNNPACKBackend.cpp", + "runtime/XNNWeightsCache.cpp", + "runtime/profiling/XNNProfiler.cpp", ] +XNNPACK_BACKEND_SRCS = ["backends/xnnpack/" + x for x in XNNPACK_BACKEND_BUCK_SRCS] + XNNPACK_SCHEMA_SRCS = [ "backends/xnnpack/serialization/runtime_schema.fbs", ] @@ -478,13 +493,15 @@ VULKAN_SCHEMA_SRCS = [ "backends/vulkan/serialization/schema.fbs", ] -CUSTOM_OPS_SRCS = [ - "extension/llm/custom_ops/op_fallback.cpp", - "extension/llm/custom_ops/op_fast_hadamard_transform.cpp", - "extension/llm/custom_ops/op_sdpa.cpp", - "extension/llm/custom_ops/op_update_cache.cpp", +EXTENSION_LLM_CUSTOM_OPS_BUCK_SRCS = [ + "op_fallback.cpp", + "op_fast_hadamard_transform.cpp", + "op_sdpa.cpp", + "op_update_cache.cpp", +] + +CUSTOM_OPS_SRCS = ["extension/llm/custom_ops/" + x for x in EXTENSION_LLM_CUSTOM_OPS_BUCK_SRCS] + [ "extension/llm/custom_ops/spinquant/fast_hadamard_transform.cpp", - "kernels/portable/cpu/util/reduce_util.cpp", ] LLAMA_RUNNER_SRCS = [ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5ad429e822f..870da77deb6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -23,6 +23,7 @@ set(CMAKE_CXX_STANDARD 17) set(EXECUTORCH_ROOT ${CMAKE_CURRENT_SOURCE_DIR}/..) +include(${EXECUTORCH_ROOT}/tools/cmake/Codegen.cmake) include(${EXECUTORCH_ROOT}/tools/cmake/Utils.cmake) # Find prebuilt executorch library @@ -34,13 +35,9 @@ find_package(executorch CONFIG REQUIRED FIND_ROOT_PATH_BOTH) set(_common_include_directories ${EXECUTORCH_ROOT}/..) # -# The `__srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}. +# The `__srcs` lists are defined by executorch_load_build_variables. # -set(EXECUTORCH_SRCS_FILE "${CMAKE_CURRENT_BINARY_DIR}/../executorch_srcs.cmake") - -extract_sources(${EXECUTORCH_SRCS_FILE}) - -include(${EXECUTORCH_SRCS_FILE}) +executorch_load_build_variables() # Since extract_sources.py is not returning absolute values, we need to patch # the source paths. diff --git a/tools/cmake/Codegen.cmake b/tools/cmake/Codegen.cmake index e3fb2024ee1..93ba0f890a8 100644 --- a/tools/cmake/Codegen.cmake +++ b/tools/cmake/Codegen.cmake @@ -349,13 +349,16 @@ endfunction() function(executorch_append_filelist name outputvar) # configure_file adds its input to the list of CMAKE_RERUN dependencies configure_file( - ${PROJECT_SOURCE_DIR}/shim_et/xplat/executorch/build/build_variables.bzl + ${EXECUTORCH_ROOT}/shim_et/xplat/executorch/build/build_variables.bzl ${PROJECT_BINARY_DIR}/build_variables.bzl COPYONLY ) + if(NOT PYTHON_EXECUTABLE) + resolve_python_executable() + endif() execute_process( COMMAND "${PYTHON_EXECUTABLE}" -c - "exec(open('${PROJECT_SOURCE_DIR}/shim_et/xplat/executorch/build/build_variables.bzl').read());print(';'.join(${name}))" + "exec(open('${EXECUTORCH_ROOT}/shim_et/xplat/executorch/build/build_variables.bzl').read());print(';'.join(${name}))" WORKING_DIRECTORY "${_rootdir}" RESULT_VARIABLE _retval OUTPUT_VARIABLE _tempvar @@ -375,6 +378,73 @@ function(executorch_append_filelist name outputvar) ) endfunction() +set(EXECUTORCH_BUILD_VARIABLES_FILELISTS + EXECUTORCH_SRCS + EXECUTORCH_CORE_SRCS + PORTABLE_KERNELS_SRCS + KERNELS_UTIL_ALL_DEPS_SRCS + OPTIMIZED_KERNELS_SRCS + QUANTIZED_KERNELS_SRCS + PROGRAM_SCHEMA_SRCS + OPTIMIZED_CPUBLAS_SRCS + OPTIMIZED_NATIVE_CPU_OPS_SRCS + TEST_BACKEND_COMPILER_LIB_SRCS + EXTENSION_DATA_LOADER_SRCS + EXTENSION_EVALUE_UTIL_SRCS + EXTENSION_FLAT_TENSOR_SRCS + EXTENSION_MODULE_SRCS + EXTENSION_RUNNER_UTIL_SRCS + EXTENSION_LLM_RUNNER_SRCS + EXTENSION_TENSOR_SRCS + EXTENSION_THREADPOOL_SRCS + EXTENSION_TRAINING_SRCS + TRAIN_XOR_SRCS + EXECUTOR_RUNNER_SRCS + SIZE_TEST_SRCS + MPS_EXECUTOR_RUNNER_SRCS + MPS_BACKEND_SRCS + MPS_SCHEMA_SRCS + XNN_EXECUTOR_RUNNER_SRCS + XNNPACK_BACKEND_SRCS + XNNPACK_SCHEMA_SRCS + VULKAN_SCHEMA_SRCS + CUSTOM_OPS_SRCS + LLAMA_RUNNER_SRCS +) +set(EXECUTORCH_BUILD_VARIABLES_VARNAMES + _executorch__srcs + _executorch_core__srcs + _portable_kernels__srcs + _kernels_util_all_deps__srcs + _optimized_kernels__srcs + _quantized_kernels__srcs + _program_schema__srcs + _optimized_cpublas__srcs + _optimized_native_cpu_ops__srcs + _test_backend_compiler_lib__srcs + _extension_data_loader__srcs + _extension_evalue_util__srcs + _extension_flat_tensor__srcs + _extension_module__srcs + _extension_runner_util__srcs + _extension_llm_runner__srcs + _extension_tensor__srcs + _extension_threadpool__srcs + _extension_training__srcs + _train_xor__srcs + _executor_runner__srcs + _size_test__srcs + _mps_executor_runner__srcs + _mps_backend__srcs + _mps_schema__srcs + _xnn_executor_runner__srcs + _xnnpack_backend__srcs + _xnnpack_schema__srcs + _vulkan_schema__srcs + _custom_ops__srcs + _llama_runner__srcs +) + # Fail the build if the src lists in build_variables.bzl do not match the src # lists extracted from Buck and placed into EXECUTORCH_SRCS_FILE. This is # intended to be a safety mechanism while we are in the process of removing Buck @@ -386,78 +456,10 @@ endfunction() # involve getting these lists to match! function(executorch_validate_build_variables) include(${EXECUTORCH_SRCS_FILE}) - set(BUILD_VARIABLES_FILELISTS - EXECUTORCH_SRCS - EXECUTORCH_CORE_SRCS - PORTABLE_KERNELS_SRCS - KERNELS_UTIL_ALL_DEPS_SRCS - OPTIMIZED_KERNELS_SRCS - QUANTIZED_KERNELS_SRCS - PROGRAM_SCHEMA_SRCS - OPTIMIZED_CPUBLAS_SRCS - OPTIMIZED_NATIVE_CPU_OPS_SRCS - TEST_BACKEND_COMPILER_LIB_SRCS - EXTENSION_DATA_LOADER_SRCS - EXTENSION_EVALUE_UTIL_SRCS - EXTENSION_FLAT_TENSOR_SRCS - EXTENSION_MODULE_SRCS - EXTENSION_RUNNER_UTIL_SRCS - EXTENSION_LLM_RUNNER_SRCS - EXTENSION_TENSOR_SRCS - EXTENSION_THREADPOOL_SRCS - EXTENSION_TRAINING_SRCS - TRAIN_XOR_SRCS - EXECUTOR_RUNNER_SRCS - SIZE_TEST_SRCS - MPS_EXECUTOR_RUNNER_SRCS - MPS_BACKEND_SRCS - MPS_SCHEMA_SRCS - XNN_EXECUTOR_RUNNER_SRCS - XNNPACK_BACKEND_SRCS - XNNPACK_SCHEMA_SRCS - VULKAN_SCHEMA_SRCS - CUSTOM_OPS_SRCS - LLAMA_RUNNER_SRCS - ) - set(BUILD_VARIABLES_VARNAMES - _executorch__srcs - _executorch_core__srcs - _portable_kernels__srcs - _kernels_util_all_deps__srcs - _optimized_kernels__srcs - _quantized_kernels__srcs - _program_schema__srcs - _optimized_cpublas__srcs - _optimized_native_cpu_ops__srcs - _test_backend_compiler_lib__srcs - _extension_data_loader__srcs - _extension_evalue_util__srcs - _extension_flat_tensor__srcs - _extension_module__srcs - _extension_runner_util__srcs - _extension_llm_runner__srcs - _extension_tensor__srcs - _extension_threadpool__srcs - _extension_training__srcs - _train_xor__srcs - _executor_runner__srcs - _size_test__srcs - _mps_executor_runner__srcs - _mps_backend__srcs - _mps_schema__srcs - _xnn_executor_runner__srcs - _xnnpack_backend__srcs - _xnnpack_schema__srcs - _vulkan_schema__srcs - _custom_ops__srcs - _llama_runner__srcs + foreach(filelist_and_varname IN + ZIP_LISTS EXECUTORCH_BUILD_VARIABLES_FILELISTS + EXECUTORCH_BUILD_VARIABLES_VARNAMES ) - foreach(filelist_and_varname IN ZIP_LISTS BUILD_VARIABLES_FILELISTS - BUILD_VARIABLES_VARNAMES - ) - if("${filelist_and_varname_1}" STREQUAL "_custom_ops__srcs") - continue() - endif() executorch_append_filelist( ${filelist_and_varname_0} "${filelist_and_varname_1}_from_build_variables" @@ -507,3 +509,18 @@ function(executorch_validate_build_variables) endif() endforeach() endfunction() + +function(executorch_load_build_variables) + foreach(filelist_and_varname IN + ZIP_LISTS EXECUTORCH_BUILD_VARIABLES_FILELISTS + EXECUTORCH_BUILD_VARIABLES_VARNAMES + ) + executorch_append_filelist( + ${filelist_and_varname_0} "${filelist_and_varname_1}" + ) + set(${filelist_and_varname_1} + "${${filelist_and_varname_1}}" + PARENT_SCOPE + ) + endforeach() +endfunction()