Skip to content

Commit a6d2a31

Browse files
lucylqlucylq
committed
Introduce public MergedDataMap
Pull Request resolved: #14766 Add public merged data map. Module can use this to resolve multiple named data maps. Creating as a sep dependency rather than inside module/ so it can be used independently of module. (think there may be some other internal usages soon) Add support for BUCK and CMake. On CMake side, have to add the dep for everyone who uses module .. ghstack-source-id: 314455997 @exported-using-ghexport Differential Revision: [D83527299](https://our.internmc.facebook.com/intern/diff/D83527299/)
1 parent d8a2126 commit a6d2a31

31 files changed

+611
-1
lines changed

‎.ci/scripts/build-qnn-sdk.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ set_up_aot() {
3838
-DEXECUTORCH_BUILD_EXTENSION_EXTENSION_LLM=ON \
3939
-DEXECUTORCH_BUILD_EXTENSION_EXTENSION_LLM_RUNNER=ON \
4040
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
41+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
4142
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
4243
-DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
4344
-DPYTHON_EXECUTABLE=python3

‎.ci/scripts/test_llama_torchao_lowbit.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ cmake -DPYTHON_EXECUTABLE=python \
3131
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
3232
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
3333
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
34+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
3435
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
3536
-DEXECUTORCH_BUILD_XNNPACK=OFF \
3637
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \

‎.ci/scripts/test_torchao_huggingface_checkpoints.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ if [[ "$TEST_WITH_RUNNER" -eq 1 ]]; then
129129
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
130130
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
131131
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
132+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
132133
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
133134
-DEXECUTORCH_BUILD_XNNPACK=ON \
134135
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \

‎.ci/scripts/test_yolo12.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ cmake_install_executorch_libraries() {
119119
-DEXECUTORCH_BUILD_XNNPACK="$XNNPACK" \
120120
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
121121
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
122+
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
123+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
122124
-DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
123125
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
124126
-B"${build_dir}"
@@ -131,6 +133,8 @@ cmake_install_executorch_libraries() {
131133
-DEXECUTORCH_BUILD_XNNPACK="$XNNPACK" \
132134
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
133135
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
136+
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
137+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
134138
-DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
135139
-DEXECUTORCH_ENABLE_LOGGING=ON \
136140
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \

‎.github/workflows/trunk.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -852,6 +852,7 @@ jobs:
852852
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
853853
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
854854
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
855+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
855856
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
856857
-DEXECUTORCH_BUILD_XNNPACK=ON \
857858
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \

‎CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -630,6 +630,11 @@ if(EXECUTORCH_BUILD_EXTENSION_MODULE)
630630
list(APPEND _executorch_extensions extension_module_static)
631631
endif()
632632

633+
if(EXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP)
634+
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/named_data_map)
635+
list(APPEND _executorch_extensions extension_named_data_map)
636+
endif()
637+
633638
if(EXECUTORCH_BUILD_EXTENSION_LLM)
634639
if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER)
635640
set(SUPPORT_REGEX_LOOKAHEAD ON)

‎backends/mediatek/scripts/mtk_build.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ cmake -DCMAKE_INSTALL_PREFIX="${build_dir}" \
3030
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
3131
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
3232
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
33+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
3334
-DEXECUTORCH_BUILD_NEURON=ON \
3435
-B"${build_dir}"
3536

‎backends/qualcomm/scripts/build.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ if [ "$BUILD_AARCH64" = true ]; then
8686
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
8787
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
8888
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
89+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
8990
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
9091
-DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
9192
-DEXECUTORCH_ENABLE_LOGGING=ON \
@@ -155,6 +156,7 @@ if [ "$BUILD_X86_64" = true ]; then
155156
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
156157
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
157158
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
159+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
158160
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
159161
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
160162
-DEXECUTORCH_ENABLE_EVENT_TRACER=ON \

‎backends/samsung/build.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ function build_x86_64() {
4545
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
4646
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
4747
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
48+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
4849
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
4950
-S ${PROJECT_DIR} \
5051
-B ${X86_64_BUILD_DIR}
@@ -77,6 +78,7 @@ function build_android() {
7778
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
7879
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
7980
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
81+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
8082
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
8183
-DEXECUTORCH_ENABLE_LOGGING=1 \
8284
-DEXECUTORCH_BUILD_DEVTOOLS=ON \

‎backends/vulkan/test/scripts/test_model.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ build_core_libraries_and_devtools() {
111111
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
112112
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
113113
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
114+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
114115
-DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
115116
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
116117
-DEXECUTORCH_BUILD_EXECUTOR_RUNNER=ON \

0 commit comments

Comments
 (0)