Update on "[ET-VK] 7/n Split dispatches between multiple command buffers. Split execute dispatch into multiple commands based on dispatch count."

Differential Revision: [D78360039](https://our.internmc.facebook.com/intern/diff/D78360039/)

[ghstack-poisoned]

Author: trivedivivek

.ci/scripts/test_phi_3_mini.sh

@@ -22,31 +22,14 @@ NPROC=8
 if hash nproc &> /dev/null; then NPROC=$(nproc); fi
 
 cmake_install_executorch_libraries() {
-  cmake -DPYTHON_EXECUTABLE=python \
-      -DCMAKE_INSTALL_PREFIX=${BUILD_DIR} \
-      -DEXECUTORCH_ENABLE_LOGGING=1 \
-      -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-      -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
-      -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
-      -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
-      -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
-      -DEXECUTORCH_BUILD_XNNPACK=ON \
-      -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
-      -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
-      -DEXECUTORCH_BUILD_KERNELS_LLM=ON \
-      -B${BUILD_DIR} .
-
-  cmake --build ${BUILD_DIR} -j${NPROC} --target install --config ${BUILD_TYPE}
+  rm -rf cmake-out
+  cmake --preset llm -DCMAKE_INSTALL_PREFIX=cmake-out -DCMAKE_BUILD_TYPE=${BUILD_TYPE}
+  cmake --build cmake-out -j16 --target install --config ${BUILD_TYPE}
 }
 
 cmake_build_phi_3_mini() {
-  cmake -DPYTHON_EXECUTABLE=$PYTHON_EXECUTABLE \
-      -DCMAKE_INSTALL_PREFIX=${BUILD_DIR} \
+  cmake -DCMAKE_PREFIX_PATH=${BUILD_DIR} \
       -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-      -DEXECUTORCH_BUILD_KERNELS_LLM=ON \
-      -DEXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON \
-      -DEXECUTORCH_BUILD_XNNPACK=ON \
-      -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
       -B${BUILD_DIR}/${MODEL_DIR} \
       ${MODEL_DIR}
 
@@ -81,7 +64,7 @@ run_and_verify() {
     ${BUILD_DIR}/${MODEL_DIR}/phi_3_mini_runner \
     --model_path=phi-3-mini.pte \
     --tokenizer_path=tokenizer.bin \
-    --seq_len=128 \
+    --seq_len=60 \
     --temperature=0 \
     --prompt="<|system|>
 You are a helpful assistant.<|end|>

.github/workflows/build-wheels-linux.yml

@@ -9,6 +9,8 @@ on:
       - examples/**/*
       - pyproject.toml
       - setup.py
+    tags:
+      - ciflow/binaries/*
   push:
     branches:
       - nightly

.github/workflows/build-wheels-macos.yml

@@ -9,6 +9,8 @@ on:
       - examples/**/*
       - pyproject.toml
       - setup.py
+    tags:
+      - ciflow/binaries/*
   push:
     branches:
       - nightly

.github/workflows/pull.yml

@@ -603,7 +603,7 @@ jobs:
         bash examples/models/phi-3-mini/install_requirements.sh
 
         # run e2e (export, tokenizer and runner)
-        PYTHON_EXECUTABLE=python bash .ci/scripts/test_phi_3_mini.sh
+        PYTHON_EXECUTABLE=python bash .ci/scripts/test_phi_3_mini.sh Release
 
   test-eval_llama-wikitext-linux:
     name: test-eval_llama-wikitext-linux
@@ -762,3 +762,66 @@ jobs:
 
         # Test selective build
         PYTHON_EXECUTABLE=python bash examples/wasm/test_build_wasm.sh
+
+  unittest-nxp-neutron:
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-clang12
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        set -eux
+
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        # Build and install Executorch
+        PYTHON_EXECUTABLE=python \
+        CMAKE_ARGS="-DEXECUTORCH_BUILD_NXP_NEUTRON=ON" \
+        .ci/scripts/setup-linux.sh --build-tool "cmake"
+
+        # Install test requirements
+        pip install -r backends/nxp/requirements-tests.txt
+
+        # Run pytest
+        PYTHON_EXECUTABLE=python bash backends/nxp/run_unittests.sh
+                
+        # Run aot example: 
+        PYTHON_EXECUTABLE=python bash examples/nxp/run_aot_example.sh
+
+
+  nxp-build-test:
+    name: nxp-build-test
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      runner: linux.2xlarge
+      docker-image: executorch-ubuntu-22.04-arm-sdk
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        # Build
+        cmake -DEXECUTORCH_BUILD_NXP_NEUTRON=ON -Bcmake-out .
+        cmake --build cmake-out --target executorch_delegate_neutron --config Release
+
+        # Build check for the neutron backend library
+        lib_neutron="cmake-out/backends/nxp/libexecutorch_delegate_neutron.a"
+        if [ -f $lib_neutron ]; then
+            echo "Neutron backend library built."
+        else
+            echo "Neutron backend library not found!"
+            exit 1
+        fi

.github/workflows/trunk.yml

@@ -302,36 +302,6 @@ jobs:
           exit 1
         fi
 
-  nxp-build-test:
-    name: nxp-build-test
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      runner: linux.2xlarge
-      docker-image: executorch-ubuntu-22.04-arm-sdk
-      submodules: 'recursive'
-      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout: 90
-      script: |
-        # The generic Linux job chooses to use base env, not the one setup by the image
-        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
-        conda activate "${CONDA_ENV}"
-
-        # Build
-        cmake -DEXECUTORCH_BUILD_NXP_NEUTRON=ON -Bcmake-out .
-        cmake --build cmake-out --target executorch_delegate_neutron --config Release
-
-        # Build check for the neutron backend library
-        lib_neutron="cmake-out/backends/nxp/libexecutorch_delegate_neutron.a"
-        if [ -f $lib_neutron ]; then
-            echo "Neutron backend library built."
-        else
-            echo "Neutron backend library not found!"
-            exit 1
-        fi
-
   test-coreml-delegate:
     name: test-coreml-delegate
     uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
@@ -771,32 +741,3 @@ jobs:
       build-mode: Release
       build-tool: cmake
       docker-image: executorch-ubuntu-22.04-clang12
-
-  unittest-nxp-neutron:
-    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
-    permissions:
-      id-token: write
-      contents: read
-    with:
-      runner: linux.2xlarge
-      docker-image: executorch-ubuntu-22.04-clang12
-      submodules: 'recursive'
-      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
-      timeout: 90
-      script: |
-        set -eux
-
-        # The generic Linux job chooses to use base env, not the one setup by the image
-        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
-        conda activate "${CONDA_ENV}"
-
-        # Build and install Executorch
-        PYTHON_EXECUTABLE=python \
-        CMAKE_ARGS="-DEXECUTORCH_BUILD_NXP_NEUTRON=ON" \
-        .ci/scripts/setup-linux.sh --build-tool "cmake"
-
-        # Install test requirements
-        pip install -r backends/nxp/requirements-tests.txt
-
-        # Run pytest
-        PYTHON_EXECUTABLE=python bash backends/nxp/run_unittests.sh

CMakeLists.txt

@@ -112,11 +112,12 @@ set(CMAKE_SKIP_BUILD_RPATH OFF)
 set(CMAKE_BUILD_WITH_INSTALL_RPATH ON)
 # Automatically add all linked folders that are NOT in the build directory to
 # the rpath (per library?)
-# TODO: Doesn't work for us right now because we are
-# not installing .so's into the correct locations. For example we have
-# libcustom_ops_aot_lib.so depending on _portable_lib.so, which was eventually
-# put under <site-packages>/executorch/extension/pybindings/ but this rpath is
-# not automatically added because at build time it seems `portable_lib` is being
+#
+# TODO: Doesn't work for us right now because we are not installing .so's into
+# the correct locations. For example we have libcustom_ops_aot_lib.so depending
+# on _portable_lib.so, which was eventually put under
+# <site-packages>/executorch/extension/pybindings/ but this rpath is not
+# automatically added because at build time it seems `portable_lib` is being
 # built under the same directory, so no extra rpath is being added. To properly
 # fix this we need to install `portable_lib` into the correct path.
 set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON)
@@ -321,8 +322,9 @@ if(EXECUTORCH_USE_CPP_CODE_COVERAGE)
            " -fprofile-instr-generate -fcoverage-mapping"
     )
   else()
-    message(FATAL_ERROR
-            "Code coverage for compiler ${CMAKE_CXX_COMPILER_ID} is unsupported"
+    message(
+      FATAL_ERROR
+        "Code coverage for compiler ${CMAKE_CXX_COMPILER_ID} is unsupported"
     )
   endif()
 endif()
@@ -633,8 +635,8 @@ if(EXECUTORCH_BUILD_PYBIND)
   endif()
 
   if(EXECUTORCH_BUILD_XNNPACK)
-    # need to explicitly specify XNNPACK and xnnpack-microkernels-prod here otherwise
-    # uses XNNPACK and microkernel-prod symbols from libtorch_cpu
+    # need to explicitly specify XNNPACK and xnnpack-microkernels-prod here
+    # otherwise uses XNNPACK and microkernel-prod symbols from libtorch_cpu
     list(APPEND _dep_libs xnnpack_backend XNNPACK xnnpack-microkernels-prod)
   endif()
 

backends/apple/coreml/CMakeLists.txt

@@ -104,7 +104,9 @@ if(APPLE)
 endif()
 
 add_library(coreml_util ${UTIL_SOURCES})
-target_include_directories(coreml_util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/runtime/util)
+target_include_directories(
+  coreml_util PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/runtime/util
+)
 if(APPLE)
   target_link_libraries(coreml_util PRIVATE ${FOUNDATION_FRAMEWORK})
 endif()
@@ -119,21 +121,25 @@ install(
 
 # CoreML inmemoryfs
 
-set(
-  INMEMORYFS_SOURCES
-  runtime/inmemoryfs/inmemory_filesystem.cpp
-  runtime/inmemoryfs/memory_buffer.cpp
-  runtime/inmemoryfs/memory_stream.cpp
-  runtime/inmemoryfs/reversed_memory_stream.cpp
+set(INMEMORYFS_SOURCES
+    runtime/inmemoryfs/inmemory_filesystem.cpp
+    runtime/inmemoryfs/memory_buffer.cpp runtime/inmemoryfs/memory_stream.cpp
+    runtime/inmemoryfs/reversed_memory_stream.cpp
 )
 if(APPLE)
-  list(APPEND INMEMORYFS_SOURCES runtime/inmemoryfs/inmemory_filesystem_utils.mm)
+  list(APPEND INMEMORYFS_SOURCES
+       runtime/inmemoryfs/inmemory_filesystem_utils.mm
+  )
 endif()
 
 add_library(coreml_inmemoryfs ${INMEMORYFS_SOURCES})
-target_include_directories(coreml_inmemoryfs PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/runtime/inmemoryfs)
+target_include_directories(
+  coreml_inmemoryfs PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/runtime/inmemoryfs
+)
 if(APPLE)
-  target_link_libraries(coreml_inmemoryfs PRIVATE coreml_util ${FOUNDATION_FRAMEWORK})
+  target_link_libraries(
+    coreml_inmemoryfs PRIVATE coreml_util ${FOUNDATION_FRAMEWORK}
+  )
 endif()
 target_compile_options(coreml_inmemoryfs PUBLIC -fPIC)
 
@@ -148,17 +154,12 @@ install(
 
 if(EXECUTORCH_BUILD_PYBIND)
   pybind11_add_module(
-    executorchcoreml
-    SHARED
-      runtime/inmemoryfs/inmemory_filesystem_py.cpp
-      runtime/inmemoryfs/inmemory_filesystem_utils.cpp
+    executorchcoreml SHARED runtime/inmemoryfs/inmemory_filesystem_py.cpp
+    runtime/inmemoryfs/inmemory_filesystem_utils.cpp
   )
   target_link_libraries(
-    executorchcoreml
-    PRIVATE
-      coreml_util
-      coreml_inmemoryfs
-      nlohmann_json::nlohmann_json
+    executorchcoreml PRIVATE coreml_util coreml_inmemoryfs
+                             nlohmann_json::nlohmann_json
   )
   target_compile_options(executorchcoreml PUBLIC -fPIC)
 endif()
@@ -179,8 +180,12 @@ if(APPLE)
     coremldelegate PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/runtime/delegate
   )
   target_include_directories(coremldelegate PRIVATE ${PROJECT_SOURCE_DIR}/..)
-  target_include_directories(coremldelegate PRIVATE ${PROJECT_SOURCE_DIR}/runtime/core/portable_type/c10)
-  target_compile_definitions(coremldelegate PRIVATE C10_USING_CUSTOM_GENERATED_MACROS)
+  target_include_directories(
+    coremldelegate PRIVATE ${PROJECT_SOURCE_DIR}/runtime/core/portable_type/c10
+  )
+  target_compile_definitions(
+    coremldelegate PRIVATE C10_USING_CUSTOM_GENERATED_MACROS
+  )
 
   if(EXECUTORCH_BUILD_DEVTOOLS)
     target_sources(coremldelegate PRIVATE ${SDK_SOURCES} ${PROTOBUF_SOURCES})
@@ -200,13 +205,9 @@ if(APPLE)
 
   target_link_libraries(
     coremldelegate
-    PUBLIC  coreml_util
-            coreml_inmemoryfs
-    PRIVATE executorch_core
-            ${ACCELERATE_FRAMEWORK}
-            ${COREML_FRAMEWORK}
-            ${FOUNDATION_FRAMEWORK}
-            ${SQLITE_LIBRARY}
+    PUBLIC coreml_util coreml_inmemoryfs
+    PRIVATE executorch_core ${ACCELERATE_FRAMEWORK} ${COREML_FRAMEWORK}
+            ${FOUNDATION_FRAMEWORK} ${SQLITE_LIBRARY}
   )
 
   target_link_options_shared_lib(coremldelegate)
@@ -218,13 +219,8 @@ if(APPLE)
   endif()
 
   target_compile_options(
-    coremldelegate
-    PRIVATE
-      -fobjc-arc
-      -fno-exceptions
-      -x objective-c++
-      -Wno-null-character
-      -Wno-receiver-expr
+    coremldelegate PRIVATE -fobjc-arc -fno-exceptions -x objective-c++
+                           -Wno-null-character -Wno-receiver-expr
   )
 
   if(EXECUTORCH_BUILD_DEVTOOLS)

backends/arm/test/tester/arm_tester.py

@@ -726,7 +726,7 @@ def _get_dtype_distribution(
         if node.op == "placeholder":
             placeholder_dtypes.append(str(node.meta["val"].dtype))
         if node.op == "call_function":
-            if "val" in node.meta:
+            if "val" in node.meta and isinstance(node.meta["val"], torch.Tensor):
                 dtype, _, _ = extract_tensor_meta(node.meta, tosa_spec)
                 call_function_dtypes.append(ts.DTypeNames[dtype])
     return Counter(placeholder_dtypes), Counter(call_function_dtypes)

backends/mediatek/CMakeLists.txt

@@ -24,13 +24,9 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/runtime/include)
 # targets
 add_library(neuron_backend SHARED)
 target_compile_options(neuron_backend PRIVATE "-frtti" "-fexceptions")
-target_link_libraries(neuron_backend
-    PRIVATE
-    executorch_core
-    portable_ops_lib
-    portable_kernels
-    android
-    log
+target_link_libraries(
+  neuron_backend PRIVATE executorch_core portable_ops_lib portable_kernels
+                         android log
 )
 target_sources(
   neuron_backend