Update

[ghstack-poisoned]

Author: abhinaykukkadapu

.ci/scripts/setup-vulkan-linux-deps.sh

@@ -23,6 +23,7 @@ install_swiftshader() {
 
   export VK_ICD_FILENAMES="${_swiftshader_dir}/swiftshader/build/Linux/vk_swiftshader_icd.json"
   export LD_LIBRARY_PATH="${_swiftshader_dir}/swiftshader/build/Linux/"
+  export ETVK_USING_SWIFTSHADER=1
 }
 
 install_vulkan_sdk() {

.github/workflows/pull.yml

@@ -864,6 +864,43 @@ jobs:
         PYTHON_EXECUTABLE=python bash examples/nxp/run_aot_example.sh
 
 
+  test-vulkan-models-linux:
+    name: test-vulkan-models-linux
+    uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
+    permissions:
+      id-token: write
+      contents: read
+    with:
+      runner: linux.2xlarge
+      docker-image: ci-image:executorch-ubuntu-22.04-clang12
+      submodules: 'recursive'
+      ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
+      script: |
+        set -eux
+
+        # The generic Linux job chooses to use base env, not the one setup by the image
+        CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
+        conda activate "${CONDA_ENV}"
+
+        # Setup swiftshader and Vulkan SDK which are required to build the Vulkan delegate
+        source .ci/scripts/setup-vulkan-linux-deps.sh
+
+        # Setup python
+        PYTHON_EXECUTABLE=python \
+        CMAKE_ARGS="-DEXECUTORCH_BUILD_VULKAN=ON" \
+        .ci/scripts/setup-linux.sh --build-tool "cmake"
+
+        PYTHON_EXECUTABLE=python bash backends/vulkan/test/scripts/test_model.sh --build
+
+        # Test models serially
+        models="mv2 mv3 edsr resnet18 resnet50 dl3"
+        for model in $models; do
+          python -m examples.vulkan.export --model_name=$model --test
+        done
+
+
+
   nxp-build-test:
     name: nxp-build-test
     uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

.gitignore

@@ -24,6 +24,7 @@ pip-out/
 # Any exported models and profiling outputs
 *.bin
 *.model
+*.etdump
 tokenizer.json
 *.pte
 *.ptd
@@ -58,6 +59,7 @@ xcuserdata/
 /include/
 /share/
 /version.py
+*.csv
 
 # Android
 *.aar

backends/test/harness/TARGETS

@@ -4,10 +4,7 @@ oncall("executorch")
 
 runtime.python_library(
     name = "tester",
-    srcs = [
-        "__init__.py",
-        "tester.py",
-    ] + native.glob(["stages/*.py"]),
+    srcs = native.glob(["*.py", "stages/*.py"]),
     visibility = [
         "//executorch/...",
         "@EXECUTORCH_CLIENTS",

backends/test/harness/stages/to_edge_transform_and_lower.py

@@ -14,11 +14,15 @@
 class ToEdgeTransformAndLower(Stage):
     def __init__(
         self,
-        default_partitioner_cls: Type,
+        default_partitioner_cls: Type | None = None,
         partitioners: Optional[List[Partitioner]] = None,
         edge_compile_config: Optional[EdgeCompileConfig] = None,
     ):
-        self.partitioners = partitioners or [default_partitioner_cls()]
+        self.partitioners = (
+            partitioners or [default_partitioner_cls()]
+            if default_partitioner_cls is not None
+            else []
+        )
         self.edge_compile_conf = edge_compile_config or EdgeCompileConfig()
         self.edge_dialect_program = None
 

backends/test/harness/tester.py

@@ -34,12 +34,12 @@ def __init__(
         self,
         module: torch.nn.Module,
         example_inputs: Tuple[torch.Tensor],
-        stage_classes: Dict[StageType, Callable],
+        stage_classes: Dict[StageType, Callable] | None = None,
         dynamic_shapes: Optional[Tuple[Any]] = None,
     ):
         module.eval()
 
-        self.stage_classes = stage_classes
+        self.stage_classes = stage_classes or Tester.default_stage_classes()
         self.original_module = module
         self.example_inputs = example_inputs
         self.dynamic_shapes = dynamic_shapes

backends/test/suite/flow.py

@@ -1,6 +1,6 @@
 import logging
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Callable
 
 from executorch.backends.test.harness import Tester
@@ -26,16 +26,25 @@ class TestFlow:
     tester_factory: Callable[..., Tester]
     """ A factory function that returns a Tester instance for this lowering flow. """
 
-    quantize: bool = field(default=False)
+    quantize: bool = False
     """ Whether to tester should run the quantize stage on the model. """
 
     quantize_stage_factory: Callable[..., Quantize] | None = None
     """ A factory function which instantiates a Quantize stage. Can be None to use the tester's default. """
 
+    is_delegated: bool = True
+    """ Indicates whether the flow is expected to generate CALL_DELEGATE nodes. """
+
 
 def all_flows() -> dict[str, TestFlow]:
     flows = []
 
+    from executorch.backends.test.suite.flows.portable import PORTABLE_TEST_FLOW
+
+    flows += [
+        PORTABLE_TEST_FLOW,
+    ]
+
     try:
         from executorch.backends.test.suite.flows.xnnpack import (
             XNNPACK_STATIC_INT8_PER_CHANNEL_TEST_FLOW,

backends/test/suite/flows/portable.py

@@ -0,0 +1,19 @@
+import logging
+
+from executorch.backends.test.harness import Tester
+from executorch.backends.test.suite.flow import TestFlow
+
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.INFO)
+
+
+def _create_portable_flow() -> TestFlow:
+    return TestFlow(
+        "portable",
+        backend="portable",
+        tester_factory=Tester,
+        is_delegated=False,
+    )
+
+
+PORTABLE_TEST_FLOW = _create_portable_flow()

backends/test/suite/runner.py

@@ -125,8 +125,8 @@ def build_result(
         if n.op == "call_function"
     )
 
-    # Only run the runtime portion if something was delegated.
-    if is_delegated:
+    # Only run the runtime portion if something was delegated (or the flow doesn't delegate).
+    if is_delegated or not flow.is_delegated:
         try:
             tester.to_executorch().serialize()
             extra_stats["pte_size_bytes"] = len(tester.get_artifact())

backends/vulkan/cmake/ShaderLibrary.cmake

@@ -49,6 +49,15 @@ function(gen_vulkan_shader_lib_cpp shaders_path)
   set(VULKAN_SHADERGEN_ENV "")
   set(VULKAN_SHADERGEN_OUT_PATH ${CMAKE_BINARY_DIR}/vulkan_compute_shaders)
 
+  set(GEN_SPV_ARGS "--optimize")
+  if(DEFINED ENV{ETVK_USING_SWIFTSHADER})
+    if("$ENV{ETVK_USING_SWIFTSHADER}" STREQUAL "1"
+       OR "$ENV{ETVK_USING_SWIFTSHADER}" STREQUAL "True"
+    )
+      list(APPEND GEN_SPV_ARGS "--replace-u16vecn")
+    endif()
+  endif()
+
   add_custom_command(
     COMMENT "Generating Vulkan Compute Shaders"
     OUTPUT ${VULKAN_SHADERGEN_OUT_PATH}/spv.cpp
@@ -58,7 +67,7 @@ function(gen_vulkan_shader_lib_cpp shaders_path)
       ${shaders_path} --output-path ${VULKAN_SHADERGEN_OUT_PATH}
       --glslc-path=${GLSLC_PATH}
       --tmp-dir-path=${VULKAN_SHADERGEN_OUT_PATH}/shader_cache/ --env
-      ${VULKAN_GEN_ARG_ENV} --optimize
+      ${VULKAN_GEN_ARG_ENV} ${GEN_SPV_ARGS}
     DEPENDS ${shaders_path}/*
             ${EXECUTORCH_ROOT}/backends/vulkan/runtime/gen_vulkan_spv.py
   )