Update

[ghstack-poisoned]

Author: GregoryComer

.ci/scripts/unittest-windows.ps1

@@ -24,9 +24,12 @@ if ($LASTEXITCODE -ne 0) {
     exit $LASTEXITCODE
 }
 
+# Install test dependencies
+pip install -r .ci/docker/requirements-ci.txt
+
 # Run pytest with coverage
 # pytest -n auto --cov=./ --cov-report=xml
-pytest --continue-on-collection-errors -vv --full-trace
+pytest --continue-on-collection-errors -vv --full-trace -c pytest-windows.ini
 if ($LASTEXITCODE -ne 0) {
     Write-Host "Pytest invocation was unsuccessful. Exit code: $LASTEXITCODE."
     exit $LASTEXITCODE

pytest-windows.ini

@@ -0,0 +1,116 @@
+# NOTE: This file is a copy of pytest.ini, but with additional tests disabled for Windows. This
+# is intended to be a short-term solution to allow for incrementally enabling tests on Windows.
+# This file is intended to be deleted once the enablement is complete.
+
+[pytest]
+addopts =
+    # show summary of all tests that did not pass
+    -rEfX
+    # Make tracebacks shorter
+    --tb=native
+    # capture only Python print and C++ py::print, but not C output (low-level Python errors)
+    --capture=sys
+    # don't suppress warnings, but don't shove them all to the end either
+    -p no:warnings
+    # Ignore backends/arm tests you need to run examples/arm/setup.sh to install some tool to make them work
+    # For GitHub testing this is setup/executed in the unittest-arm job see .github/workflows/pull.yml for more info.
+    --ignore-glob=backends/arm/**/*
+    # explicitly list out tests that are running successfully in oss
+    .ci/scripts/tests
+    examples/models/test
+    devtools/
+    --ignore=devtools/visualization/visualization_utils_test.py
+    # examples
+    # examples/models/llava/test TODO: enable this
+    # exir
+    exir/_serialize/test
+    exir/backend/test
+    exir/dialects/backend/test
+    exir/dialects/edge/test
+    exir/dialects/test
+    exir/emit/test
+    exir/program/test
+    exir/tests/
+    # executorch/export
+    export/tests
+    --ignore=export/tests/test_export_stages.py
+    # kernels/
+    kernels/prim_ops/test
+    kernels/quantized
+    # Because this test depends on test only cpp ops lib
+    # Will add test only cmake targets to re-enable this test
+    # but maybe it is a bit of anti-pattern
+    --ignore=kernels/quantized/test/test_quant_dequant_per_token.py
+    kernels/test/test_case_gen.py
+    # backends/test
+    # This effort is WIP and will be enabled in CI once testing infra
+    # is stable and signal to noise ratio is good (no irrelevant failures).
+    # See https://github.com/pytorch/executorch/discussions/11140
+    --ignore=backends/test
+    backends/test/harness/tests
+    backends/test/suite/tests
+    # backends/xnnpack
+    backends/xnnpack/test/ops
+    --ignore=backends/xnnpack/test/ops/test_bmm.py
+    --ignore=backends/xnnpack/test/ops/test_conv2d.py
+    --ignore=backends/xnnpack/test/ops/test_linear.py
+    --ignore=backends/xnnpack/test/ops/test_sdpa.py
+    backends/xnnpack/test/passes
+    backends/xnnpack/test/recipes
+    backends/xnnpack/test/serialization
+    # backends/apple/coreml
+    backends/apple/coreml/test
+    # extension/
+    extension/llm/custom_ops/test_sdpa_with_kv_cache.py
+    extension/llm/custom_ops/test_update_cache.py
+    extension/llm/custom_ops/test_quantized_sdpa.py
+    extension/pybindings/test
+    extension/training/pybindings/test
+    # Runtime
+    runtime
+    # Tools
+    codegen/test
+    tools/cmake
+    # test TODO: fix these tests
+    # test/end2end/test_end2end.py
+    --ignore=backends/xnnpack/test/ops/linear.py
+    --ignore=backends/xnnpack/test/models/llama2_et_example.py
+    # T200992559: Add torchao to ET as core dependency
+    --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py
+    --ignore=exir/backend/test/demos
+    --ignore=exir/backend/test/test_backends.py
+    --ignore=exir/backend/test/test_backends_lifted.py
+    --ignore=exir/backend/test/test_partitioner.py
+    --ignore=exir/tests/test_common.py
+    --ignore=exir/tests/test_memory_format_ops_pass_aten.py
+    --ignore=exir/tests/test_memory_planning.py
+    --ignore=exir/tests/test_op_convert.py
+    --ignore=exir/tests/test_passes.py
+    --ignore=exir/tests/test_quant_fusion_pass.py
+    --ignore=exir/tests/test_quantization.py
+    --ignore=exir/tests/test_verification.py
+
+    # Tests that are (temporarily) disabled for Windows
+    # TODO(gjcomer) Re-enable the LLM tests when tokenizers library is available on Windows.
+    #examples/models/llama3_2_vision/preprocess
+    #examples/models/llama3_2_vision/vision_encoder/test
+    #examples/models/llama3_2_vision/text_decoder/test
+    #examples/models/llama/tests
+    #examples/models/llama/config
+    #extension/llm/modules/test
+    #extension/llm/export
+    --ignore=extension/pybindings/test/test_pybindings.py::PybindingsTest::test_method_quantized_ops
+    --ignore=extension/pybindings/test/test_pybindings.py::PybindingsTest::test_quantized_ops
+    --ignore=runtime/test/test_runtime.py::RuntimeTest::test_load_program_with_path
+    --ignore=exir/backend/test/test_compatibility.py::TestCompatibility::test_compatibility_in_runtime
+    --ignore=exir/backend/test/test_compatibility.py::TestCompatibility::test_compatibility_in_runtime_edge_program_manager
+    --ignore=exir/backend/test/test_lowered_backend_module.py::TestBackendAPI::test_emit_lowered_backend_module_end_to_end
+    --ignore=exir/backend/test/test_to_backend_multi_method.py::TestToBackendMultiMethod::test_multi_method_end_to_end
+    --ignore=extension/llm/custom_ops/test_sdpa_with_kv_cache.py::SDPATestForSpeculativeDecode::test_sdpa_with_cache_seq_len_130
+    --ignore=devtools/inspector/tests/inspector_test.py::TestInspector::test_etrecord_populates_correct_edge_dialect_aot_intermediate_outputs
+    --ignore=devtools/inspector/tests/inspector_test.py::TestInspector::test_etrecord_populates_correct_export_program_aot_intermediate_outputs
+
+# run the same tests multiple times to determine their
+# flakiness status. Default to 50 re-runs
+flake-finder = true
+flake-runs = 50