Try testing more pytest

pytest.ini

@@ -8,90 +8,109 @@ addopts =
     --capture=sys
     # don't suppress warnings, but don't shove them all to the end either
     -p no:warnings
-    # Ignore backends/arm tests you need to run examples/arm/setup.sh to install some tool to make them work
-    # For GitHub testing this is setup/executed in the unittest-arm job see .github/workflows/pull.yml for more info.
-    --ignore-glob=backends/arm/**/*
-    # explicitly list out tests that are running successfully in oss
+
+    # === TEST DIRECTORIES TO RUN ===
+
+    # ci/scripts
     .ci/scripts/tests
-    examples/models/test
+
+    # backends
+    backends/apple/coreml/test
+    backends/test/harness/tests
+    backends/test/suite/tests
+    backends/transforms
+    backends/xnnpack/test
+    # Ignore ARM backend - requires running examples/arm/setup.sh to install tools
+    --ignore-glob=backends/arm/**/*
+    # Ignore failing xnnpack tests
+    --ignore=backends/xnnpack/test/ops/test_linear.py
+    --ignore=backends/xnnpack/test/ops/test_bmm.py
+    --ignore=backends/xnnpack/test/quantizer/test_xnnpack_quantizer.py
+    # Ignore backends/test root - WIP testing infra, see https://github.com/pytorch/executorch/discussions/11140
+    --ignore=backends/test
+
+    # codegen
+    codegen/test
+
+    # devtools
     devtools/
+    # Ignore test with missing dependencies
     --ignore=devtools/visualization/visualization_utils_test.py
+
     # examples
+    examples/models/test
     examples/models/llama/tests
     examples/models/llama/config
     examples/models/llama3_2_vision/preprocess
     examples/models/llama3_2_vision/vision_encoder/test
     examples/models/llama3_2_vision/text_decoder/test
-    # examples/models/llava/test TODO: enable this
+    examples/models/llava/test
+    # Ignore test requiring torchao - T200992559: Add torchao to ET as core dependency
+    --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py
+    # Failing test
+    --ignore=examples/models/llama/tests/test_ring_attention.py
+    # Ignore test with missing dependencies
+    --ignore=examples/models/llava/test/test_pte.py
+    # Ignore failing llava tests (missing accelerate dependency)
+    --ignore=examples/models/llava/test/test_llava.py
+
     # exir
-    exir/_serialize/test
-    exir/backend/test
-    exir/dialects/backend/test
-    exir/dialects/edge/test
-    exir/dialects/test
-    exir/emit/test
-    exir/program/test
-    exir/tests/
-    # executorch/export
+    exir/
+    # Ignore tests with missing custom_ops_generated_lib dependencies
+    --ignore=exir/backend/test/demos/test_delegate_aten_mode.py
+    --ignore=exir/backend/test/demos/test_xnnpack_qnnpack.py
+    --ignore=exir/tests/test_memory_format_ops_pass_aten.py
+    --ignore=exir/tests/test_memory_planning.py
+    --ignore=exir/tests/test_passes.py
+    --ignore=exir/tests/test_quantization.py
+    --ignore=exir/tests/test_verification.py
+    --ignore=exir/verification/test/test_verifier.py
+    # Ignore failing tests
+    --ignore=exir/backend/test/demos/rpc/test_rpc.py
+    --ignore=exir/backend/test/test_backends.py
+    --ignore=exir/backend/test/test_backends_lifted.py
+    --ignore=exir/backend/test/test_partitioner.py
+    --ignore=exir/operator/test/test_operator.py
+    --ignore=exir/tests/test_common.py
+    --ignore=exir/tests/test_op_convert.py
+
+    # export
     export/tests
     --ignore=export/tests/test_export_stages.py
-    # kernels/
+
+    # extension
+    extension/
+    # Ignore tests with missing dependencies or build issues
+    --ignore=extension/flat_tensor/test/test_serialize.py
+    --ignore=extension/llm/custom_ops/test_preprocess_custom_ops.py
+    --ignore=extension/llm/tokenizers/third-party/re2/python/re2_test.py
+    --ignore=extension/llm/tokenizers/third-party/sentencepiece/python/test/sentencepiece_test.py
+    # Ignore failing tokenizer tests
+    --ignore=extension/llm/tokenizers/test/test_tekken_python.py
+
+    # kernels
     kernels/prim_ops/test
     kernels/quantized
-    # Because this test depends on test only cpp ops lib
-    # Will add test only cmake targets to re-enable this test
-    # but maybe it is a bit of anti-pattern
-    --ignore=kernels/quantized/test/test_quant_dequant_per_token.py
     kernels/test/test_case_gen.py
-    # backends/test
-    # This effort is WIP and will be enabled in CI once testing infra
-    # is stable and signal to noise ratio is good (no irrelevant failures).
-    # See https://github.com/pytorch/executorch/discussions/11140
-    --ignore=backends/test
-    backends/test/harness/tests
-    backends/test/suite/tests
-    # backends/xnnpack
-    backends/xnnpack/test/ops
-    --ignore=backends/xnnpack/test/ops/test_bmm.py
-    --ignore=backends/xnnpack/test/ops/test_conv2d.py
-    --ignore=backends/xnnpack/test/ops/test_linear.py
-    --ignore=backends/xnnpack/test/ops/test_sdpa.py
-    backends/xnnpack/test/passes
-    backends/xnnpack/test/recipes
-    backends/xnnpack/test/serialization
-    # backends/apple/coreml
-    backends/apple/coreml/test
-    # extension/
-    extension/llm/modules/test
-    extension/llm/export
-    extension/llm/custom_ops/test_sdpa_with_kv_cache.py
-    extension/llm/custom_ops/test_update_cache.py
-    extension/llm/custom_ops/test_quantized_sdpa.py
-    extension/pybindings/test
-    extension/training/pybindings/test
-    # Runtime
+    # Ignore test depending on test-only cpp ops lib
+    --ignore=kernels/quantized/test/test_quant_dequant_per_token.py
+
+    # profiler
+    profiler/
+    # Ignore test with missing dependencies
+    --ignore=profiler/test/test_profiler_e2e.py
+
+    # runtime
     runtime
-    # Tools
-    codegen/test
+
+    # test
+    test/
+    # Ignore tests with missing dependencies
+    --ignore=test/end2end/test_end2end.py
+    --ignore=test/end2end/test_temp_allocator_fix.py
+
+    # tools
     tools/cmake
-    # test TODO: fix these tests
-    # test/end2end/test_end2end.py
-    --ignore=backends/xnnpack/test/ops/linear.py
-    --ignore=backends/xnnpack/test/models/llama2_et_example.py
-    # T200992559: Add torchao to ET as core dependency
-    --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py
-    --ignore=exir/backend/test/demos
-    --ignore=exir/backend/test/test_backends.py
-    --ignore=exir/backend/test/test_backends_lifted.py
-    --ignore=exir/backend/test/test_partitioner.py
-    --ignore=exir/tests/test_common.py
-    --ignore=exir/tests/test_memory_format_ops_pass_aten.py
-    --ignore=exir/tests/test_memory_planning.py
-    --ignore=exir/tests/test_op_convert.py
-    --ignore=exir/tests/test_passes.py
-    --ignore=exir/tests/test_quant_fusion_pass.py
-    --ignore=exir/tests/test_quantization.py
-    --ignore=exir/tests/test_verification.py
 
 # run the same tests multiple times to determine their
 # flakiness status. Default to 50 re-runs