From ce542cea1f8297144ce64914c13f4022fd9a0f13 Mon Sep 17 00:00:00 2001 From: Mergen Nachin Date: Sat, 6 Sep 2025 12:47:51 -0400 Subject: [PATCH] Try testing more pytest --- pytest.ini | 153 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 67 deletions(-) diff --git a/pytest.ini b/pytest.ini index aae87f242a7..3a97b72d504 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,90 +8,109 @@ addopts = --capture=sys # don't suppress warnings, but don't shove them all to the end either -p no:warnings - # Ignore backends/arm tests you need to run examples/arm/setup.sh to install some tool to make them work - # For GitHub testing this is setup/executed in the unittest-arm job see .github/workflows/pull.yml for more info. - --ignore-glob=backends/arm/**/* - # explicitly list out tests that are running successfully in oss + + # === TEST DIRECTORIES TO RUN === + + # ci/scripts .ci/scripts/tests - examples/models/test + + # backends + backends/apple/coreml/test + backends/test/harness/tests + backends/test/suite/tests + backends/transforms + backends/xnnpack/test + # Ignore ARM backend - requires running examples/arm/setup.sh to install tools + --ignore-glob=backends/arm/**/* + # Ignore failing xnnpack tests + --ignore=backends/xnnpack/test/ops/test_linear.py + --ignore=backends/xnnpack/test/ops/test_bmm.py + --ignore=backends/xnnpack/test/quantizer/test_xnnpack_quantizer.py + # Ignore backends/test root - WIP testing infra, see https://github.com/pytorch/executorch/discussions/11140 + --ignore=backends/test + + # codegen + codegen/test + + # devtools devtools/ + # Ignore test with missing dependencies --ignore=devtools/visualization/visualization_utils_test.py + # examples + examples/models/test examples/models/llama/tests examples/models/llama/config examples/models/llama3_2_vision/preprocess examples/models/llama3_2_vision/vision_encoder/test examples/models/llama3_2_vision/text_decoder/test - # examples/models/llava/test TODO: enable this + examples/models/llava/test + # Ignore test requiring torchao - T200992559: Add torchao to ET as core dependency + --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py + # Failing test + --ignore=examples/models/llama/tests/test_ring_attention.py + # Ignore test with missing dependencies + --ignore=examples/models/llava/test/test_pte.py + # Ignore failing llava tests (missing accelerate dependency) + --ignore=examples/models/llava/test/test_llava.py + # exir - exir/_serialize/test - exir/backend/test - exir/dialects/backend/test - exir/dialects/edge/test - exir/dialects/test - exir/emit/test - exir/program/test - exir/tests/ - # executorch/export + exir/ + # Ignore tests with missing custom_ops_generated_lib dependencies + --ignore=exir/backend/test/demos/test_delegate_aten_mode.py + --ignore=exir/backend/test/demos/test_xnnpack_qnnpack.py + --ignore=exir/tests/test_memory_format_ops_pass_aten.py + --ignore=exir/tests/test_memory_planning.py + --ignore=exir/tests/test_passes.py + --ignore=exir/tests/test_quantization.py + --ignore=exir/tests/test_verification.py + --ignore=exir/verification/test/test_verifier.py + # Ignore failing tests + --ignore=exir/backend/test/demos/rpc/test_rpc.py + --ignore=exir/backend/test/test_backends.py + --ignore=exir/backend/test/test_backends_lifted.py + --ignore=exir/backend/test/test_partitioner.py + --ignore=exir/operator/test/test_operator.py + --ignore=exir/tests/test_common.py + --ignore=exir/tests/test_op_convert.py + + # export export/tests --ignore=export/tests/test_export_stages.py - # kernels/ + + # extension + extension/ + # Ignore tests with missing dependencies or build issues + --ignore=extension/flat_tensor/test/test_serialize.py + --ignore=extension/llm/custom_ops/test_preprocess_custom_ops.py + --ignore=extension/llm/tokenizers/third-party/re2/python/re2_test.py + --ignore=extension/llm/tokenizers/third-party/sentencepiece/python/test/sentencepiece_test.py + # Ignore failing tokenizer tests + --ignore=extension/llm/tokenizers/test/test_tekken_python.py + + # kernels kernels/prim_ops/test kernels/quantized - # Because this test depends on test only cpp ops lib - # Will add test only cmake targets to re-enable this test - # but maybe it is a bit of anti-pattern - --ignore=kernels/quantized/test/test_quant_dequant_per_token.py kernels/test/test_case_gen.py - # backends/test - # This effort is WIP and will be enabled in CI once testing infra - # is stable and signal to noise ratio is good (no irrelevant failures). - # See https://github.com/pytorch/executorch/discussions/11140 - --ignore=backends/test - backends/test/harness/tests - backends/test/suite/tests - # backends/xnnpack - backends/xnnpack/test/ops - --ignore=backends/xnnpack/test/ops/test_bmm.py - --ignore=backends/xnnpack/test/ops/test_conv2d.py - --ignore=backends/xnnpack/test/ops/test_linear.py - --ignore=backends/xnnpack/test/ops/test_sdpa.py - backends/xnnpack/test/passes - backends/xnnpack/test/recipes - backends/xnnpack/test/serialization - # backends/apple/coreml - backends/apple/coreml/test - # extension/ - extension/llm/modules/test - extension/llm/export - extension/llm/custom_ops/test_sdpa_with_kv_cache.py - extension/llm/custom_ops/test_update_cache.py - extension/llm/custom_ops/test_quantized_sdpa.py - extension/pybindings/test - extension/training/pybindings/test - # Runtime + # Ignore test depending on test-only cpp ops lib + --ignore=kernels/quantized/test/test_quant_dequant_per_token.py + + # profiler + profiler/ + # Ignore test with missing dependencies + --ignore=profiler/test/test_profiler_e2e.py + + # runtime runtime - # Tools - codegen/test + + # test + test/ + # Ignore tests with missing dependencies + --ignore=test/end2end/test_end2end.py + --ignore=test/end2end/test_temp_allocator_fix.py + + # tools tools/cmake - # test TODO: fix these tests - # test/end2end/test_end2end.py - --ignore=backends/xnnpack/test/ops/linear.py - --ignore=backends/xnnpack/test/models/llama2_et_example.py - # T200992559: Add torchao to ET as core dependency - --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py - --ignore=exir/backend/test/demos - --ignore=exir/backend/test/test_backends.py - --ignore=exir/backend/test/test_backends_lifted.py - --ignore=exir/backend/test/test_partitioner.py - --ignore=exir/tests/test_common.py - --ignore=exir/tests/test_memory_format_ops_pass_aten.py - --ignore=exir/tests/test_memory_planning.py - --ignore=exir/tests/test_op_convert.py - --ignore=exir/tests/test_passes.py - --ignore=exir/tests/test_quant_fusion_pass.py - --ignore=exir/tests/test_quantization.py - --ignore=exir/tests/test_verification.py # run the same tests multiple times to determine their # flakiness status. Default to 50 re-runs