SNOW-2304052 - Add the ability to filter out certain tests when running under hybrid mode (#3750)

SNOW-2304052 - Add the ability to filter out certain tests when running under hybrid mode
Not all tests pass under hybrid mode, largely because they were built for the snowflake backend specifically. Still we want to run a significant number of these with hybrid mode enabled to help triage and categorize potiential semantic issues.

This PR adds the ability to park specific tests (itegration only) with a pytest marker, @pytest.mark.no_hybrid for skipping. We also have a reference csv of a test run with hybrid which is used for skipping certain tests.

This allows us to guard against regressions, provide for incremental development of hybrid, and triage existing differences to determine where effort should be spent. 
Co-authored-by: Jonathan Shi <[email protected]>
Co-authored-by: Mahesh Vashishtha <[email protected]>

Author: 3 people

tests/integ/modin/conftest.py

@@ -2,6 +2,7 @@
 # Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
 
+import os
 import pathlib
 import re
 from datetime import datetime
@@ -47,6 +48,68 @@ def setup_modin_hybrid_mode(pytestconfig):
         MODIN_HYBRID_TEST_MODE_ENABLED = False
 
 
+def read_hybrid_known_failures():
+    """
+    Read `modin_hybrid_integ_results.csv` and create a pandas
+    dataframe filtered down to only the failed tests. You can regenerate
+    this file by:
+    * Collecting the hybrid test results with pytest:
+        pytest tests/integ/modin -n 10
+               --enable_modin_hybrid_mode
+               --csv tests/integ/modin/modin_hybrid_integ_results.csv
+    * (Recommended) Pre-Filtering the results to reduce the file size:
+      import pandas as pd
+      df = pd.read_csv("tests/integ/modin/modin_hybrid_integ_results.csv")
+      filtered = df[["module", "name", "message", "status"]][
+               df["status"].isin(["failed", "xfailed", "error"])
+            ]
+      filtered.to_csv("tests/integ/modin/modin_hybrid_integ_results.csv")
+    """
+    HYBRID_RESULTS_PATH = os.path.normpath(
+        os.path.join(
+            os.path.dirname(__file__), "../modin/modin_hybrid_integ_results.csv"
+        )
+    )
+    df = pandas.read_csv(HYBRID_RESULTS_PATH)
+    return df[["module", "name", "message", "status"]][
+        df["status"].isin(["failed", "xfailed", "error"])
+    ]
+
+
+HYBRID_KNOWN_FAILURES = read_hybrid_known_failures()
+
+
+def is_hybrid_known_failure(module_name, test_name) -> dict[bool, str]:
+    """
+    Determine whether the module/test is a known hybrid mode failure
+    and return the result along with the error message if applicable.
+    """
+    module_mask = HYBRID_KNOWN_FAILURES.module == module_name
+    testname_mask = HYBRID_KNOWN_FAILURES.name == test_name
+    test_data = HYBRID_KNOWN_FAILURES[module_mask & testname_mask]
+    failed = len(test_data) >= 1
+    msg = None
+    if failed:
+        msg = test_data["message"].iloc[0]
+    return (failed, msg)
+
+
+def pytest_runtest_setup(item):
+    """
+    pytest hook to filter out tests when running under hybrid mode
+    """
+    config = item.config
+    if not config.option.enable_modin_hybrid_mode:
+        return
+    # When a test is annotated with @pytest.mark.skip_hybrid it will be skipped
+    if len(list(item.iter_markers(name="skip_hybrid"))) > 0:
+        pytest.skip("Skipped for Hybrid: pytest.mark.skip_hybrid")
+    # Check the known failure list and skip those with a message
+    (failed, msg) = is_hybrid_known_failure(item.module.__name__, item.name)
+    if failed:
+        pytest.skip(f"Skipped for Hybrid: {msg}")
+
+
 @pytest.fixture(scope="module", autouse=True)
 def f(session):
     # create a snowpark pandas dataframe so that modin keeps an empty query compiler