SNOW-2389912: Ensure hybrid execution is actually enabled by default (#3841)

Author: sfc-gh-joshi

.github/workflows/precommit.yml

@@ -575,6 +575,16 @@ jobs:
         run: uv pip install -U setuptools pip wheel --system
       - name: Install tox
         run: uv pip install tox --system
+      - if: ${{ matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws' }}
+        name: Check AutoSwitchBackend default value
+        # SNOW-2389912: Ensure hybrid is correctly enabled by default. We can't use pytest because
+        # our test suite sets the AutoSwitchBackend variable explicitly.
+        run: |
+          python -m tox -e "py${PYTHON_VERSION}-snowparkpandashybridcheck_sessionfirst-modin-ci" \
+          && python -m tox -e "py${PYTHON_VERSION}-snowparkpandashybridcheck_pluginfirst-modin-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
         # only run doctest for macos on aws
       - if: ${{ matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws' }}
         name: Run Snowpark pandas API doctests

src/snowflake/snowpark/session.py

@@ -759,13 +759,16 @@ def __init__(
             try:
                 from modin.config import AutoSwitchBackend
 
-                pandas_hybrid_execution_enabled: bool = (
-                    self._conn._get_client_side_session_parameter(
-                        _SNOWPARK_PANDAS_HYBRID_EXECUTION_ENABLED,
-                        AutoSwitchBackend().get(),
-                    )
+                pandas_hybrid_execution_enabled: Union[
+                    bool, None
+                ] = self._conn._get_client_side_session_parameter(
+                    _SNOWPARK_PANDAS_HYBRID_EXECUTION_ENABLED, None
                 )
-                AutoSwitchBackend.put(pandas_hybrid_execution_enabled)
+                # Only set AutoSwitchBackend if the session parameter was already set.
+                # snowflake.snowpark.modin.plugin sets AutoSwitchBackend to True if it was
+                # not already set, so we should not change the variable if it's in its default state.
+                if pandas_hybrid_execution_enabled is not None:
+                    AutoSwitchBackend.put(pandas_hybrid_execution_enabled)
             except Exception:
                 # Continue session initialization even if Modin configuration fails
                 pass

tox.ini

@@ -135,6 +135,9 @@ commands =
     # This one is only called by jenkins job and the only difference from `snowparkpandasnotdoctest` is that it uses
     # MODIN_PYTEST_NO_COV_CMD instead of MODIN_PYTEST_CMD
     snowparkpandasjenkins: {env:MODIN_PYTEST_NO_COV_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} tests/unit/modin tests/integ/modin
+    # These jobs run outside pytest since we manipulate AutoSwitchBackend during pytest setup and we want to test a fresh session.
+    snowparkpandashybridcheck_sessionfirst: python -c "from tests.parameters import CONNECTION_PARAMETERS; from snowflake.snowpark.session import Session; Session.builder.configs(CONNECTION_PARAMETERS).create(); import snowflake.snowpark.modin.plugin; from modin.config import AutoSwitchBackend; assert AutoSwitchBackend.get() is True"
+    snowparkpandashybridcheck_pluginfirst: python -c "from tests.parameters import CONNECTION_PARAMETERS; import snowflake.snowpark.modin.plugin; from snowflake.snowpark.session import Session; Session.builder.configs(CONNECTION_PARAMETERS).create(); from modin.config import AutoSwitchBackend; assert AutoSwitchBackend.get() is True"
     # Snowpark IR commands:
     ast: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} tests