SNOW-1878542: Enable python 3.12 (#2852)

Author: sfc-gh-jrose

.github/workflows/daily_precommit.yml

@@ -98,7 +98,7 @@ jobs:
             download_name: macos
           - image_name: windows-latest-64-cores
             download_name: windows
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         cloud-provider: [aws, azure, gcp]
     steps:
       - name: Checkout Code
@@ -145,14 +145,25 @@ jobs:
           # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support
           # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
         shell: bash
-      - name: Run tests (excluding doctests)
+      - if: ${{ matrix.python-version != '3.12' }}
+        name: Run tests (excluding doctests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-dailynotdoctest-ci"
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
+      - if: ${{ matrix.python-version == '3.12' }}
+        name: Run tests (excluding doctests and udf tests)
+        run: python -m tox -e "py${PYTHON_VERSION/\./}-dailynotdoctestnotudf-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
+        shell: bash
       - name: Combine coverages
         run: python -m tox -e coverage --skip-missing-interpreters false
         shell: bash
@@ -217,7 +228,7 @@ jobs:
         os:
           - image_name: macos-latest
             download_name: macos  # it includes doctest
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         cloud-provider: [aws]
     steps:
       - name: Checkout Code
@@ -258,14 +269,25 @@ jobs:
           PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
-      - name: Run tests (excluding doctests)
+      - if: ${{ matrix.python-version != '3.12' }}
+        name: Run tests (excluding doctests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-dailynotdoctest-ci"
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
+      - if: ${{ matrix.python-version == '3.12' }}
+        name: Run tests (excluding doctests and udf tests)
+        run: python -m tox -e "py${PYTHON_VERSION/\./}-dailynotdoctestnotudf-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short --disable_sql_simplifier
+          TOX_PARALLEL_NO_SPINNER: 1
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
+        shell: bash
       - name: Combine coverages
         run: python -m tox -e coverage --skip-missing-interpreters false
         shell: bash
@@ -354,7 +376,7 @@ jobs:
         os:
           - image_name: macos-latest
             download_name: macos  # it includes doctest
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         cloud-provider: [aws]
     steps:
       - name: Checkout Code
@@ -484,7 +506,7 @@ jobs:
         os:
           - image_name: macos-latest
             download_name: macos  # it includes doctest
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         cloud-provider: [aws]
     steps:
       - name: Checkout Code
@@ -525,14 +547,23 @@ jobs:
           PYTEST_ADDOPTS: --color=yes --tb=short --disable_cte_optimization
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
-      - name: Run tests (excluding doctests)
+      - if: ${{ matrix.python-version != '3.12' }}
+        name: Run tests (excluding doctests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-dailynotdoctest-ci"
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short --disable_cte_optimization
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
+      - if: ${{ matrix.python-version == '3.12' }}
+        name: Run tests (excluding doctests and udf tests)
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short --disable_cte_optimization
+          TOX_PARALLEL_NO_SPINNER: 1
+        shell: bash
       - name: Combine coverages
         run: python -m tox -e coverage --skip-missing-interpreters false
         shell: bash

.github/workflows/precommit.yml

@@ -94,7 +94,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, windows-latest-64-cores, ubuntu-latest-64-cores]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
         cloud-provider: [aws, gcp, azure]
         exclude:
           # only run macos with aws py3.9 for doctest
@@ -169,19 +169,19 @@ jobs:
       - name: Install tox
         run: python -m pip install tox
       # we only run doctest on macos
-      - if: ${{ matrix.os == 'macos-latest' }}
+      - if: ${{ matrix.os == 'macos-latest' && matrix.python-version != '3.12'}}
         name: Run doctests
         run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
-          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.12 with no server-side support
+          # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.13 with no server-side support
           # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
         shell: bash
       # do not run other tests for macos
-      - if: ${{ matrix.os != 'macos-latest' }}
+      - if: ${{ matrix.os != 'macos-latest' && matrix.python-version != '3.12' }}
         name: Run tests (excluding doctests)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
         env:
@@ -190,6 +190,16 @@ jobs:
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
+      - if: ${{ matrix.python-version == '3.12' }}
+        name: Run tests (excluding doctests and udf tests)
+        run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short
+          TOX_PARALLEL_NO_SPINNER: 1
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
+        shell: bash
       - name: Combine coverages
         run: python -m tox -e coverage --skip-missing-interpreters false
         shell: bash
@@ -211,7 +221,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, windows-latest, ubuntu-latest]
-        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
         cloud-provider: [aws]
     steps:
       - name: Checkout Code
@@ -402,7 +412,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest ]
-        python-version: [ "3.9", "3.11" ]
+        python-version: [ "3.9", "3.11", "3.12" ]
         cloud-provider: [ aws ]
     steps:
       - name: Checkout Code
@@ -434,14 +444,25 @@ jobs:
         run: python -m pip install -U setuptools pip wheel
       - name: Install tox
         run: python -m pip install tox
-      - name: Run tests for AST (II/II)
+      - if: ${{ matrix.python-version != '3.12' }}
+        name: Run tests for AST (II/II)
         run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short --enable_ast
           TOX_PARALLEL_NO_SPINNER: 1
         shell: bash
+      - if: ${{ matrix.python-version == '3.12' }}
+        name: Run tests for AST (II/II) (excluding udf tests)
+        run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest-ci"
+        env:
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          cloud_provider: ${{ matrix.cloud-provider }}
+          PYTEST_ADDOPTS: --color=yes --tb=short --enable_ast
+          TOX_PARALLEL_NO_SPINNER: 1
+          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
+        shell: bash
       - name: Combine coverages
         run: python -m tox -e coverage --skip-missing-interpreters false
         shell: bash

setup.py

@@ -31,7 +31,7 @@
     "tzlocal",  # Snowpark IR
     "snowflake.core>=1.0.0, <2",  # Catalog
 ]
-REQUIRED_PYTHON_VERSION = ">=3.8, <3.12"
+REQUIRED_PYTHON_VERSION = ">=3.8, <3.13"
 
 if os.getenv("SNOWFLAKE_IS_PYTHON_RUNTIME_TEST", False):
     REQUIRED_PYTHON_VERSION = ">=3.8"
@@ -228,6 +228,7 @@ def run(self):
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Topic :: Database",
         "Topic :: Software Development",
         "Topic :: Software Development :: Libraries",

tests/integ/scala/test_async_job_suite.py

@@ -474,8 +474,8 @@ def test_async_job_to_df(session, create_async_job_from_query_id):
 
 
 def test_async_job_result_wait_no_result(session):
-    async_job = session.sql("select system$wait(3)").collect_nowait()
     t0 = time()
+    async_job = session.sql("select system$wait(3)").collect_nowait()
     result = async_job.result("no_result")
     t1 = time()
     assert t1 - t0 >= 3.0

tests/integ/test_dataframe.py

@@ -4272,9 +4272,14 @@ def test_create_empty_dataframe(session):
 def test_dataframe_to_local_iterator_with_to_pandas_isolation(
     session, local_testing_mode
 ):
-    df = session.create_dataframe(
-        [["xyz", int("1" * 19)] for _ in range(200000)], schema=["a1", "b1"]
-    )
+    if local_testing_mode:
+        df = session.create_dataframe(
+            [["xyz", int("1" * 19)] for _ in range(200000)], schema=["a1", "b1"]
+        )
+    else:
+        df = session.sql(
+            "select 'xyz' as A1, 1111111111111111111 as B1 from table(generator(rowCount => 200000))"
+        )
     trigger_df = session.create_dataframe(
         [[1.0]], schema=StructType([StructField("A", DecimalType())])
     )

tests/integ/test_packaging.py

@@ -599,6 +599,7 @@ def check_if_package_works() -> str:
     )
 
 
+@pytest.mark.udf
 @pytest.mark.skipif(
     IS_IN_STORED_PROC,
     reason="Subprocess calls are not allowed within stored procedures.",
@@ -609,7 +610,7 @@ def test_add_packages_with_native_dependency_without_force_push(session):
         with pytest.raises(
             RuntimeError, match="Your code depends on packages that contain native code"
         ):
-            session.add_packages(["catboost==1.2"])
+            session.add_packages(["catboost==1.2.3"])
 
 
 @pytest.fixture(scope="function")
@@ -625,7 +626,7 @@ def requirements_file_with_local_path():
 
     # Generate a requirements file
     requirements = f"""
-    pyyaml==6.0
+    pyyaml==6.0.2
     matplotlib
     {new_path}
     """
@@ -657,7 +658,7 @@ def test_add_requirements_with_local_filepath(
     session.add_requirements(requirements_file_with_local_path)
     assert session.get_packages() == {
         "matplotlib": "matplotlib",
-        "pyyaml": "pyyaml==6.0",
+        "pyyaml": "pyyaml==6.0.2",
     }
 
     udf_name = Utils.random_name_for_temp_object(TempObjectType.FUNCTION)

tox.ini

@@ -101,6 +101,7 @@ commands =
     notudfdoctest: {env:SNOWFLAKE_PYTEST_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} tests
     local: {env:SNOWFLAKE_PYTEST_CMD} --local_testing_mode -m "integ or unit or mock" {posargs:} tests
     dailynotdoctest: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} or udf" {posargs:} tests
+    dailynotdoctestnotudf: {env:SNOWFLAKE_PYTEST_DAILY_CMD} -m "{env:SNOWFLAKE_TEST_TYPE} and not udf" {posargs:} tests
     # Snowpark pandas commands:
     snowparkpandasnotdoctest: {env:MODIN_PYTEST_CMD} --durations=20 -m "{env:SNOWFLAKE_TEST_TYPE}" {posargs:} {env:SNOW_1314507_WORKAROUND_RERUN_FLAGS} tests/unit/modin tests/integ/modin tests/integ/test_df_to_snowpark_pandas.py
     # This one only run doctest but we still need to include the tests folder to let tests/conftest.py to mark the doctest files for us
@@ -143,7 +144,7 @@ commands = coverage combine
            coverage report -m
            coverage xml -o {env:COV_REPORT_DIR:{toxworkdir}}/coverage.xml
            coverage html -d {env:COV_REPORT_DIR:{toxworkdir}}/htmlcov --show-contexts
-depends = py39, py310, py311
+depends = py39, py310, py311, py312
 
 [testenv:docs]
 basepython = python3.9