Merge remote-tracking branch 'origin/main' into SNOW-1800374

Author: sfc-gh-batur

.github/workflows/daily_precommit.yml

@@ -546,75 +546,6 @@ jobs:
             .tox/.coverage
             .tox/coverage.xml
 
-  test-snowpark-disable-multithreading-mode:
-    name: Test Snowpark Multithreading Disabled py-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
-    needs: build
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest-64-cores]
-        python-version: ["3.9"]
-        cloud-provider: [aws]
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Display Python version
-        run: python -c "import sys; print(sys.version)"
-      - name: Decrypt parameters.py
-        shell: bash
-        run: .github/scripts/decrypt_parameters.sh
-        env:
-          PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
-          CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
-      - name: Install protoc
-        shell: bash
-        run: .github/scripts/install_protoc.sh
-      - name: Download wheel(s)
-        uses: actions/download-artifact@v4
-        with:
-          name: wheel
-          path: dist
-      - name: Show wheels downloaded
-        run: ls -lh dist
-        shell: bash
-      - name: Upgrade setuptools, pip and wheel
-        run: python -m pip install -U setuptools pip wheel
-      - name: Install tox
-        run: python -m pip install tox
-      - name: Run tests (excluding doctests)
-        run: python -m tox -e "py${PYTHON_VERSION/\./}-notmultithreaded-ci"
-        env:
-          PYTHON_VERSION: ${{ matrix.python-version }}
-          cloud_provider: ${{ matrix.cloud-provider }}
-          PYTEST_ADDOPTS: --color=yes --tb=short
-          TOX_PARALLEL_NO_SPINNER: 1
-        shell: bash
-      - name: Run local tests
-        run: python -m tox -e "py${PYTHON_VERSION/\./}-localnotmultithreaded-ci"
-        env:
-          PYTHON_VERSION: ${{ matrix.python-version }}
-          cloud_provider: ${{ matrix.cloud-provider }}
-          PYTEST_ADDOPTS: --color=yes --tb=short
-          TOX_PARALLEL_NO_SPINNER: 1
-        shell: bash
-      - name: Combine coverages
-        run: python -m tox -e coverage --skip-missing-interpreters false
-        shell: bash
-        env:
-          SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
-      - uses: actions/upload-artifact@v4
-        with:
-          include-hidden-files: true
-          name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-snowpark-multithreading
-          path: |
-            .tox/.coverage
-            .tox/coverage.xml
-
   combine-coverage:
     if: ${{ success() || failure() }}
     name: Combine coverage

CHANGELOG.md

@@ -14,16 +14,46 @@
   - `map_keys`
   - `nullifzero`
   - `snowflake_cortex_sentiment`
+  - `acosh`
+  - `asinh`
+  - `atanh`
+  - `bit_length`
+  - `bitmap_bit_position`
+  - `bitmap_bucket_number`
+  - `bitmap_construct_agg`
+  - `cbrt`
+  - `equal_null`
+  - `ifnull`
+  - `localtimestamp`
+  - `max_by`
+  - `min_by`
+  - `nvl`
+  - `octet_length`
+  - `position`
+  - `regr_avgx`
+  - `regr_avgy`
+  - `regr_count`
+  - `regr_intercept`
+  - `regr_r2`
+  - `regr_slope`
+  - `regr_sxx`
+  - `regr_sxy`
+  - `regr_syy`
+  - `try_to_binary`
+
 - Added `Catalog` class to manage snowflake objects. It can be accessed via `Session.catalog`.
 
 #### Improvements
 
 - Updated README.md to include instructions on how to verify package signatures using `cosign`.
+- Added an option `keep_column_order` for keeping original column order in `DataFrame.with_column` and `DataFrame.with_columns`.
 
 #### Bug Fixes
 
 - Fixed a bug in local testing mode that caused a column to contain None when it should contain 0
-- Fixed a bug in StructField.from_json that prevented TimestampTypes with tzinfo from being parsed correctly.
+- Fixed a bug in `StructField.from_json` that prevented TimestampTypes with `tzinfo` from being parsed correctly.
+- Fixed a bug in function `date_format` that caused an error when the input column was date type or timestamp type.
+- Fixed a bug in `replace` when passing `Column` expression objects.
 
 ### Snowpark pandas API Updates
 
@@ -49,6 +79,7 @@
   - %X: Locale’s appropriate time representation.
   - %%: A literal '%' character.
 - Added support for `Series.between`.
+- Added support for `include_groups=False` in `DataFrameGroupBy.apply`.
 
 #### Bug Fixes
 
@@ -59,8 +90,9 @@
 - Updated integration testing for `session.lineage.trace` to exclude deleted objects
 - Added documentation for `DataFrame.map`.
 - Improve performance of `DataFrame.apply` by mapping numpy functions to snowpark functions if possible.
-- Added documentation on the extent of Snowpark pandas interoperability with scikit-learn
+- Added documentation on the extent of Snowpark pandas interoperability with scikit-learn.
 - Infer return type of functions in `Series.map`, `Series.apply` and `DataFrame.map` if type-hint is not provided.
+- Added `call_count` to telemetry that counts method calls including interchange protocol calls.
 
 ## 1.26.0 (2024-12-05)
 

docs/source/modin/supported/groupby_supported.rst

@@ -39,8 +39,8 @@ Function application
 +-----------------------------+---------------------------------+----------------------------------+----------------------------------------------------+
 | ``apply``                   | P                               | ``axis`` other than 0 is not     | ``Y`` if the following are true, otherwise ``N``:  |
 |                             |                                 | implemented.                     |   - ``func`` is a callable that always returns     |
-|                             |                                 | ``include_groups = False`` is    |     either a pandas DataFrame, a pandas Series, or |
-|                             |                                 | not implemented.                 |     objects that are neither DataFrame nor Series. |
+|                             |                                 |                                  |     either a pandas DataFrame, a pandas Series, or |
+|                             |                                 |                                  |     objects that are neither DataFrame nor Series. |
 |                             |                                 |                                  |   - grouping on axis=0                             |
 |                             |                                 |                                  |   - Not applying transform to a dataframe with a   |
 |                             |                                 |                                  |     non-unique index                               |

docs/source/snowpark/functions.rst

@@ -20,6 +20,7 @@ Functions
 
     abs
     acos
+    acosh
     add_months
     any_value
     approx_count_distinct
@@ -75,9 +76,15 @@ Functions
     asc_nulls_last
     ascii
     asin
+    asinh
     atan
+    atanh
     atan2
     avg
+    bit_length
+    bitmap_bit_position
+    bitmap_bucket_number
+    bitmap_construct_agg
     bitnot
     bitshiftleft
     bitshiftright
@@ -88,6 +95,7 @@ Functions
     call_table_function
     call_udf
     cast
+    cbrt
     ceil
     char
     charindex
@@ -151,6 +159,7 @@ Functions
     divnull
     endswith
     equal_nan
+    equal_null
     exp
     explode
     explode_outer
@@ -171,6 +180,7 @@ Functions
     hash
     hour
     iff
+    ifnull
     in_
     initcap
     insert
@@ -205,6 +215,7 @@ Functions
     lit
     ln
     locate
+    localtimestamp
     log
     lower
     lpad
@@ -215,10 +226,12 @@ Functions
     map_contains_key
     map_keys
     max
+    max_by
     md5
     mean
     median
     min
+    min_by
     minute
     mode
     monotonically_increasing_id
@@ -230,19 +243,22 @@ Functions
     not_
     ntile
     nullifzero
+    nvl
     object_agg
     object_construct
     object_construct_keep_null
     object_delete
     object_insert
     object_keys
     object_pick
+    octet_length
     pandas_udf
     pandas_udtf
     parse_json
     parse_xml
     percent_rank
     percentile_cont
+    position
     pow
     previous_day
     quarter
@@ -252,6 +268,15 @@ Functions
     regexp_count
     regexp_extract
     regexp_replace
+    regr_avgx
+    regr_avgy
+    regr_count
+    regr_intercept
+    regr_r2
+    regr_slope
+    regr_sxx
+    regr_sxy
+    regr_syy
     repeat
     replace
     right
@@ -320,6 +345,7 @@ Functions
     trim
     trunc
     try_cast
+    try_to_binary
     typeof
     udaf
     udf

license_header.txt

@@ -1,3 +1,3 @@
 
-Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
+Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 

pyproject.toml

@@ -0,0 +1,7 @@
+[build-system]
+requires = [
+    "setuptools",
+    "protoc-wheel-0==21.1", # Protocol buffer compiler for Snowpark IR
+    "mypy-protobuf", # used in generating typed Python code from protobuf for Snowpark IR
+]
+build-backend = "setuptools.build_meta"

setup.py

@@ -58,9 +58,8 @@
     "graphviz",  # used in plot tests
     "pytest-assume",  # sql counter check
     "decorator",  # sql counter check
-    "protoc-wheel-0==21.1",  # Protocol buffer compiler, for Snowpark IR
-    "mypy-protobuf",  # used in generating typed Python code from protobuf for Snowpark IR
     "lxml",  # used in read_xml tests
+    "tox",  # used for setting up testing environments
 ]
 
 # read the version

src/conftest.py

@@ -41,6 +41,27 @@ def pytest_runtest_makereport(item, call):
     return tr
 
 
+# These tests require python packages that are no longer built for python 3.8
+PYTHON_38_SKIPS = {
+    "snowpark.session.Session.replicate_local_environment",
+    "snowpark.session.Session.table_function",
+}
+
+DocTestFinder = doctest.DocTestFinder
+
+
+class CustomDocTestFinder(DocTestFinder):
+    def _find(self, tests, obj, name, module, source_lines, globs, seen):
+        if name in PYTHON_38_SKIPS and sys.version_info < (3, 9):
+            return
+        return DocTestFinder._find(
+            self, tests, obj, name, module, source_lines, globs, seen
+        )
+
+
+doctest.DocTestFinder = CustomDocTestFinder
+
+
 # scope is module so that we ensure we delete the session before
 # moving onto running the tests in the tests dir. Having only one
 # session is important to certain UDF tests to pass , since they

src/snowflake/snowpark/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 #
-# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
+# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #
 
 """

src/snowflake/snowpark/_internal/__init__.py

@@ -1,3 +1,3 @@
 #
-# Copyright (c) 2012-2024 Snowflake Computing Inc. All rights reserved.
+# Copyright (c) 2012-2025 Snowflake Computing Inc. All rights reserved.
 #