Skip to content

Commit 7f58f3c

Browse files
sfc-gh-aalamsfc-gh-aalam
authored
Merge branch 'main' into release-v1.29.0
2 parents 80a7ae7 + 015dc94 commit 7f58f3c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+6345
-684
lines changed

‎.github/workflows/daily_precommit.yml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -576,6 +576,98 @@ jobs:
576576
.tox/.coverage
577577
.tox/coverage.xml
578578
579+
test-enable-fix-join-alias:
580+
name: Test Fixing Join Alias py-${{ matrix.os }}-${{ matrix.python-version }}
581+
needs: build
582+
runs-on: ${{ matrix.os }}
583+
strategy:
584+
fail-fast: false
585+
matrix:
586+
include:
587+
- os: macos-latest
588+
python-version: "3.9"
589+
cloud-provider: azure
590+
- os: ubuntu-latest
591+
python-version: "3.12"
592+
cloud-provider: aws
593+
steps:
594+
- name: Checkout Code
595+
uses: actions/checkout@v4
596+
- name: Set up Python
597+
uses: actions/setup-python@v4
598+
with:
599+
python-version: ${{ matrix.python-version }}
600+
- name: Display Python version
601+
run: python -c "import sys; print(sys.version)"
602+
- name: Decrypt parameters.py
603+
shell: bash
604+
run: .github/scripts/decrypt_parameters.sh
605+
env:
606+
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
607+
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
608+
- name: Install protoc
609+
shell: bash
610+
run: .github/scripts/install_protoc.sh
611+
- name: Add protoc to Windows path
612+
if: runner.os == 'Windows'
613+
run: |
614+
echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
615+
- name: Download wheel(s)
616+
uses: actions/download-artifact@v4
617+
with:
618+
name: wheel
619+
path: dist
620+
- name: Show wheels downloaded
621+
run: ls -lh dist
622+
shell: bash
623+
- name: Upgrade setuptools, pip and wheel
624+
run: python -m pip install -U setuptools pip wheel
625+
- name: Install tox
626+
run: python -m pip install tox
627+
# we only run doctest on macos
628+
- if: ${{ matrix.os == 'macos-latest' && matrix.python-version != '3.12'}}
629+
name: Run doctests
630+
run: python -m tox -e "py${PYTHON_VERSION}-doctest-notudf-ci"
631+
env:
632+
PYTHON_VERSION: ${{ matrix.python-version }}
633+
cloud_provider: ${{ matrix.cloud-provider }}
634+
PYTEST_ADDOPTS: --color=yes --tb=short --join_alias_fix
635+
TOX_PARALLEL_NO_SPINNER: 1
636+
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.13 with no server-side support
637+
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
638+
shell: bash
639+
# do not run other tests for macos
640+
- if: ${{ matrix.os != 'macos-latest' && matrix.python-version != '3.12' }}
641+
name: Run tests (excluding doctests)
642+
run: python -m tox -e "py${PYTHON_VERSION/\./}-notdoctest-ci"
643+
env:
644+
PYTHON_VERSION: ${{ matrix.python-version }}
645+
cloud_provider: ${{ matrix.cloud-provider }}
646+
PYTEST_ADDOPTS: --color=yes --tb=short --join_alias_fix
647+
TOX_PARALLEL_NO_SPINNER: 1
648+
shell: bash
649+
- if: ${{ matrix.python-version == '3.12' }}
650+
name: Run tests (excluding doctests and udf tests)
651+
run: python -m tox -e "py${PYTHON_VERSION/\./}-notudfdoctest-ci"
652+
env:
653+
PYTHON_VERSION: ${{ matrix.python-version }}
654+
cloud_provider: ${{ matrix.cloud-provider }}
655+
PYTEST_ADDOPTS: --color=yes --tb=short --join_alias_fix
656+
TOX_PARALLEL_NO_SPINNER: 1
657+
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
658+
shell: bash
659+
- name: Combine coverages
660+
run: python -m tox -e coverage --skip-missing-interpreters false
661+
shell: bash
662+
env:
663+
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
664+
- uses: actions/upload-artifact@v4
665+
with:
666+
include-hidden-files: true
667+
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
668+
path: |
669+
.tox/.coverage
670+
.tox/coverage.xml
579671
combine-coverage:
580672
if: ${{ success() || failure() }}
581673
name: Combine coverage

‎.github/workflows/precommit.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,89 @@ jobs:
537537
.tox/.coverage
538538
.tox/coverage.xml
539539
540+
test-enable-join-fix-snowpark-pandas:
541+
name: Test modin-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
542+
needs: build
543+
runs-on: ${{ matrix.os }}
544+
strategy:
545+
fail-fast: false
546+
matrix:
547+
include:
548+
- os: windows-latest-64-cores
549+
python-version: "3.10"
550+
cloud-provider: gcp
551+
- os: ubuntu-latest-64-cores
552+
python-version: "3.9"
553+
cloud-provider: aws
554+
steps:
555+
- name: Checkout Code
556+
uses: actions/checkout@v4
557+
- name: Set up Python
558+
uses: actions/setup-python@v4
559+
with:
560+
python-version: ${{ matrix.python-version }}
561+
- name: Display Python version
562+
run: python -c "import sys; print(sys.version)"
563+
- name: Decrypt parameters.py
564+
shell: bash
565+
run: .github/scripts/decrypt_parameters.sh
566+
env:
567+
PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
568+
CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
569+
- name: Install protoc
570+
shell: bash
571+
run: .github/scripts/install_protoc.sh
572+
- name: Add protoc to Windows path
573+
if: runner.os == 'Windows'
574+
run: |
575+
echo "$HOME/local/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
576+
- name: Download wheel(s)
577+
uses: actions/download-artifact@v4
578+
with:
579+
name: wheel
580+
path: dist
581+
- name: Show wheels downloaded
582+
run: ls -lh dist
583+
shell: bash
584+
- name: Upgrade setuptools, pip and wheel
585+
run: python -m pip install -U setuptools pip wheel
586+
- name: Install tox
587+
run: python -m pip install tox
588+
# only run doctest for macos on aws
589+
- if: ${{ matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws' }}
590+
name: Run Snowpark pandas API doctests
591+
run: python -m tox -e "py${PYTHON_VERSION}-doctest-snowparkpandasdoctest-modin-ci"
592+
env:
593+
PYTHON_VERSION: ${{ matrix.python-version }}
594+
cloud_provider: ${{ matrix.cloud-provider }}
595+
PYTEST_ADDOPTS: --color=yes --tb=short --join_alias_fix
596+
TOX_PARALLEL_NO_SPINNER: 1
597+
# Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
598+
# For example, see https://github.com/snowflakedb/snowpark-python/pull/681
599+
shell: bash
600+
# do not run other tests for macos on aws
601+
- if: ${{ !(matrix.os == 'macos-latest' && matrix.cloud-provider == 'aws') }}
602+
name: Run Snowpark pandas API tests (excluding doctests)
603+
run: python -m tox -e "py${PYTHON_VERSION/\./}-snowparkpandasnotdoctest-modin-ci"
604+
env:
605+
PYTHON_VERSION: ${{ matrix.python-version }}
606+
cloud_provider: ${{ matrix.cloud-provider }}
607+
PYTEST_ADDOPTS: --color=yes --tb=short --join_alias_fix
608+
TOX_PARALLEL_NO_SPINNER: 1
609+
shell: bash
610+
- name: Combine coverages
611+
run: python -m tox -e coverage --skip-missing-interpreters false
612+
shell: bash
613+
env:
614+
SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1
615+
- uses: actions/upload-artifact@v4
616+
with:
617+
include-hidden-files: true
618+
name: coverage_${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}-snowpark-pandas-testing
619+
path: |
620+
.tox/.coverage
621+
.tox/coverage.xml
622+
540623
test-snowpark-pandas:
541624
name: Test modin-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
542625
needs: build

‎CHANGELOG.md‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,32 @@
66

77
#### New Features
88

9-
- Added support for the following AI-powered functions in `functions.py`
9+
- Added support for the following AI-powered functions in `functions.py` (private preview):
1010
- `ai_filter`
1111
- `ai_agg`
1212
- `summarize_agg`
13+
- Added support for the new FILE SQL type support, with the following related functions in `functions.py` (private preview):
14+
- fl_get_content_type
15+
- fl_get_etag
16+
- fl_get_file_type
17+
- fl_get_last_modified
18+
- fl_get_relative_path
19+
- fl_get_scoped_file_url
20+
- fl_get_size
21+
- fl_get_stage
22+
- fl_get_stage_file_url
23+
- fl_is_audio
24+
- fl_is_compressed
25+
- fl_is_document
26+
- fl_is_image
27+
- fl_is_video
1328

1429
#### Bug Fixes
1530

1631
- Fixed a bug where creating a Dataframe with large number of values raised `Unsupported feature 'SCOPED_TEMPORARY'.` error if thread-safe session was disabled.
1732
- Fixed a bug where `df.describe` raised internal SQL execution error when the dataframe is created from reading a stage file and CTE optimization is enabled.
33+
- Fixed a bug where `df.order_by(A).select(B).distinct()` would generate invalid SQL when simplified query generation was enabled using `session.conf.set("use_simplified_query_generation", True)`.
34+
- Disabled simplified query generation by default.
1835

1936
#### Improvements
2037

@@ -38,11 +55,13 @@
3855
- Fixed a bug where `pd.get_dummies` didn't ignore NULL/NaN values by default.
3956
- Fixed a bug where repeated calls to `pd.get_dummies` results in 'Duplicated column name error'.
4057
- Fixed a bug in `pd.get_dummies` where passing list of columns generated incorrect column labels in output DataFrame.
58+
- Update `pd.get_dummies` to return bool values instead of int.
4159

4260
#### Improvements
4361

4462
- Improve error message for `pd.to_snowflake`, `DataFrame.to_snowflake`, and `Series.to_snowflake` when the table does not exist.
4563
- Improve readability of docstring for the `if_exists` parameter in `pd.to_snowflake`, `DataFrame.to_snowflake`, and `Series.to_snowflake`.
64+
- Improve error message for all pandas functions that use UDFs with Snowpark objects.
4665

4766
## 1.28.0 (2025-02-20)
4867

‎docs/source/snowpark/functions.rst‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ Functions
9393
bitnot
9494
bitshiftleft
9595
bitshiftright
96+
build_stage_file_url
9697
builtin
9798
bround
9899
call_builtin
@@ -174,6 +175,20 @@ Functions
174175
first_value
175176
flatten
176177
floor
178+
fl_get_content_type
179+
fl_get_etag
180+
fl_get_file_type
181+
fl_get_last_modified
182+
fl_get_relative_path
183+
fl_get_scoped_file_url
184+
fl_get_size
185+
fl_get_stage
186+
fl_get_stage_file_url
187+
fl_is_audio
188+
fl_is_compressed
189+
fl_is_document
190+
fl_is_image
191+
fl_is_video
177192
from_unixtime
178193
from_utc_timestamp
179194
function
@@ -348,6 +363,7 @@ Functions
348363
to_date
349364
to_decimal
350365
to_double
366+
to_file
351367
to_geography
352368
to_geometry
353369
to_json

‎recipe/meta.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ requirements:
3737
run:
3838
- python
3939
- cloudpickle >=1.6.0,<=3.0.0,!=2.1.0,!=2.2.0
40-
- snowflake-connector-python >=3.12.0,<4.0.0
40+
- snowflake-connector-python >=3.14.0,<4.0.0
4141
- typing-extensions >=4.1.0,<5.0.0
4242
# need to pin libffi because of problems in cryptography.
4343
# This might no longer hold true but keep it just to avoid it from biting us again

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
SRC_DIR = os.path.join(THIS_DIR, "src")
1616
SNOWPARK_SRC_DIR = os.path.join(SRC_DIR, "snowflake", "snowpark")
1717
MODIN_DEPENDENCY_VERSION = "==0.30.1" # Snowpark pandas requires modin 0.30.1, which is compatible with pandas 2.2.x
18-
CONNECTOR_DEPENDENCY_VERSION = ">=3.12.0, <4.0.0"
18+
CONNECTOR_DEPENDENCY_VERSION = ">=3.14.0, <4.0.0"
1919
CONNECTOR_DEPENDENCY = f"snowflake-connector-python{CONNECTOR_DEPENDENCY_VERSION}"
2020
INSTALL_REQ_LIST = [
2121
"setuptools>=40.6.0",

0 commit comments

Comments
 (0)