Merge branch 'main' into SNOW-2129602-ensure-scheme-in-gcs-stage-endpoints

Author: sfc-gh-mmishchenko

.github/workflows/build_test.yml

@@ -173,7 +173,7 @@ jobs:
       - name: Run tests
        # To run a single test on GHA use the below command:
 #        run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-single-ci | sed 's/ /,/g'`
-        run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci | sed 's/ /,/g'`
+        run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit-parallel,integ-parallel,pandas-parallel,sso}-ci | sed 's/ /,/g'`
 
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
@@ -193,6 +193,12 @@ jobs:
           path: |
             .tox/.coverage
             .tox/coverage.xml
+      - uses: actions/upload-artifact@v4
+        with:
+          include-hidden-files: true
+          name: junit_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+          path: |
+            .tox/junit.*.xml
 
   test-olddriver:
     name: Old Driver Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
@@ -307,6 +313,12 @@ jobs:
           path: |
             .coverage
             coverage.xml
+      - uses: actions/upload-artifact@v4
+        with:
+          include-hidden-files: true
+          name: junit_linux-fips-3.9-${{ matrix.cloud-provider }}
+          path: |
+            junit.*.xml
 
   test-lambda:
     name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
@@ -354,6 +366,12 @@ jobs:
           path: |
             .coverage.py${{ env.shortver }}-lambda-ci
             junit.py${{ env.shortver }}-lambda-ci-dev.xml
+      - uses: actions/upload-artifact@v4
+        with:
+          include-hidden-files: true
+          name: junit_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
+          path: |
+            junit.py${{ env.shortver }}-lambda-ci-dev.xml
 
   combine-coverage:
     if: ${{ success() || failure() }}
@@ -388,6 +406,21 @@ jobs:
               dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
               print("{} copy to {}".format(src_file, dst_file))
               shutil.copy(str(src_file), str(dst_file))'
+      - name: Collect all JUnit XML files to one dir
+        run: |
+          python -c '
+          from pathlib import Path
+          import shutil
+
+          src_dir = Path("artifacts")
+          dst_dir = Path(".") / "junit_results"
+          dst_dir.mkdir()
+          # Collect all JUnit XML files with different naming patterns
+          for pattern in ["*/junit.*.xml", "*/junit.py*-lambda-ci-dev.xml"]:
+              for src_file in src_dir.glob(pattern):
+                  dst_file = dst_dir / src_file.name
+                  print("{} copy to {}".format(src_file, dst_file))
+                  shutil.copy(str(src_file), str(dst_file))'
       - name: Combine coverages
         run: python -m tox run -e coverage
       - name: Publish html coverage
@@ -406,3 +439,9 @@ jobs:
         with:
           files: .tox/coverage.xml
           token: ${{ secrets.CODECOV_TOKEN }}
+      - name: Upload test results to Codecov
+        if: ${{ !cancelled() }}
+        uses: codecov/test-results-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: junit_results/junit.*.xml

.github/workflows/parameters/private/parameters_aws_auth_tests.json.gpg

@@ -129,3 +129,8 @@ src/snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_arrow_iterator.cpp
 # Prober files
 prober/parameters.json
 prober/snowflake_prober.egg-info/
+
+# SSH private key for WIF tests
+ci/wif/parameters/rsa_wif_aws_azure
+ci/wif/parameters/rsa_wif_gcp
+ci/wif/parameters/parameters_wif.json

.gitignore

@@ -7,19 +7,26 @@ https://docs.snowflake.com/
 Source code is also available at: https://github.com/snowflakedb/snowflake-connector-python
 
 # Release Notes
-- v3.16(TBD)
-  - Bumped numpy dependency from <2.1.0 to <=2.2.4
-  - Added Windows support for Python 3.13.
-  - Add `bulk_upload_chunks` parameter to `write_pandas` function. Setting this parameter to True changes the behaviour of write_pandas function to first write all the data chunks to the local disk and then perform the wildcard upload of the chunks folder to the stage. In default behaviour the chunks are being saved, uploaded and deleted one by one.
+- v3.16.1(TBD)
+  - Added in-band OCSP exception telemetry.
+  - Added `APPLICATION_PATH` within `CLIENT_ENVIRONMENT` to distinguish between multiple scripts using the PythonConnector in the same environment.
+  - Disabled token caching for OAuth Client Credentials authentication
+  - Added in-band HTTP exception telemetry.
+  - Fixed a bug where timezoned timestamps fetched as pandas.DataFrame or pyarrow.Table would overflow for the sake of unnecessary precision. In the case where an overflow cannot be prevented a clear error will be raised now.
+  - Fix OAuth authenticator values.
+  - Add `unsafe_skip_file_permissions_check` flag to skip file permissions check on cache and config.
+  - Introduce snowflake_version property to the connection
   - Fix GCS staging by ensuring the endpoint has a scheme.
+
+- v3.16.0(July 04,2025)
+  - Bumped numpy dependency from <2.1.0 to <=2.2.4.
+  - Added Windows support for Python 3.13.
+  - Added `bulk_upload_chunks` parameter to `write_pandas` function. Setting this parameter to True changes the behaviour of write_pandas function to first write all the data chunks to the local disk and then perform the wildcard upload of the chunks folder to the stage. In default behaviour the chunks are being saved, uploaded and deleted one by one.
   - Added support for new authentication mechanism PAT with external session ID.
   - Added `client_fetch_use_mp` parameter that enables multiprocessed fetching of result batches.
-
-- v3.15.1(May 20, 2025)
   - Added basic arrow support for Interval types.
-  - Fix `write_pandas` special characters usage in the location name.
-  - Fix usage of `use_virtual_url` when building the location for gcs storage client.
-  - Bind cryptography to <=44.0.3 to avoid issues with 45.0.0.
+  - Fixed `write_pandas` special characters usage in the location name.
+  - Fixed usage of `use_virtual_url` when building the location for gcs storage client.
   - Added support for Snowflake OAuth for local applications.
 
 - v3.15.0(Apr 29,2025)

DESCRIPTION.md

@@ -71,6 +71,18 @@ timestamps {
             '''.stripMargin()
           }
         }
+      },
+      'Test WIF': {
+        stage('Test WIF') {
+          withCredentials([
+            string(credentialsId: 'sfctest0-parameters-secret', variable: 'PARAMETERS_SECRET')
+          ]) {
+            sh '''\
+            |#!/bin/bash -e
+            |$WORKSPACE/ci/test_wif.sh
+            '''.stripMargin()
+          }
+        }
       }
     )
   }

Jenkinsfile

@@ -6,7 +6,6 @@ set -o pipefail
 export WORKSPACE=${WORKSPACE:-/mnt/workspace}
 export SOURCE_ROOT=${SOURCE_ROOT:-/mnt/host}
 
-MVNW_EXE=$SOURCE_ROOT/mvnw
 AUTH_PARAMETER_FILE=./.github/workflows/parameters/private/parameters_aws_auth_tests.json
 eval $(jq -r '.authtestparams | to_entries | map("export \(.key)=\(.value|tostring)")|.[]' $AUTH_PARAMETER_FILE)
 

ci/container/test_authentication.sh

@@ -23,5 +23,5 @@ docker run \
   -v $(cd $THIS_DIR/.. && pwd):/mnt/host \
   -v $WORKSPACE:/mnt/workspace \
   --rm \
-  nexus.int.snowflakecomputing.com:8086/docker/snowdrivers-test-external-browser-python:1 \
+    nexus.int.snowflakecomputing.com:8086/docker/snowdrivers-test-external-browser-python:2 \
   "/mnt/host/ci/container/test_authentication.sh"

ci/test_authentication.sh

@@ -14,6 +14,10 @@ curl https://repo1.maven.org/maven2/org/wiremock/wiremock-standalone/3.11.0/wire
 python3 -m venv fips_env
 source fips_env/bin/activate
 pip install -U setuptools pip
+
+# Install pytest-xdist for parallel execution
+pip install pytest-xdist
+
 pip install "${CONNECTOR_WHL}[pandas,secure-local-storage,development]"
 
 echo "!!! Environment description !!!"
@@ -24,6 +28,8 @@ python -c  "from cryptography.hazmat.backends.openssl import backend;print('Cryp
 pip freeze
 
 cd $CONNECTOR_DIR
-pytest -vvv --cov=snowflake.connector --cov-report=xml:coverage.xml test
+
+# Run tests in parallel using pytest-xdist
+pytest -n auto -vvv --cov=snowflake.connector --cov-report=xml:coverage.xml test
 
 deactivate

ci/test_fips.sh

@@ -40,7 +40,7 @@ else
         echo "[Info] Testing with ${PYTHON_VERSION}"
         SHORT_VERSION=$(python3.10 -c "print('${PYTHON_VERSION}'.replace('.', ''))")
         CONNECTOR_WHL=$(ls $CONNECTOR_DIR/dist/snowflake_connector_python*cp${SHORT_VERSION}*manylinux2014*.whl | sort -r | head -n 1)
-        TEST_LIST=`echo py${PYTHON_VERSION/\./}-{unit,integ,pandas,sso}-ci | sed 's/ /,/g'`
+        TEST_LIST=`echo py${PYTHON_VERSION/\./}-{unit-parallel,integ,pandas-parallel,sso}-ci | sed 's/ /,/g'`
         TEST_ENVLIST=fix_lint,$TEST_LIST,py${PYTHON_VERSION/\./}-coverage
         echo "[Info] Running tox for ${TEST_ENVLIST}"
 

ci/test_linux.sh

@@ -0,0 +1,79 @@
+#!/bin/bash -e
+
+set -o pipefail
+
+export THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+export RSA_KEY_PATH_AWS_AZURE="$THIS_DIR/wif/parameters/rsa_wif_aws_azure"
+export RSA_KEY_PATH_GCP="$THIS_DIR/wif/parameters/rsa_wif_gcp"
+export PARAMETERS_FILE_PATH="$THIS_DIR/wif/parameters/parameters_wif.json"
+
+run_tests_and_set_result() {
+  local provider="$1"
+  local host="$2"
+  local snowflake_host="$3"
+  local rsa_key_path="$4"
+
+  ssh -i "$rsa_key_path" -o IdentitiesOnly=yes -p 443 "$host" env BRANCH="$BRANCH" SNOWFLAKE_TEST_WIF_HOST="$snowflake_host" SNOWFLAKE_TEST_WIF_PROVIDER="$provider" SNOWFLAKE_TEST_WIF_ACCOUNT="$SNOWFLAKE_TEST_WIF_ACCOUNT" bash << EOF
+      set -e
+      set -o pipefail
+      docker run \
+        --rm \
+        -e BRANCH \
+        -e SNOWFLAKE_TEST_WIF_PROVIDER \
+        -e SNOWFLAKE_TEST_WIF_HOST \
+        -e SNOWFLAKE_TEST_WIF_ACCOUNT \
+        snowflakedb/client-python-test:1 \
+          bash -c "
+            echo 'Running tests on branch: \$BRANCH'
+            if [[ \"\$BRANCH\" =~ ^PR-[0-9]+\$ ]]; then
+              curl -L https://github.com/snowflakedb/snowflake-connector-python/archive/refs/pull/\$(echo \$BRANCH | cut -d- -f2)/head.tar.gz | tar -xz
+              mv snowflake-connector-python-* snowflake-connector-python
+            else
+              curl -L https://github.com/snowflakedb/snowflake-connector-python/archive/refs/heads/\$BRANCH.tar.gz | tar -xz
+              mv snowflake-connector-python-\$BRANCH snowflake-connector-python
+            fi
+            cd snowflake-connector-python
+            bash ci/wif/test_wif.sh
+          "
+EOF
+  local status=$?
+
+  if [[ $status -ne 0 ]]; then
+    echo "$provider tests failed with exit status: $status"
+    EXIT_STATUS=1
+  else
+    echo "$provider tests passed"
+  fi
+}
+
+get_branch() {
+  local branch
+  branch=$(git rev-parse --abbrev-ref HEAD)
+  if [[ "$branch" == "HEAD" ]]; then
+    branch=$(git name-rev --name-only HEAD | sed 's#^remotes/origin/##;s#^origin/##')
+  fi
+  echo "$branch"
+}
+
+setup_parameters() {
+  gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" --output "$RSA_KEY_PATH_AWS_AZURE" "${RSA_KEY_PATH_AWS_AZURE}.gpg"
+  gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" --output "$RSA_KEY_PATH_GCP" "${RSA_KEY_PATH_GCP}.gpg"
+  chmod 600 "$RSA_KEY_PATH_AWS_AZURE"
+  chmod 600 "$RSA_KEY_PATH_GCP"
+  gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" --output "$PARAMETERS_FILE_PATH" "${PARAMETERS_FILE_PATH}.gpg"
+  eval $(jq -r '.wif | to_entries | map("export \(.key)=\(.value|tostring)")|.[]' $PARAMETERS_FILE_PATH)
+}
+
+BRANCH=$(get_branch)
+export BRANCH
+setup_parameters
+
+# Run tests for all cloud providers
+EXIT_STATUS=0
+set +e  # Don't exit on first failure
+run_tests_and_set_result "AZURE" "$HOST_AZURE" "$SNOWFLAKE_TEST_WIF_HOST_AZURE" "$RSA_KEY_PATH_AWS_AZURE"
+run_tests_and_set_result "AWS" "$HOST_AWS" "$SNOWFLAKE_TEST_WIF_HOST_AWS" "$RSA_KEY_PATH_AWS_AZURE"
+run_tests_and_set_result "GCP" "$HOST_GCP" "$SNOWFLAKE_TEST_WIF_HOST_GCP" "$RSA_KEY_PATH_GCP"
+set -e  # Re-enable exit on error
+echo "Exit status: $EXIT_STATUS"
+exit $EXIT_STATUS