[DO NOT MERGE] Add support for async io

.github/workflows/create_req_files.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
@@ -37,9 +37,10 @@ jobs:
       - name: Show created req file
         shell: bash
         run: cat ${{ env.requirements_file }}
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
-          path: temp_requirement
+          name: tested_requirement-py${{ matrix.python-version }}
+          path: ${{ env.requirements_file }}
 
   push-files:
     needs: create-req-files
@@ -50,10 +51,11 @@ jobs:
         with:
           token: ${{ secrets.PAT }}
       - name: Download requirement files
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: artifact
+          pattern: tested_requirement-py*
           path: tested_requirements
+          merge-multiple: true
       - name: Commit and push new requirements files
         run: |
           git config user.name github-actions

.github/workflows/snyk-issue.yml

@@ -15,19 +15,19 @@ jobs:
   snyk:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout Action
-      uses: actions/checkout@v3
+    - name: checkout action
+      uses: actions/checkout@v4
       with:
         repository: snowflakedb/whitesource-actions
-        token: ${{ secrets.whitesource_action_token }}
+        token: ${{ secrets.WHITESOURCE_ACTION_TOKEN }}
         path: whitesource-actions
-    - name: Set Env
-      run: echo "repo=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
+    - name: set-env
+      run: echo "REPO=$(basename $GITHUB_REPOSITORY)" >> $GITHUB_ENV
     - name: Jira Creation
       uses: ./whitesource-actions/snyk-issue
       with:
-        snyk_org: ${{ secrets.snyk_org_id_public_repo }}
-        snyk_token:  ${{ secrets.snyk_github_integration_token_public_repo }}
-        jira_token: ${{ secrets.jira_token_public_repo }}
+        snyk_org: ${{ secrets.SNYK_ORG_ID_PUBLIC_REPO }}
+        snyk_token:  ${{ secrets.SNYK_GITHUB_INTEGRATION_TOKEN_PUBLIC_REPO }}
+        jira_token: ${{ secrets.JIRA_TOKEN_PUBLIC_REPO }}
       env:
-        gh_token: ${{ secrets.github_token }}
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -125,3 +125,12 @@ core.*
 # Compiled Cython
 src/snowflake/connector/arrow_iterator.cpp
 src/snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_arrow_iterator.cpp
+
+# Prober files
+prober/parameters.json
+prober/snowflake_prober.egg-info/
+
+# SSH private key for WIF tests
+ci/wif/parameters/rsa_wif_aws_azure
+ci/wif/parameters/rsa_wif_gcp
+ci/wif/parameters/parameters_wif.json

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
         - id: check-hooks-apply
         - id: check-useless-excludes
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v4.4.0
     hooks:
     -   id: trailing-whitespace
         exclude: >
@@ -23,45 +23,12 @@ repos:
         exclude: .github/repo_meta.yaml
     -   id: debug-statements
     -   id: check-ast
--   repo: https://github.com/Lucas-C/pre-commit-hooks.git
-    rev: v1.5.1
-    hooks:
-    -   id: insert-license
-        name: insert-py-license
-        files: >
-            (?x)^(
-                src/snowflake/connector/.*\.pyx?|
-                test/.*\.py|
-            )$
-        exclude: >
-            (?x)^(
-                src/snowflake/connector/version.py|
-                src/snowflake/connector/nanoarrow_cpp|
-            )$
-        args:
-            - --license-filepath
-            - license_header.txt
-    -   id: insert-license
-        name: insert-cpp-license
-        files: >
-            (?x)^(
-                src/snowflake/connector/nanoarrow_cpp/.*\.(cpp|hpp)|
-            )$
-        args:
-            - --comment-style
-            - //
-            - --license-filepath
-            - license_header.txt
-        exclude: >
-            (?x)^(
-                src/snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow.hpp|
-            )$
 -   repo: https://github.com/asottile/yesqa
     rev: v1.5.0
     hooks:
     -   id: yesqa
 -   repo: https://github.com/mgedmin/check-manifest
-    rev: "0.49"
+    rev: "0.50"
     hooks:
     -   id: check-manifest
 -   repo: https://github.com/PyCQA/isort
@@ -76,18 +43,32 @@ repos:
               - --append-only
           files: ^src/snowflake/connector/.*\.py$
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.19.0
     hooks:
         - id: pyupgrade
           args: [--py38-plus]
+-   repo: local
+    hooks:
+    -   id: check-no-native-http
+        name: Check for native HTTP calls
+        entry: python ci/pre-commit/check_no_native_http.py
+        language: system
+        files: ^src/snowflake/connector/.*\.py$
+        exclude: |
+            (?x)^(
+                src/snowflake/connector/session_manager\.py|
+                src/snowflake/connector/aio/_session_manager\.py|
+                src/snowflake/connector/vendored/.*
+            )$
+        args: [--show-fixes]
 -   repo: https://github.com/PyCQA/flake8
-    rev: 7.0.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         additional_dependencies:
           - flake8-bugbear
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.10.0'
+    rev: 'v1.13.0'
     hooks:
     -   id: mypy
         files: |
@@ -120,14 +101,14 @@ repos:
             - types-pyOpenSSL
             - types-setuptools
 -   repo: https://github.com/psf/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
         - id: black
           args:
               - --safe
           language_version: python3
 - repo: https://github.com/pre-commit/mirrors-clang-format
-  rev: v17.0.6
+  rev: v19.1.3
   hooks:
   - id: clang-format
     types_or: [c++, c]

DESCRIPTION.md

@@ -7,6 +7,127 @@ https://docs.snowflake.com/
 Source code is also available at: https://github.com/snowflakedb/snowflake-connector-python
 
 # Release Notes
+- v3.18.0(TBD)
+  - Added the `workload_identity_impersonation_path` parameter to support service account impersonation for Workload Identity Federation on GCP and AWS workloads only
+  - Fixed `get_results_from_sfqid` when using `DictCursor` and executing multiple statements at once
+  - Added the `oauth_credentials_in_body` parameter supporting an option to send the oauth client credentials in the request body
+  - Fix retry behavior for `ECONNRESET` error
+
+- v3.17.4(September 22,2025)
+  - Added support for intermediate certificates as roots when they are stored in the trust store
+  - Bumped up vendored `urllib3` to `2.5.0` and `requests` to `v2.32.5`
+  - Dropped support for OpenSSL versions older than 1.1.1
+
+- v3.17.3(September 02,2025)
+  - Enhanced configuration file permission warning messages.
+    - Improved warning messages for readable permission issues to include clear instructions on how to skip warnings using the `SF_SKIP_WARNING_FOR_READ_PERMISSIONS_ON_CONFIG_FILE` environment variable.
+  - Fixed the bug with staging pandas dataframes on AWS - the regional endpoint is used when required
+    - This addresses the issue with `create_dataframe` call on Snowpark
+
+- v3.17.2(August 23,2025)
+  - Fixed a bug where platform_detection was retrying failed requests with warnings to non-existent endpoints.
+  - Added disabling endpoint-based platform detection by setting `platform_detection_timeout_seconds` to zero.
+
+- v3.17.1(August 17,2025)
+  - Added `infer_schema` parameter to `write_pandas` to perform schema inference on the passed data.
+  - Namespace `snowlake` reverted back to non-module.
+
+- v3.17.0(August 16,2025)
+  - Added in-band HTTP exception telemetry.
+  - Added an `unsafe_skip_file_permissions_check` flag to skip file permission checks on the cache and configuration.
+  - Added `APPLICATION_PATH` within `CLIENT_ENVIRONMENT` to distinguish between multiple scripts using the Python Connector in the same environment.
+  - Added basic JSON support for Interval types.
+  - Added in-band OCSP exception telemetry.
+  - Added support for new authentication methods with Workload Identity Federation (WIF).
+    - Added the `WORKLOAD_IDENTITY` value for the authenticator type.
+    - Added the `workload_identity_provider` and `workload_identity_entra_resource` parameters.
+  - Added support for the `use_vectorized_scanner` parameter in the write_pandas function.
+  - Added support of proxy setup using connection parameters without emitting environment variables.
+  - Added populating of `type_code` in `ResultMetadata` for interval types.
+  - Introduced the `snowflake_version` property to the connection.
+  - Moved `OAUTH_TYPE` to `CLIENT_ENVIROMENT`.
+  - Relaxed the `pyarrow` version constrain; versions >= 19 can now be used.
+  - Disabled token caching for OAuth Client Credentials authentication.
+  - Fixed OAuth authenticator values.
+  - Fixed a bug where a PAT with an external session authenticator was used while `external_session_id` was not provided in `SnowflakeRestful.fetch`.
+  - Fixed the case-sensitivity of `Oauth` and `programmatic_access_token` authenticator values.
+  - Fixed unclear error messages for incorrect `authenticator` values.
+  - Fixed GCS staging by ensuring the endpoint has a scheme.
+  - Fixed a bug where time-zoned timestamps fetched as a `pandas.DataFrame` or `pyarrow.Table` would overflow due to unnecessary precision. A clear error will now be raised if an overflow cannot be prevented.
+
+- v3.16.0(July 04,2025)
+  - Bumped numpy dependency from <2.1.0 to <=2.2.4.
+  - Added Windows support for Python 3.13.
+  - Added `bulk_upload_chunks` parameter to `write_pandas` function. Setting this parameter to True changes the behaviour of write_pandas function to first write all the data chunks to the local disk and then perform the wildcard upload of the chunks folder to the stage. In default behaviour the chunks are being saved, uploaded and deleted one by one.
+  - Added support for new authentication mechanism PAT with external session ID.
+  - Added `client_fetch_use_mp` parameter that enables multiprocessed fetching of result batches.
+  - Added basic arrow support for Interval types.
+  - Fixed `write_pandas` special characters usage in the location name.
+  - Fixed usage of `use_virtual_url` when building the location for gcs storage client.
+  - Added support for Snowflake OAuth for local applications.
+
+- v3.15.0(Apr 29,2025)
+  - Bumped up min boto and botocore version to 1.24.
+  - OCSP: terminate certificates chain traversal if a trusted certificate already reached.
+  - Added new authentication methods support for programmatic access tokens (PATs), OAuth 2.0 Authorization Code Flow, OAuth 2.0 Client Credentials Flow, and OAuth Token caching.
+    - For OAuth 2.0 Authorization Code Flow:
+      - Added the `oauth_client_id`, `oauth_client_secret`, `oauth_authorization_url`, `oauth_token_request_url`, `oauth_redirect_uri`, `oauth_scope`, `oauth_disable_pkce`, `oauth_enable_refresh_tokens` and `oauth_enable_single_use_refresh_tokens` parameters.
+      - Added the `OAUTH_AUTHORIZATION_CODE` value for the parameter authenticator.
+    - For OAuth 2.0 Client Credentials Flow:
+      - Added the `oauth_client_id`, `oauth_client_secret`, `oauth_token_request_url`, and `oauth_scope` parameters.
+      - Added the `OAUTH_CLIENT_CREDENTIALS` value for the parameter authenticator.
+    - For OAuth Token caching: Passing a username to driver configuration is required, and the `client_store_temporary_credential property` is to be set to `true`.
+
+- v3.14.1(April 21, 2025)
+  - Added support for Python 3.13.
+    - NOTE: Windows 64 support is still experimental and should not yet be used for production environments.
+  - Dropped support for Python 3.8.
+  - Added basic decimal floating-point type support.
+  - Added experimental authentication methods.
+  - Added support of GCS regional endpoints.
+  - Added support of GCS virtual urls. See more: https://cloud.google.com/storage/docs/request-endpoints#xml-api
+  - Added `client_fetch_threads` experimental parameter to better utilize threads for fetching query results.
+  - Added `check_arrow_conversion_error_on_every_column` connection property that can be set to `False` to restore previous behaviour in which driver will ignore errors until it occurs in the last column. This flag's purpose is to unblock workflows that may be impacted by the bugfix and will be removed in later releases.
+  - Lowered log levels from info to debug for some of the messages to make the output easier to follow.
+  - Allowed the connector to inherit a UUID4 generated upstream, provided in statement parameters (field: `requestId`), rather than automatically generate a UUID4 to use for the HTTP Request ID.
+  - Improved logging in urllib3, boto3, botocore - assured data masking even after migration to the external owned library in the future.
+  - Improved error message for client-side query cancellations due to timeouts.
+  - Improved security and robustness for the temporary credentials cache storage.
+  - Fixed a bug that caused driver to fail silently on `TO_DATE` arrow to python conversion when invalid date was followed by the correct one.
+  - Fixed expired S3 credentials update and increment retry when expired credentials are found.
+  - Deprecated `insecure_mode` connection property and replaced it with `disable_ocsp_checks` with the same behavior as the former property.
+
+- v3.14.0(March 03, 2025)
+  - Bumped pyOpenSSL dependency upper boundary from <25.0.0 to <26.0.0.
+  - Added a <19.0.0 pin to pyarrow as a workaround to a bug affecting Azure Batch.
+  - Optimized distribution package lookup to speed up import.
+  - Fixed a bug where privatelink OCSP Cache url could not be determined if privatelink account name was specified in uppercase.
+  - Added support for iceberg tables to `write_pandas`.
+  - Fixed base64 encoded private key tests.
+  - Fixed a bug where file permission check happened on Windows.
+  - Added support for File types.
+  - Added `unsafe_file_write` connection parameter that restores the previous behaviour of saving files downloaded with GET with 644 permissions.
+
+- v3.13.2(January 29, 2025)
+  - Changed not to use scoped temporary objects.
+
+- v3.13.1(January 29, 2025)
+  - Remedied SQL injection vulnerability in snowflake.connector.pandas_tools.write_pandas. See more https://github.com/snowflakedb/snowflake-connector-python/security/advisories/GHSA-2vpq-fh52-j3wv
+  - Remedied vulnerability in deserialization of the OCSP response cache. See more: https://github.com/snowflakedb/snowflake-connector-python/security/advisories/GHSA-m4f6-vcj4-w5mx
+  - Remedied vulnerability connected to cache files permissions. See more: https://github.com/snowflakedb/snowflake-connector-python/security/advisories/GHSA-r2x6-cjg7-8r43
+
+- v3.13.0(January 23,2025)
+  - Added a feature to limit the sizes of IO-bound ThreadPoolExecutors during PUT and GET commands.
+  - Updated README.md to include instructions on how to verify package signatures using `cosign`.
+  - Updated the log level for cursor's chunk rowcount from INFO to DEBUG.
+  - Added a feature to verify if the connection is still good enough to send queries over.
+  - Added support for base64-encoded DER private key strings in the `private_key` authentication type.
+
+- v3.12.4(December 3,2024)
+  - Fixed a bug where multipart uploads to Azure would be missing their MD5 hashes.
+  - Fixed a bug where OpenTelemetry header injection would sometimes cause Exceptions to be thrown.
+  - Fixed a bug where OCSP checks would throw TypeError and make mainly GCP blob storage unreachable.
+  - Bumped pyOpenSSL dependency from >=16.2.0,<25.0.0 to >=22.0.0,<25.0.0.
 
 - v3.12.3(October 25,2024)
   - Improved the error message for SSL-related issues to provide clearer guidance when an SSL error occurs.

Jenkinsfile

@@ -35,27 +35,58 @@ timestamps {
         string(name: 'parent_job', value: env.JOB_NAME),
         string(name: 'parent_build_number', value: env.BUILD_NUMBER)
       ]
-      stage('Test') {
-          try {
-          def commit_hash = "main" // default which we want to override
-          def bptp_tag = "bptp-built"
-          def response = authenticatedGithubCall("https://api.github.com/repos/snowflakedb/snowflake/git/ref/tags/${bptp_tag}")
-          commit_hash = response.object.sha
-          // Append the bptp-built commit sha to params
-          params += [string(name: 'svn_revision', value: commit_hash)]
-          } catch(Exception e) {
-          println("Exception computing commit hash from: ${response}")
+      parallel(
+      'Test': {
+        stage('Test') {
+            try {
+            def commit_hash = "main" // default which we want to override
+            def bptp_tag = "bptp-stable"
+            def response = authenticatedGithubCall("https://api.github.com/repos/snowflakedb/snowflake/git/ref/tags/${bptp_tag}")
+            commit_hash = response.object.sha
+            // Append the bptp-stable commit sha to params
+            params += [string(name: 'svn_revision', value: commit_hash)]
+            } catch(Exception e) {
+            println("Exception computing commit hash from: ${response}")
+            }
+          parallel (
+            'Test Python 39': { build job: 'RT-PyConnector39-PC',parameters: params},
+            'Test Python 310': { build job: 'RT-PyConnector310-PC',parameters: params},
+            'Test Python 311': { build job: 'RT-PyConnector311-PC',parameters: params},
+            'Test Python 312': { build job: 'RT-PyConnector312-PC',parameters: params},
+            'Test Python 313': { build job: 'RT-PyConnector313-PC',parameters: params},
+            'Test Python 39 OldDriver': { build job: 'RT-PyConnector39-OldDriver-PC',parameters: params},
+            'Test Python 39 FIPS': { build job: 'RT-FIPS-PyConnector39',parameters: params},
+            )
+          }
+        },
+      'Test Authentication': {
+        stage('Test Authentication') {
+          withCredentials([
+            string(credentialsId: 'a791118f-a1ea-46cd-b876-56da1b9bc71c', variable: 'NEXUS_PASSWORD'),
+            string(credentialsId: 'sfctest0-parameters-secret', variable: 'PARAMETERS_SECRET')
+          ]) {
+            sh '''\
+            |#!/bin/bash -e
+            |$WORKSPACE/ci/test_authentication.sh
+            '''.stripMargin()
+          }
+        }
+      },
+      'Test WIF': {
+        stage('Test WIF') {
+          withCredentials([
+            string(credentialsId: 'sfctest0-parameters-secret', variable: 'PARAMETERS_SECRET')
+          ]) {
+            sh '''\
+            |#!/bin/bash -e
+            |$WORKSPACE/ci/test_wif.sh
+            '''.stripMargin()
           }
-        parallel (
-          'Test Python 38': { build job: 'RT-PyConnector38-PC',parameters: params},
-          'Test Python 39': { build job: 'RT-PyConnector39-PC',parameters: params},
-          'Test Python 310': { build job: 'RT-PyConnector310-PC',parameters: params},
-          'Test Python 311': { build job: 'RT-PyConnector311-PC',parameters: params},
-          'Test Python 312': { build job: 'RT-PyConnector312-PC',parameters: params},
-          )
         }
       }
-    }
+    )
+  }
+}
 
 
 pipeline {