Merge branch 'main' into SNOW-2112225-make-boto-libs-optional

Author: sfc-gh-mmishchenko

.github/workflows/build_test.yml

@@ -133,15 +133,7 @@ jobs:
            download_name: win_amd64
         python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         cloud-provider: [aws, azure, gcp]
-        # TODO: When there are prebuilt wheels accessible for our dependencies (i.e. numpy)
-        # for Python 3.13 windows runs can be re-enabled. Currently, according to numpy:
-        # "Numpy built with MINGW-W64 on Windows 64 bits is experimental, and only available for
-        # testing. You are advised not to use it for production."
-        exclude:
-          - os:
-              image_name: windows-latest
-              download_name: win_amd64
-            python-version: "3.13"
+
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
@@ -179,12 +171,17 @@ jobs:
       - name: Install tox
         run: python -m pip install tox>=4
       - name: Run tests
+       # To run a single test on GHA use the below command:
+#        run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-single-ci | sed 's/ /,/g'`
         run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci | sed 's/ /,/g'`
+
         env:
           PYTHON_VERSION: ${{ matrix.python-version }}
           cloud_provider: ${{ matrix.cloud-provider }}
           PYTEST_ADDOPTS: --color=yes --tb=short
           TOX_PARALLEL_NO_SPINNER: 1
+         # To specify the test name (in single test mode) pass this env variable:
+#          SINGLE_TEST_NAME: test/path/filename.py::test_name
         shell: bash
       - name: Combine coverages
         run: python -m tox run -e coverage --skip-missing-interpreters false

.gitignore

@@ -125,3 +125,7 @@ core.*
 # Compiled Cython
 src/snowflake/connector/arrow_iterator.cpp
 src/snowflake/connector/nanoarrow_cpp/ArrowIterator/nanoarrow_arrow_iterator.cpp
+
+# Prober files
+prober/parameters.json
+prober/snowflake_prober.egg-info/

DESCRIPTION.md

@@ -7,12 +7,20 @@ https://docs.snowflake.com/
 Source code is also available at: https://github.com/snowflakedb/snowflake-connector-python
 
 # Release Notes
-- v3.16.0(TBD)
+- v3.16(TBD)
+  - Bumped numpy dependency from <2.1.0 to <=2.2.4
+  - Added Windows support for Python 3.13.
+  - Add `bulk_upload_chunks` parameter to `write_pandas` function. Setting this parameter to True changes the behaviour of write_pandas function to first write all the data chunks to the local disk and then perform the wildcard upload of the chunks folder to the stage. In default behaviour the chunks are being saved, uploaded and deleted one by one.
+  - Added support for new authentication mechanism PAT with external session ID.
+  - Added `client_fetch_use_mp` parameter that enables multiprocessed fetching of result batches.
+  - Make botocore and boto3 an optional dependencies pack.
+
+- v3.15.1(May 20, 2025)
   - Added basic arrow support for Interval types.
   - Fix `write_pandas` special characters usage in the location name.
   - Fix usage of `use_virtual_url` when building the location for gcs storage client.
   - Bind cryptography to <=44.0.3 to avoid issues with 45.0.0.
-  - Make botocore and boto3 an optional dependencies pack
+  - Added support for Snowflake OAuth for local applications.
 
 - v3.15.0(Apr 29,2025)
   - Bumped up min boto and botocore version to 1.24.

MANIFEST.in

@@ -28,3 +28,4 @@ prune tested_requirements
 prune src/snowflake/connector/nanoarrow_cpp/scripts
 prune __pycache__
 prune samples
+prune prober

ci/docker/connector_test/Dockerfile

@@ -3,8 +3,7 @@ FROM $BASE_IMAGE
 
 RUN yum install -y java-11-openjdk
 
-# TODO: When there are prebuilt wheels accessible for our dependencies (i.e. numpy)
-# for Python 3.13 this rust cargo install command can be removed.
+# Our dependencies rely on the Rust toolchain being available in the build-time environment (https://github.com/pyca/cryptography/issues/5771)
 RUN yum -y install rust cargo
 
 # This is to solve permission issue, read https://denibertovic.com/posts/handling-permissions-with-docker-volumes/

ci/docker/connector_test_lambda/Dockerfile313

@@ -2,24 +2,13 @@ FROM public.ecr.aws/lambda/python:3.13-x86_64
 
 WORKDIR /home/user/snowflake-connector-python
 
-# TODO: When there are prebuilt wheels accessible for our dependencies (i.e. numpy)
-# for Python 3.13 all dnf ... commands installing building kits can be removed.
-
-# Install necessary packages and compilers - we need to build numpy for newer version
-# Update dnf and install development tools
 RUN dnf -y update && \
-    dnf -y install \
-    gcc \
-    gcc-c++ \
-    make \
-    python3-devel \
-    openblas-devel \
-    lapack-devel && \
     dnf clean all
+
+# Our dependencies rely on the Rust toolchain being available in the build-time environment (https://github.com/pyca/cryptography/issues/5771)
 RUN dnf -y install rust cargo
 RUN dnf -y upgrade
 
-
 RUN chmod 777 /home/user/snowflake-connector-python
 ENV PATH="${PATH}:/opt/python/cp313-cp313/bin/"
 ENV PYTHONPATH="${PYTHONPATH}:/home/user/snowflake-connector-python/ci/docker/connector_test_lambda/"

prober/Dockerfile

@@ -0,0 +1,77 @@
+FROM alpine:3.18
+
+# boilerplate labels required by validation when pushing to ACR, ECR & GCR
+LABEL org.opencontainers.image.source="https://github.com/snowflakedb/snowflake-connector-python"
+LABEL com.snowflake.owners.email="[email protected]"
+LABEL com.snowflake.owners.slack="triage-snow-drivers-warsaw-dl"
+LABEL com.snowflake.owners.team="Snow Drivers"
+LABEL com.snowflake.owners.jira_area="Developer Platform"
+LABEL com.snowflake.owners.jira_component="Python Driver"
+# fake layers label to pass the validation
+LABEL com.snowflake.ugcbi.layers="sha256:850959b749c07b254308a4d1a84686fd7c09fcb94aeae33cc5748aa07e5cb232,sha256:b79d3c4628a989cbb8bc6f0bf0940ff33a68da2dca9c1ffbf8cfb2a27ac8d133,sha256:1cbcc0411a84fbce85e7ee2956c8c1e67b8e0edc81746a33d9da48c852037c3e,sha256:07e89b796f91d37255c6eec926b066d6818f3f2edc344a584d1b9566f77e1c27,sha256:84ff92691f909a05b224e1c56abb4864f01b4f8e3c854e4bb4c7baf1d3f6d652,sha256:3ab72684daee4eea64c3ae78a43ea332b86358446b6f2904dca4b634712e1537"
+
+RUN apk add --no-cache \
+    bash \
+    git \
+    make \
+    g++ \
+    zlib-dev \
+    openssl-dev \
+    libffi-dev \
+    jq
+
+ENV HOME="/home/driveruser"
+
+# Create a group with GID=1000 and a user with UID=1000
+RUN addgroup -g 1000 drivergroup && \
+    adduser -u 1000 -G drivergroup -D driveruser
+
+# Set permissions for the non-root user
+RUN mkdir -p ${HOME} && \
+    chown -R driveruser:drivergroup ${HOME}
+
+# Switch to the non-root user
+USER driveruser
+WORKDIR ${HOME}
+
+# Set environment variables
+ENV PYENV_ROOT="${HOME}/.pyenv"
+ENV PATH="${PYENV_ROOT}/shims:${PYENV_ROOT}/bin:${PATH}"
+
+
+# Install pyenv
+RUN git clone --depth=1 https://github.com/pyenv/pyenv.git ${PYENV_ROOT}
+
+# Build arguments for Python versions and Snowflake connector versions
+ARG MATRIX_VERSION='{"3.13.4": ["3.15.0", "3.13.2", "3.14.0", "3.12.3", "3.12.1", "3.12.4", "3.11.0", "3.12.2", "3.6.0", "3.7.0"], "3.9.22": ["3.15.0", "3.13.2", "3.14.0", "3.12.3", "3.12.1", "3.12.4", "3.11.0", "3.12.2", "3.6.0", "3.7.0"]}'
+
+
+# Install Python versions from ARG MATRIX_VERSION
+RUN eval "$(pyenv init --path)" && \
+    for python_version in $(echo $MATRIX_VERSION | jq -r 'keys[]'); do \
+        pyenv install $python_version || echo "Failed to install Python $python_version"; \
+    done
+
+# Create virtual environments for each combination of Python and Snowflake connector versions
+RUN for python_version in $(echo $MATRIX_VERSION | jq -r 'keys[]'); do \
+    for connector_version in $(echo $MATRIX_VERSION | jq -r ".\"${python_version}\"[]"); do \
+        venv_path="${HOME}/venvs/python_${python_version}_connector_${connector_version}"; \
+        $PYENV_ROOT/versions/$python_version/bin/python -m venv $venv_path && \
+        $venv_path/bin/pip install --upgrade pip && \
+        $venv_path/bin/pip install snowflake-connector-python==$connector_version; \
+    done; \
+done
+
+# Copy the prober script into the container
+RUN mkdir -p prober/probes/
+COPY __init__.py prober
+COPY setup.py prober
+COPY entrypoint.sh prober
+COPY probes/* prober/probes
+
+# Install /prober in editable mode for each virtual environment
+RUN for venv in ${HOME}/venvs/*; do \
+    source $venv/bin/activate && \
+    pip install -e ${HOME}/prober && \
+    deactivate; \
+done

prober/Jenkinsfile.groovy

@@ -0,0 +1,65 @@
+pipeline {
+    agent { label 'regular-memory-node' }
+
+    options {
+        ansiColor('xterm')
+        timestamps()
+    }
+
+    environment {
+        VAULT_CREDENTIALS = credentials('vault-jenkins')
+        COMMIT_SHA_SHORT = sh(script: 'cd PythonConnector/prober && git rev-parse --short HEAD', returnStdout: true).trim()
+        IMAGE_NAME = 'snowdrivers/python-driver-prober'
+        TEAM_NAME = 'Snow Drivers'
+        TEAM_JIRA_DL = 'triage-snow-drivers-warsaw-dl'
+        TEAM_JIRA_AREA = 'Developer Platform'
+        TEAM_JIRA_COMPONENT = 'Python Driver'
+    }
+
+    stages {
+        stage('Build Image') {
+            steps {
+                dir('./PythonConnector/prober') {
+                    sh """
+                    ls -l
+                    docker build \
+                    -t ${IMAGE_NAME}:${COMMIT_SHA_SHORT} \
+                    --label "org.opencontainers.image.revision=${COMMIT_SHA_SHORT}" \
+                    -f ./Dockerfile .
+                    """
+                }
+            }
+        }
+
+        stage('Checkout Jenkins Push Scripts') {
+            steps {
+                dir('k8sc-jenkins_scripts') {
+                    git branch: 'master',
+                    credentialsId: 'jenkins-snowflake-github-app-3',
+                    url: 'https://github.com/snowflakedb/k8sc-jenkins_scripts.git'
+                }
+            }
+        }
+
+        stage('Push Image') {
+            steps {
+                sh """
+                ./k8sc-jenkins_scripts/jenkins_push.sh \
+                -r "${VAULT_CREDENTIALS_USR}" \
+                -s "${VAULT_CREDENTIALS_PSW}" \
+                -i "${IMAGE_NAME}" \
+                -v "${COMMIT_SHA_SHORT}" \
+                -n "${TEAM_JIRA_DL}" \
+                -a "${TEAM_JIRA_AREA}" \
+                -C "${TEAM_JIRA_COMPONENT}"
+                """
+            }
+        }
+    }
+
+    post {
+        always {
+            cleanWs()
+        }
+    }
+}

test/integ/lambda/__init__.py prober/__init__.pytest/integ/lambda/__init__.py renamed to prober/__init__.py

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Initialize an empty string to hold all parameters
+python_version=""
+connector_version=""
+params=""
+
+# Parse command-line arguments
+while [[ "$#" -gt 0 ]]; do
+    if [[ "$1" == "--python_version" ]]; then
+        python_version="$2"
+        shift 2
+    elif [[ "$1" == "--connector_version" ]]; then
+        connector_version="$2"
+        shift 2
+    else
+        params+="$1 $2 "
+        shift 2
+    fi
+done
+
+# Construct the virtual environment path
+venv_path="${HOME}/venvs/python_${python_version}_connector_${connector_version}"
+
+# Check if the virtual environment exists
+if [[ ! -d "$venv_path" ]]; then
+    echo "Error: Virtual environment not found at $venv_path"
+    exit 1
+fi
+
+# Run main.py with given venv
+echo "Running main.py with virtual environment: $venv_path"
+source "$venv_path/bin/activate"
+prober $params
+status=$?
+deactivate
+
+# Check the exit status of prober
+if [[ $status -ne 0 ]]; then
+    echo "Error: prober returned failure."
+    exit 1
+else
+    echo "Success: prober returned success."
+    exit 0
+fi