build_test.yml

name: Build and Test

on:
    push:
        branches:
            - master
            - main
        tags:
            - v*
    pull_request:
        branches:
            - '**'
    workflow_dispatch:
        inputs:
          logLevel:
            default: warning
            description: "Log level"
            required: true
          tags:
            description: "Test scenario tags"

concurrency:
  # older builds for the same pull request number or branch should be cancelled
  cancel-in-progress: true
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

jobs:
  lint:
    name: Check linting
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Display Python version
        run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox>=4
      - name: Set PY
        run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV
      - uses: actions/cache@v4
        with:
          path: ~/.cache/pre-commit
          key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }}
      - name: Run fix_lint
        run: python -m tox run -e fix_lint

  dependency:
    name: Check dependency
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Install tox
        run: python -m pip install tox>=4
      - name: Run tests
        run: python -m tox run -e dependency

  build:
    needs: lint
    strategy:
      matrix:
        os:
          - image: ubuntu-latest
            id: manylinux_x86_64
          - image: ubuntu-latest
            id: manylinux_aarch64
          - image: windows-latest
            id: win_amd64
          - image: windows-11-arm
            id: win_arm64
          - image: macos-latest
            id: macosx_x86_64
          - image: macos-latest
            id: macosx_arm64
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        exclude:
          - os:
              image: windows-11-arm
              id: win_arm64
            python-version: "3.9"
          - os:
              image: windows-11-arm
              id: win_arm64
            python-version: "3.10"
    name: Build ${{ matrix.os.id }}-py${{ matrix.python-version }}
    runs-on: ${{ matrix.os.image }}
    steps:
      - name: Set shortver
        run: echo "shortver=${longver//./}" >> $GITHUB_ENV
        env:
          longver: ${{ matrix.python-version }}
        shell: bash
      - name: Set up QEMU
        if: ${{ matrix.os.id == 'manylinux_aarch64' }}
        uses: docker/setup-qemu-action@v2
        with:
          # xref https://github.com/docker/setup-qemu-action/issues/188
          # xref https://github.com/tonistiigi/binfmt/issues/215
          image: tonistiigi/binfmt:qemu-v8.1.5
          platforms: all
      - uses: actions/checkout@v4
      - name: Building wheel
        uses: pypa/cibuildwheel@v2.21.3
        env:
          CIBW_BUILD: cp${{ env.shortver }}-${{ matrix.os.id }}
          CIBW_ARCHS_WINDOWS: ${{ matrix.os.id == 'win_arm64' && 'ARM64' || 'auto' }}
          MACOSX_DEPLOYMENT_TARGET: 10.14  # Should be kept in sync with ci/build_darwin.sh
        with:
          output-dir: dist
      - name: Show wheels generated
        run: ls -lh dist
        shell: bash
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: ${{ matrix.os.id }}_py${{ matrix.python-version }}
          path: dist/

  test:
    name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
    needs: build
    runs-on: ${{ matrix.os.image_name }}
    strategy:
      fail-fast: false
      matrix:
        os:
         - image_name: ubuntu-latest
           download_name: manylinux_x86_64
         - image_name: macos-latest
           download_name: macosx_x86_64
         - image_name: windows-latest
           download_name: win_amd64
         - image_name: windows-11-arm
           download_name: win_arm64
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        cloud-provider: [aws, azure, gcp]
        exclude:
          - os:
              image_name: windows-11-arm
              download_name: win_arm64
            python-version: "3.9"
          - os:
              image_name: windows-11-arm
              download_name: win_arm64
            python-version: "3.10"
          - os:
              image_name: windows-11-arm
              download_name: win_arm64
            python-version: "3.11"
          - os:
              image_name: windows-11-arm
              download_name: win_arm64
            python-version: "3.12"
          - os:
              image_name: windows-11-arm
              download_name: win_arm64
            python-version: "3.13"

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Set up Java
        uses: actions/setup-java@v4 # for wiremock
        with:
          java-version: ${{ matrix.os.download_name == 'win_arm64' && '21.0.5+11.0.LTS' || '11' }}
          distribution: 'temurin'
          java-package: 'jre'
      - name: Fetch Wiremock
        shell: bash
        run: curl https://repo1.maven.org/maven2/org/wiremock/wiremock-standalone/3.11.0/wiremock-standalone-3.11.0.jar --output .wiremock/wiremock-standalone.jar
      - name: Setup parameters file
        shell: bash
        env:
          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
        run: |
          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
          .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py
      - name: Setup private key file
        shell: bash
        env:
          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
        run: |
          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
          .github/workflows/parameters/public/rsa_keys/rsa_key_python_${{ matrix.cloud-provider }}.p8.gpg > test/rsa_key_python_${{ matrix.cloud-provider }}.p8
      - name: Download wheel(s)
        uses: actions/download-artifact@v4
        with:
          name: ${{ matrix.os.download_name }}_py${{ matrix.python-version }}
          path: dist
      - name: Show wheels downloaded
        run: ls -lh dist
        shell: bash
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox>=4
      - name: Run tests
       # To run a single test on GHA use the below command:
#        run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-single-ci | sed 's/ /,/g'`
        run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit-parallel,integ-parallel,pandas-parallel,sso}-ci | sed 's/ /,/g'`

        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
         # To specify the test name (in single test mode) pass this env variable:
#          SINGLE_TEST_NAME: test/path/filename.py::test_name
        shell: bash
      - name: Combine coverages
        run: python -m tox run -e coverage --skip-missing-interpreters false
        shell: bash
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
          path: |
            .tox/.coverage
            .tox/coverage.xml
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: junit_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
          path: |
            .tox/junit.*.xml

  test-olddriver:
    name: Old Driver Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
    needs: lint
    runs-on: ${{ matrix.os.image_name }}
    strategy:
      fail-fast: false
      matrix:
        os:
         # Because old the version 3.0.2 of snowflake-connector-python depends on oscrypto which causes conflicts with higher versions of libssl
         # TODO: It can be changed to ubuntu-latest, when python sf connector version in tox is above 3.4.0
         - image_name: ubuntu-22.04
           download_name: linux
        python-version: [3.9]
        cloud-provider: [aws]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Setup parameters file
        shell: bash
        env:
          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
        run: |
          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
          .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py
      - name: Setup private key file
        shell: bash
        env:
          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
        run: |
          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
          .github/workflows/parameters/public/rsa_keys/rsa_key_python_${{ matrix.cloud-provider }}.p8.gpg > test/rsa_key_python_${{ matrix.cloud-provider }}.p8
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox>=4
      - name: Run tests
        run: python -m tox run -e olddriver
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
        shell: bash

  test-noarrowextension:
    name: No Arrow Extension Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
    needs: lint
    runs-on: ${{ matrix.os.image_name }}
    strategy:
      fail-fast: false
      matrix:
        os:
         - image_name: ubuntu-latest
           download_name: linux
        python-version: [3.9]
        cloud-provider: [aws]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Upgrade setuptools, pip and wheel
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox>=4
      - name: Run tests
        run: python -m tox run -e noarrowextension
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
        shell: bash

  test-fips:
    name: Test FIPS linux-3.9-${{ matrix.cloud-provider }}
    needs: build
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        cloud-provider: [aws]
    steps:
      - uses: actions/checkout@v4
      - name: Setup parameters file
        shell: bash
        env:
          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
        run: |
          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
          .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py
      - name: Setup private key file
        shell: bash
        env:
          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
        run: |
          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
          .github/workflows/parameters/public/rsa_keys/rsa_key_python_${{ matrix.cloud-provider }}.p8.gpg > test/rsa_key_python_${{ matrix.cloud-provider }}.p8
      - name: Download wheel(s)
        uses: actions/download-artifact@v4
        with:
          name: manylinux_x86_64_py3.9
          path: dist
      - name: Show wheels downloaded
        run: ls -lh dist
        shell: bash
      - name: Run tests
        run: ./ci/test_fips_docker.sh
        env:
          PYTHON_VERSION: 3.9
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
        shell: bash
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: coverage_linux-fips-3.9-${{ matrix.cloud-provider }}
          path: |
            .coverage
            coverage.xml
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: junit_linux-fips-3.9-${{ matrix.cloud-provider }}
          path: |
            junit.*.xml

  test-lambda:
    name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
    needs: build
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
        cloud-provider: [aws]
    steps:
      - name: Set shortver
        run: echo "shortver=${longver//./}" >> $GITHUB_ENV
        env:
          longver: ${{ matrix.python-version }}
        shell: bash
      - uses: actions/checkout@v4
      - name: Setup parameters file
        shell: bash
        env:
          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
        run: |
          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
          .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py
      - name: Setup private key file
        shell: bash
        env:
          PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }}
        run: |
          gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \
          .github/workflows/parameters/public/rsa_keys/rsa_key_python_${{ matrix.cloud-provider }}.p8.gpg > test/rsa_key_python_${{ matrix.cloud-provider }}.p8
      - name: Download wheel(s)
        uses: actions/download-artifact@v4
        with:
          name: manylinux_x86_64_py${{ matrix.python-version }}
          path: dist
      - name: Show wheels downloaded
        run: ls -lh dist
        shell: bash
      - name: Run tests
        run: ./ci/test_lambda_docker.sh ${PYTHON_VERSION}
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          cloud_provider: ${{ matrix.cloud-provider }}
          PYTEST_ADDOPTS: --color=yes --tb=short
          TOX_PARALLEL_NO_SPINNER: 1
        shell: bash
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: coverage_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
          path: |
            .coverage.py${{ env.shortver }}-lambda-ci
            junit.py${{ env.shortver }}-lambda-ci-dev.xml
      - uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: junit_linux-lambda-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
          path: |
            junit.py${{ env.shortver }}-lambda-ci-dev.xml

  combine-coverage:
    if: ${{ success() || failure() }}
    name: Combine coverage
    needs: [lint, test, test-fips, test-lambda]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          path: artifacts
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.9'
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Upgrade setuptools and pip
        run: python -m pip install -U setuptools pip wheel
      - name: Install tox
        run: python -m pip install tox>=4
      - name: Collect all coverages to one dir
        run: |
          python -c '
          from pathlib import Path
          import shutil

          src_dir = Path("artifacts")
          dst_dir = Path(".") / ".tox"
          dst_dir.mkdir()
          for src_file in src_dir.glob("*/.coverage"):
              dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:])
              print("{} copy to {}".format(src_file, dst_file))
              shutil.copy(str(src_file), str(dst_file))'
      - name: Collect all JUnit XML files to one dir
        run: |
          python -c '
          from pathlib import Path
          import shutil

          src_dir = Path("artifacts")
          dst_dir = Path(".") / "junit_results"
          dst_dir.mkdir()
          # Collect all JUnit XML files with different naming patterns
          for pattern in ["*/junit.*.xml", "*/junit.py*-lambda-ci-dev.xml"]:
              for src_file in src_dir.glob(pattern):
                  dst_file = dst_dir / src_file.name
                  print("{} copy to {}".format(src_file, dst_file))
                  shutil.copy(str(src_file), str(dst_file))'
      - name: Combine coverages
        run: python -m tox run -e coverage
      - name: Publish html coverage
        uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: overall_cov_html
          path: .tox/htmlcov
      - name: Publish xml coverage
        uses: actions/upload-artifact@v4
        with:
          include-hidden-files: true
          name: overall_cov_xml
          path: .tox/coverage.xml
      - uses: codecov/codecov-action@v4
        with:
          files: .tox/coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}
      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/test-results-action@v1
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: junit_results/junit.*.xml