diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 7a0dca0b26..1c33759f2d 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -117,200 +117,6 @@ jobs: name: ${{ matrix.os.id }}_py${{ matrix.python-version }} path: dist/ - test: - name: Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - needs: build - runs-on: ${{ matrix.os.image_name }} - strategy: - fail-fast: false - matrix: - os: - - image_name: ubuntu-latest - download_name: manylinux_x86_64 - - image_name: macos-latest - download_name: macosx_x86_64 - - image_name: windows-latest - download_name: win_amd64 - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - cloud-provider: [aws, azure, gcp] - # TODO: When there are prebuilt wheels accessible for our dependencies (i.e. numpy) - # for Python 3.13 windows runs can be re-enabled. Currently, according to numpy: - # "Numpy built with MINGW-W64 on Windows 64 bits is experimental, and only available for - # testing. You are advised not to use it for production." - exclude: - - os: - image_name: windows-latest - download_name: win_amd64 - python-version: "3.13" - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Set up Java - uses: actions/setup-java@v4 # for wiremock - with: - java-version: 11 - distribution: 'temurin' - java-package: 'jre' - - name: Fetch Wiremock - shell: bash - run: curl https://repo1.maven.org/maven2/org/wiremock/wiremock-standalone/3.11.0/wiremock-standalone-3.11.0.jar --output .wiremock/wiremock-standalone.jar - - name: Setup parameters file - shell: bash - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - run: | - gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ - .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py - - name: Download wheel(s) - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.os.download_name }}_py${{ matrix.python-version }} - path: dist - - name: Show wheels downloaded - run: ls -lh dist - shell: bash - - name: Upgrade setuptools, pip and wheel - run: python -m pip install -U setuptools pip wheel - - name: Install tox - run: python -m pip install tox>=4 - - name: Run tests - run: python -m tox run -e `echo py${PYTHON_VERSION/\./}-{extras,unit,integ,pandas,sso}-ci | sed 's/ /,/g'` - env: - PYTHON_VERSION: ${{ matrix.python-version }} - cloud_provider: ${{ matrix.cloud-provider }} - PYTEST_ADDOPTS: --color=yes --tb=short - TOX_PARALLEL_NO_SPINNER: 1 - shell: bash - - name: Combine coverages - run: python -m tox run -e coverage --skip-missing-interpreters false - shell: bash - - uses: actions/upload-artifact@v4 - with: - include-hidden-files: true - name: coverage_${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - path: | - .tox/.coverage - .tox/coverage.xml - - test-olddriver: - name: Old Driver Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - needs: lint - runs-on: ${{ matrix.os.image_name }} - strategy: - fail-fast: false - matrix: - os: - # Because old the version 3.0.2 of snowflake-connector-python depends on oscrypto which causes conflicts with higher versions of libssl - # TODO: It can be changed to ubuntu-latest, when python sf connector version in tox is above 3.4.0 - - image_name: ubuntu-22.04 - download_name: linux - python-version: [3.9] - cloud-provider: [aws] - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Setup parameters file - shell: bash - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - run: | - gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ - .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py - - name: Upgrade setuptools, pip and wheel - run: python -m pip install -U setuptools pip wheel - - name: Install tox - run: python -m pip install tox>=4 - - name: Run tests - run: python -m tox run -e olddriver - env: - PYTHON_VERSION: ${{ matrix.python-version }} - cloud_provider: ${{ matrix.cloud-provider }} - PYTEST_ADDOPTS: --color=yes --tb=short - shell: bash - - test-noarrowextension: - name: No Arrow Extension Test ${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }} - needs: lint - runs-on: ${{ matrix.os.image_name }} - strategy: - fail-fast: false - matrix: - os: - - image_name: ubuntu-latest - download_name: linux - python-version: [3.9] - cloud-provider: [aws] - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Upgrade setuptools, pip and wheel - run: python -m pip install -U setuptools pip wheel - - name: Install tox - run: python -m pip install tox>=4 - - name: Run tests - run: python -m tox run -e noarrowextension - env: - PYTHON_VERSION: ${{ matrix.python-version }} - cloud_provider: ${{ matrix.cloud-provider }} - PYTEST_ADDOPTS: --color=yes --tb=short - shell: bash - - test-fips: - name: Test FIPS linux-3.9-${{ matrix.cloud-provider }} - needs: build - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - cloud-provider: [aws] - steps: - - uses: actions/checkout@v4 - - name: Setup parameters file - shell: bash - env: - PARAMETERS_SECRET: ${{ secrets.PARAMETERS_SECRET }} - run: | - gpg --quiet --batch --yes --decrypt --passphrase="$PARAMETERS_SECRET" \ - .github/workflows/parameters/public/parameters_${{ matrix.cloud-provider }}.py.gpg > test/parameters.py - - name: Download wheel(s) - uses: actions/download-artifact@v4 - with: - name: manylinux_x86_64_py3.9 - path: dist - - name: Show wheels downloaded - run: ls -lh dist - shell: bash - - name: Run tests - run: ./ci/test_fips_docker.sh - env: - PYTHON_VERSION: 3.9 - cloud_provider: ${{ matrix.cloud-provider }} - PYTEST_ADDOPTS: --color=yes --tb=short - TOX_PARALLEL_NO_SPINNER: 1 - shell: bash - - uses: actions/upload-artifact@v4 - with: - include-hidden-files: true - name: coverage_linux-fips-3.9-${{ matrix.cloud-provider }} - path: | - .coverage - coverage.xml - test-lambda: name: Test Lambda linux-${{ matrix.python-version }}-${{ matrix.cloud-provider }} needs: build @@ -357,55 +163,3 @@ jobs: path: | .coverage.py${{ env.shortver }}-lambda-ci junit.py${{ env.shortver }}-lambda-ci-dev.xml - - combine-coverage: - if: ${{ success() || failure() }} - name: Combine coverage - needs: [lint, test, test-fips, test-lambda] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - path: artifacts - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.9' - - name: Display Python version - run: python -c "import sys; print(sys.version)" - - name: Upgrade setuptools and pip - run: python -m pip install -U setuptools pip wheel - - name: Install tox - run: python -m pip install tox>=4 - - name: Collect all coverages to one dir - run: | - python -c ' - from pathlib import Path - import shutil - - src_dir = Path("artifacts") - dst_dir = Path(".") / ".tox" - dst_dir.mkdir() - for src_file in src_dir.glob("*/.coverage"): - dst_file = dst_dir / ".coverage.{}".format(src_file.parent.name[9:]) - print("{} copy to {}".format(src_file, dst_file)) - shutil.copy(str(src_file), str(dst_file))' - - name: Combine coverages - run: python -m tox run -e coverage - - name: Publish html coverage - uses: actions/upload-artifact@v4 - with: - include-hidden-files: true - name: overall_cov_html - path: .tox/htmlcov - - name: Publish xml coverage - uses: actions/upload-artifact@v4 - with: - include-hidden-files: true - name: overall_cov_xml - path: .tox/coverage.xml - - uses: codecov/codecov-action@v4 - with: - files: .tox/coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} diff --git a/test/integ/lambda/test_deadlock.py b/test/integ/lambda/test_deadlock.py new file mode 100644 index 0000000000..3bdebaee73 --- /dev/null +++ b/test/integ/lambda/test_deadlock.py @@ -0,0 +1,56 @@ +import threading + +import snowflake.connector + +# Number of threads +THREAD_COUNT = 80 + +# Create a barrier so all threads start at the same time +start_barrier = threading.Barrier(THREAD_COUNT) + + +def snowflake_task(results, index, db_parameters): + try: + # Wait for all threads to be ready + start_barrier.wait() + + with snowflake.connector.connect( + user=db_parameters["user"], + password=db_parameters["password"], + host=db_parameters["host"], + warehouse=db_parameters.get("warehouse"), + role=db_parameters.get("role"), + schema=db_parameters.get("schema"), + port=db_parameters["port"], + database=db_parameters["database"], + account=db_parameters["account"], + protocol=db_parameters["protocol"], + ) as conn: + cs = conn.cursor() + cs.execute("SELECT 1") + result = cs.fetchone()[0] + results[index] = result + print(f"RESULT OF FETCH {index} ==== {result}") + + except Exception as e: + results[index] = f"Error: {e}" + print(f"THREAD {index} ERROR: {e}") + + +# Pytest test +def test_snowflake_threads(db_parameters): + print("\n\nTesting snowflake threads\n\n") + threads = [] + results = [None] * THREAD_COUNT + + for i in range(THREAD_COUNT): + t = threading.Thread(target=snowflake_task, args=(results, i, db_parameters)) + threads.append(t) + t.start() + + for t in threads: + t.join() + + # Validate all threads returned 1 + for result in results: + assert result == 1, f"Thread failed with result: {result}"