Randomly

.github/workflows/additional_testing.yml

@@ -0,0 +1,169 @@
+name: Stress Tests
+on:
+  push:
+  workflow_dispatch:
+    inputs:
+      os:
+        description: Operating System
+        required: true
+        type: choice
+        default: ubuntu-24.04
+        options:
+          - windows-2025
+          - ubuntu-24.04
+          - ubuntu-24.04-arm
+          - macos-15
+          - macos-13
+      python_version:
+        description: Python Version
+        required: true
+        type: choice
+        default: cp314
+        options:
+          - cp39
+          - cp310
+          - cp311
+          - cp312
+          - cp313
+          - cp314
+          - cp314t
+          - cp315
+          - cp315t
+      testsuite:
+        type: choice
+        description: Testsuite to run (fast, all)
+        required: true
+        default: fast
+        options:
+          - fast
+          - all
+      duckdb-python-sha:
+        type: string
+        description: The commit or ref to build against (defaults to latest commit of current ref)
+        required: false
+      duckdb-sha:
+        type: string
+        description: Override the DuckDB submodule commit or ref to build against
+        required: false
+      test_iterations:
+        type: number
+        description: Number of times to run each test phase
+        required: false
+        default: 3
+
+env:
+  OS_TO_USE: ${{ inputs.os || 'ubuntu-24.04' }}
+  PYTHON_VERSION_TO_USE: ${{ inputs.python_version || 'cp314' }}
+  TESTSUITE_TO_USE: ${{ inputs.testsuite || 'fast' }}
+  ITERATIONS_TO_USE: ${{ inputs.test_iterations || '3' }}
+  # Disable the ccache install
+  CIBW_BEFORE_BUILD: '' 
+
+
+jobs:
+  build:
+    name: "Build wheel: ${{ inputs.python_version || 'cp314' }}-${{ inputs.os || 'ubuntu-24.04' }}"
+    runs-on: ${{ inputs.os || 'ubuntu-24.04' }}
+
+    steps:
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.duckdb-python-sha }}
+          fetch-depth: 0
+          submodules: true
+
+      - name: Checkout DuckDB
+        shell: bash
+        if: ${{ inputs.duckdb-sha }}
+        run: |
+          cd external/duckdb
+          git fetch origin
+          git checkout ${{ inputs.duckdb-sha }}
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.8.16"
+          enable-cache: false
+          cache-suffix: -${{ env.PYTHON_VERSION_TO_USE }}-${{ env.OS_TO_USE }}
+          python-version: ${{ env.PYTHON_VERSION_TO_USE }}
+
+      - name: Build wheel
+        uses: pypa/[email protected]
+        env:
+          CIBW_ARCHS: auto
+          CIBW_BUILD: ${{ env.PYTHON_VERSION_TO_USE }}-*
+          CIBW_TEST_SKIP: '*'
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ env.PYTHON_VERSION_TO_USE }}-${{ env.OS_TO_USE }}
+          path: wheelhouse/
+
+  test:
+    name: 'Test: ${{ matrix.pytest_config.name }} - ${{ inputs.python_version }}-${{ inputs.os }}'
+    runs-on: ${{ inputs.os }}
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        pytest_config:
+          - name: "Random Order"  # randomization is automatic due to pytest-randomly
+            args: ""
+            iterations: ${{ inputs.test_iterations || 3 }}
+          - name: "Multiprocess"  # uses pytest-xdist
+            args: "-n auto"
+            iterations: ${{ inputs.test_iterations || 3 }}
+          - name: "Threaded"  # uses pytest-run-parallel
+            # TODO: Update to use threading specific tests or explicitly mark unsafe tests. test_module was chosen as an example. 
+            args: "--parallel-threads=4 --iterations=8 tests/fast/test_module.py --ignore"
+            iterations: 1
+
+    steps:
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.duckdb-python-sha }}
+          fetch-depth: 0
+          submodules: true
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.8.16"
+          enable-cache: false
+          cache-suffix: -${{ env.PYTHON_VERSION_TO_USE }}-${{ env.OS_TO_USE }}
+          python-version: ${{ env.PYTHON_VERSION_TO_USE }}
+
+      - name: Download wheel
+        uses: actions/download-artifact@v4
+        with:
+          name: wheel-${{ env.PYTHON_VERSION_TO_USE }}-${{ env.OS_TO_USE }}
+          path: wheelhouse/
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          uv export --only-group test --no-emit-project --output-file pylock.toml
+          uv pip install -r pylock.toml
+          uv pip install wheelhouse/*.whl
+
+      - name: Run ${{ matrix.pytest_config.name }} tests
+        shell: bash
+        run: |
+          TEST_TARGET="${{ inputs.testsuite == 'fast' && 'tests/fast' || 'tests' }}"
+          ITERATIONS="${{ matrix.pytest_config.iterations }}"
+          PYTEST_ARGS="${{ matrix.pytest_config.args }}"
+
+          echo "Running ${{ matrix.pytest_config.name }} pytest $ITERATIONS times against: $TEST_TARGET"
+          for i in $(seq 1 $ITERATIONS); do
+            echo ""
+            echo "${{ matrix.pytest_config.name }} Run $i/$ITERATIONS:"
+            echo "--------"
+            uv run pytest $PYTEST_ARGS "$TEST_TARGET" --durations=5
+            if [ $? -ne 0 ]; then
+              echo "${{ matrix.pytest_config.name }} Run $i failed!"
+            else
+              echo "${{ matrix.pytest_config.name }} Run $i passed!"
+            fi
+          done

.github/workflows/packaging.yml

@@ -75,7 +75,7 @@ jobs:
     name: Build and test releases
     uses: ./.github/workflows/packaging_wheels.yml
     with:
-      minimal: false
+      minimal: ${{ inputs.minimal }}
       testsuite: all
       duckdb-python-sha: ${{ inputs.duckdb-python-sha != '' && inputs.duckdb-python-sha || github.sha }}
       duckdb-sha: ${{ inputs.duckdb-sha }}

pyproject.toml

@@ -227,10 +227,10 @@ test = [ # dependencies used for running tests
     "pytest",
     "pytest-reraise",
     "pytest-timeout",
-    "pytest-xdist",         # Parallel (multi-process)
-    # "pytest-randomly",      # Randomizes order of tests 
+    "pytest-xdist", #  multi-processed tests, if `-n <num_workers> | auto`
+    "pytest-randomly", # randomizes test order to ensure no test dependencies, enabled on install
+    "pytest-run-parallel", # multi-threaded tests, if `--parallel-threads=N --iterations=N`
     "pytest-timestamper",   # Adds timestamps to test output
-    "pytest-run-parallel",  # For free-threading testing
     "mypy",
     "coverage",
     "gcovr; python_version < '3.14'",
@@ -310,6 +310,7 @@ filterwarnings = [
     "ignore:distutils Version classes are deprecated:DeprecationWarning",
     "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning",
 ]
+timeout = 120  # don't let individual tests run for more than 5 minutes
 
 [tool.coverage.run]
 branch = true

tests/conftest.py

@@ -64,7 +64,7 @@ def pytest_runtest_call(item):
     if sys.version_info[:2] == (3, 14): 
         try:
             outcome.get_result()
-        except Exception as e:
+        except duckdb.InvalidInputException as e:
             if "'pandas' is required for this operation but it was not installed" in str(e):
                 pytest.skip("pandas not available - test requires pandas functionality")
             else:
@@ -267,11 +267,9 @@ def spark():
 
 
 @pytest.fixture(scope='function')
-def duckdb_cursor():
-    connection = duckdb.connect('')
-    yield connection
-    connection.close()
-
+def duckdb_cursor(tmp_path):
+    with duckdb.connect(tmp_path / "mytest") as connection:
+        yield connection
 
 @pytest.fixture(scope='function')
 def integers(duckdb_cursor):