Improving Test Concurrency

.github/workflows/additional_testing.yml

@@ -0,0 +1,154 @@
+name: Stress Tests
+on:
+  push:
+
+env:
+  OS: ubuntu-24.04
+  ARCH: x86_64
+  CIBW_SYSTEM: manylinux
+  PYTHON_VERSION: cp313
+  TESTSUITE: fast
+  ITERATIONS: 3
+
+
+
+jobs:
+  build:
+    name: "Build wheel: cp313-manylinux_x86_64"
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout DuckDB Python
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.8.16"
+          enable-cache: false
+          cache-suffix: -${{ env.PYTHON_VERSION }}-${{ env.OS }}
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Export test requirements
+        shell: bash
+        run: |
+          ls -l 
+          ls -l external
+          uv export --only-group test --no-emit-project --output-file test-requirements.txt --quiet
+      - name: Configure Cache Env
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+      - name: Build wheel
+        uses: pypa/[email protected]
+        env:
+          CIBW_ARCHS: ${{ env.ARCH }}
+          CIBW_BUILD: ${{ env.PYTHON_VERSION }}-${{ env.CIBW_SYSTEM }}_${{ env.ARCH }}
+          CIBW_TEST_SKIP: '*'
+          # Platform-specific ccache installation commands
+          CIBW_BEFORE_TEST: >
+            sccache --show-stats &&
+            uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
+            uv pip install -r pylock.toml
+          CIBW_TEST_COMMAND: sccache --show-stats
+          CIBW_BEFORE_BUILD_LINUX: >
+            if [ "$(uname -m)" = "aarch64" ]; then ARCH=aarch64; else ARCH=x86_64; fi &&
+            curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz &&
+            cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin &&
+            sccache --show-stats
+          CIBW_BEFORE_BUILD_MACOS: brew install sccache
+          CIBW_BEFORE_BUILD_WINDOWS: >
+            del "C:\Strawberry\c\bin\ccache.exe" &&
+            choco install sccache
+          SCCACHE_GHA_ENABLED: "on"
+          SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ env.PYTHON_VERSION }}-${{ env.OS }}
+          ACTIONS_CACHE_SERVICE_V2: "1"  
+          CIBW_ENVIRONMENT: CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CFLAGS="-Wno-attributes" CXXFLAGS="-Wno-attributes"
+          CIBW_ENVIRONMENT_PASS: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER PYTHON_GIL
+      - name: Upload test requirements
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-requirements-${{ env.PYTHON_VERSION }}-${{ env.CIBW_SYSTEM }}_${{ env.ARCH }}
+          path: test-requirements.txt
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ env.PYTHON_VERSION }}-${{ env.CIBW_SYSTEM }}_${{ env.ARCH }}
+          path: wheelhouse/
+
+  test:
+    name: 'Test: ${{ matrix.pytest_config.name }} - cp313-manylinux_x86_64'
+    runs-on: ubuntu-24.04
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        pytest_config:
+          - name: "Random Order"  # randomization is automatic due to pytest-randomly
+            args: ""
+            iterations: 3
+          - name: "Multiprocess"  # uses pytest-xdist
+            args: "-n auto"
+            iterations: 3
+          - name: "Threaded"  # uses pytest-run-parallel
+            # TODO: Update to use threading specific tests or explicitly mark unsafe tests. test_module was chosen as an example.
+            args: "--parallel-threads=4 --iterations=8 tests/fast/test_module.py --ignore"
+            iterations: 1
+
+    steps:
+      - uses: astral-sh/setup-uv@v6
+        with:
+          version: "0.8.16"
+          enable-cache: true
+          activate-environment: true
+          cache-suffix: -${{ env.PYTHON_VERSION }}-${{ env.OS }}
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Checkout test files only
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Download wheel
+        uses: actions/download-artifact@v4
+        with:
+          name: wheel-${{ env.PYTHON_VERSION }}-${{ env.CIBW_SYSTEM }}_${{ env.ARCH }}
+          path: wheelhouse/
+
+      - name: Download test requirements
+        uses: actions/download-artifact@v4
+        with:
+          name: test-requirements-${{ env.PYTHON_VERSION }}-${{ env.CIBW_SYSTEM }}_${{ env.ARCH }}
+          path: .
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          uv sync --no-install-project
+          uv pip install wheelhouse/*.whl
+
+      - name: Run ${{ matrix.pytest_config.name }} tests
+        shell: bash
+        run: |
+          TEST_TARGET="tests/fast"
+          ITERATIONS="${{ matrix.pytest_config.iterations }}"
+          PYTEST_ARGS="${{ matrix.pytest_config.args }}"
+
+          echo "Running ${{ matrix.pytest_config.name }} pytest $ITERATIONS times against: $TEST_TARGET"
+          for i in $(seq 1 $ITERATIONS); do
+            echo ""
+            echo "${{ matrix.pytest_config.name }} Run $i/$ITERATIONS:"
+            echo "--------"
+            pytest $PYTEST_ARGS "$TEST_TARGET" --durations=5
+            if [ $? -ne 0 ]; then
+              echo "${{ matrix.pytest_config.name }} Run $i failed!"
+            else
+              echo "${{ matrix.pytest_config.name }} Run $i passed!"
+            fi
+          done

.github/workflows/packaging_wheels.yml

@@ -30,7 +30,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp39, cp310, cp311, cp312, cp313, cp314, cp314t ]
+        python: [cp313, cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, 
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
@@ -41,10 +41,13 @@ jobs:
         minimal:
           - ${{ inputs.minimal }}
         exclude:
+          - { minimal: true, python: cp39 }
           - { minimal: true, python: cp310 }
           - { minimal: true, python: cp311 }
           - { minimal: true, python: cp312 }
-          - { minimal: true, platform: { arch: universal2 } }
+          - { minimal: true, python: cp313 }
+          - { minimal: true, platform: { os: macos-15 } }
+          - { minimal: true, platform: { os: macos-13 } }
           # Windows+cp314t disabled due to test failures in CI. 
           # TODO: Diagnose why tests fail (access violations) in some configurations
           - { python: cp314t, platform: { os: windows-2025 } }  
@@ -54,7 +57,7 @@ jobs:
       CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
       CIBW_TEST_SOURCES: tests
       CIBW_BEFORE_TEST: >
-        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
         uv pip install -r pylock.toml
       CIBW_TEST_COMMAND: >
         uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs

pyproject.toml

@@ -123,6 +123,17 @@ if.env.COVERAGE = false
 inherit.cmake.define = "append"
 cmake.define.DISABLE_UNITY = "1"
 
+# Override: CI minimal build - disable all extensions
+[[tool.scikit-build.overrides]]
+if.env.CI_MINIMAL_BUILD = true
+inherit.cmake.define = "none"
+cmake.define.CORE_EXTENSIONS = ""
+cmake.define.BUILD_EXTENSIONS = ""
+cmake.define.DISABLE_BUILTIN_EXTENSIONS = "ON"
+cmake.define.BUILD_SHELL = "OFF"
+cmake.define.BUILD_UNITTESTS = "OFF"
+cmake.define.BUILD_BENCHMARKS = "OFF"
+
 [tool.scikit-build.sdist]
 include = [
     "README.md",
@@ -227,6 +238,10 @@ test = [ # dependencies used for running tests
     "pytest",
     "pytest-reraise",
     "pytest-timeout",
+    "pytest-xdist", #  multi-processed tests, if `-n <num_workers> | auto`
+    "pytest-randomly", # randomizes test order to ensure no test dependencies, enabled on install
+    "pytest-run-parallel", # multi-threaded tests, if `--parallel-threads=N --iterations=N`
+    "pytest-timestamper",   # Adds timestamps to test output
     "mypy",
     "coverage",
     "gcovr; python_version < '3.14'",
@@ -306,6 +321,10 @@ filterwarnings = [
     "ignore:distutils Version classes are deprecated:DeprecationWarning",
     "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning",
 ]
+timeout = 300  # don't let individual tests run for more than 5 minutes
+thread_unsafe_fixtures = [  # https://pypi.org/project/pytest-run-parallel/
+    'default_con'
+]
 
 [tool.coverage.run]
 branch = true

tests/conftest.py

@@ -267,10 +267,18 @@ def spark():
 
 
 @pytest.fixture(scope='function')
-def duckdb_cursor():
-    connection = duckdb.connect('')
-    yield connection
-    connection.close()
+def duckdb_cursor(tmp_path):
+    with duckdb.connect(tmp_path / "mytest") as connection:
+        yield connection
+
+
+@pytest.fixture(scope='function')
+def default_con():
+    # ensures each test uses a fresh default connection to avoid test leakage
+    # threading_unsafe fixture, marked as such in pyproject.toml
+    duckdb.default_connection().close()
+    with duckdb.default_connection() as conn:
+        yield conn
 
 
 @pytest.fixture(scope='function')