Test the full supported platform matrix #963
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Tests | |
| on: | |
| push: | |
| branches: [ "main" ] | |
| pull_request: | |
| branches: [ "main" ] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: 'true' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| precheck: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 | |
| - name: Setup Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| with: | |
| version: "0.10.6" | |
| enable-cache: true | |
| cache-dependency-glob: "pyproject.toml" | |
| - name: Setup venv and install pip dependencies | |
| run: | | |
| uv venv --python "3.12" \ | |
| && uv sync --all-extras --all-groups --no-cache \ | |
| && uv pip install pip | |
| - name: Resolve documentation runtime with abxpkg | |
| run: .github/resolve_ci_binaries.sh docs_binaries | |
| - name: Run pre-commit checks | |
| run: uv run prek run --all-files | |
| - name: Run documentation code blocks | |
| run: uv run pytest -vv --tb=long -m "not live_required" README.md AGENTS.md skills docs | |
| live-documentation: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 | |
| - name: Setup Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| with: | |
| version: "0.10.6" | |
| enable-cache: true | |
| cache-dependency-glob: "pyproject.toml" | |
| - name: Install Nix | |
| uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 | |
| - name: Setup venv and install pip dependencies | |
| run: | | |
| uv venv --python "3.12" | |
| uv sync --all-extras --all-groups --no-cache | |
| uv pip install pip | |
| - name: Resolve live documentation runtimes with abxpkg | |
| run: .github/resolve_ci_binaries.sh manager_binaries node_npm_binaries pnpm_binaries yarn_binaries bun_binaries deno_binaries go_binaries cargo_binaries gem_binaries python_cli_binaries host_utility_binaries linux_binaries docker_binaries | |
| - name: Run live documentation code blocks | |
| run: uv run pytest -vv --tb=long -m live_required README.md AGENTS.md skills docs | |
| discover-standard-tests: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| outputs: | |
| test-files: ${{ steps.set-matrix.outputs.test-files }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 | |
| - name: Discover standard test files | |
| id: set-matrix | |
| run: | | |
| json_array="[" | |
| first=true | |
| for test_file in $(find tests -maxdepth 1 -name 'test_*.py' -type f | sort); do | |
| if grep -Eq '@pytest\.mark\.(root_required|docker_required)' "$test_file"; then | |
| continue | |
| fi | |
| if [ "$first" = true ]; then | |
| first=false | |
| else | |
| json_array+="," | |
| fi | |
| test_name=$(basename "$test_file" .py | sed 's/^test_//') | |
| json_array+="{\"path\":\"$test_file\",\"name\":\"$test_name\"}" | |
| done | |
| json_array+="]" | |
| echo "test-files=$json_array" >> "$GITHUB_OUTPUT" | |
| echo "$json_array" | |
| build: | |
| name: ${{ matrix.target.os_name }} py${{ matrix.target.python_version }} ${{ matrix.test.name }} | |
| needs: discover-standard-tests | |
| runs-on: ${{ matrix.target.os }} | |
| timeout-minutes: 20 | |
| if: ${{ needs.discover-standard-tests.outputs.test-files != '[]' }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: | |
| target: | |
| - os: ubuntu-24.04 | |
| os_name: linux | |
| python_version: '3.12.13' | |
| - os: ubuntu-24.04 | |
| os_name: linux | |
| python_version: '3.13.14' | |
| - os: ubuntu-24.04 | |
| os_name: linux | |
| python_version: '3.14.6' | |
| - os: macOS-15 | |
| os_name: macOS | |
| python_version: '3.12.10' | |
| - os: macOS-15 | |
| os_name: macOS | |
| python_version: '3.13.13' | |
| - os: macOS-15 | |
| os_name: macOS | |
| python_version: '3.14.6' | |
| test: ${{ fromJson(needs.discover-standard-tests.outputs.test-files) }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 | |
| - name: Setup Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 | |
| with: | |
| python-version: ${{ matrix.target.python_version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| with: | |
| version: "0.10.6" | |
| enable-cache: true | |
| cache-dependency-glob: "pyproject.toml" | |
| - name: Install Nix | |
| if: runner.os == 'Linux' | |
| uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 | |
| with: | |
| start-daemon: true | |
| - name: Install Nix | |
| if: runner.os != 'Linux' | |
| uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 | |
| - name: Setup venv and install pip dependencies | |
| run: | | |
| uv venv --python "${{ matrix.target.python_version }}" | |
| uv sync --all-extras | |
| uv pip install pip | |
| - name: Resolve test runtimes with abxpkg | |
| run: | | |
| sections=() | |
| while IFS= read -r section; do | |
| [[ -z "$section" ]] || sections+=("$section") | |
| done < <(.github/ci_test_sections.sh "${{ matrix.test.path }}" "${{ runner.os }}") | |
| if [[ "${#sections[@]}" -gt 0 ]]; then | |
| .github/resolve_ci_binaries.sh "${sections[@]}" | |
| fi | |
| - name: Environment diagnostic | |
| run: | | |
| [[ -z "${CI_BREW_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_BREW_BIN" --version | |
| [[ -z "${CI_NIX_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_NIX_BIN" --version | |
| [[ -z "${CI_NODE_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_NODE_BIN" --version | |
| [[ -z "${CI_NPM_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_NPM_BIN" --version | |
| [[ -z "${CI_PNPM_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_PNPM_BIN" --version | |
| [[ -z "${CI_YARN_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_YARN_BIN" --version | |
| [[ -z "${CI_YARN_BERRY_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_YARN_BERRY_BIN" --version | |
| [[ -z "${CI_BUN_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_BUN_BIN" --version | |
| [[ -z "${CI_DENO_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_DENO_BIN" --version | |
| [[ -z "${CI_GO_BIN:-}" ]] || uv run abxpkg run --install --binproviders=env "$CI_GO_BIN" version | |
| - name: Run standard test file | |
| run: uv run pytest -vv --tb=long -m "not root_required and not docker_required" "${{ matrix.test.path }}" | |
| discover-live-tests: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 20 | |
| outputs: | |
| live-tests: ${{ steps.set-matrix.outputs.live-tests }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 | |
| - name: Discover live integration test files | |
| id: set-matrix | |
| run: | | |
| json_array="[" | |
| first=true | |
| for test_file in $(find tests -maxdepth 1 -name 'test_*.py' -type f | sort); do | |
| if ! grep -Eq '@pytest\.mark\.(root_required|docker_required)' "$test_file"; then | |
| continue | |
| fi | |
| test_name=$(basename "$test_file" .py | sed 's/^test_//') | |
| needs_docker=false | |
| if grep -q "@pytest.mark.docker_required" "$test_file"; then | |
| needs_docker=true | |
| fi | |
| os_targets="linux" | |
| for os_target in $os_targets; do | |
| if [ "$os_target" = "linux" ]; then | |
| os_json='"ubuntu-24.04"' | |
| else | |
| os_json='"macOS-15"' | |
| fi | |
| entry="{\"name\":\"${test_name}-${os_target}\",\"path\":\"$test_file\",\"os\":${os_json},\"os_name\":\"${os_target}\",\"needs_docker\":$needs_docker}" | |
| if [ "$first" = true ]; then first=false; else json_array+=","; fi | |
| json_array+="$entry" | |
| done | |
| done | |
| json_array+="]" | |
| if [ "$first" = true ]; then | |
| echo "No root_required or docker_required test files were discovered" >&2 | |
| exit 1 | |
| fi | |
| echo "live-tests=$json_array" >> "$GITHUB_OUTPUT" | |
| echo "$json_array" | |
| live-integration: | |
| name: ${{ matrix.live.name }} | |
| needs: discover-live-tests | |
| runs-on: ${{ matrix.live.os }} | |
| timeout-minutes: 20 | |
| if: ${{ needs.discover-live-tests.outputs.live-tests != '[]' }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: | |
| live: ${{ fromJson(needs.discover-live-tests.outputs.live-tests) }} | |
| steps: | |
| - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 | |
| - name: Setup Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 | |
| with: | |
| python-version: '3.12' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 | |
| with: | |
| version: "0.10.6" | |
| enable-cache: true | |
| cache-dependency-glob: "pyproject.toml" | |
| - name: Install Nix | |
| if: runner.os == 'Linux' | |
| uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 | |
| with: | |
| start-daemon: true | |
| - name: Install Nix | |
| if: runner.os != 'Linux' | |
| uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 | |
| - name: Setup venv and install pip dependencies | |
| run: | | |
| uv venv --python "3.12" | |
| uv sync --all-extras | |
| uv pip install pip | |
| - name: Resolve integration runtimes with abxpkg | |
| run: | | |
| sections=() | |
| while IFS= read -r section; do | |
| [[ -z "$section" ]] || sections+=("$section") | |
| done < <(.github/ci_test_sections.sh "${{ matrix.live.path }}" "Linux") | |
| sections+=(linux_binaries) | |
| if [[ "${{ matrix.live.needs_docker }}" == "true" ]]; then | |
| sections+=(docker_binaries) | |
| fi | |
| .github/resolve_ci_binaries.sh "${sections[@]}" | |
| - name: Inspect Docker | |
| if: ${{ matrix.live.needs_docker }} | |
| run: | | |
| "$CI_DOCKER_BIN" version | |
| "$CI_DOCKER_BIN" info | |
| - name: Run every privileged provider test | |
| run: uv run pytest -vv --tb=long "${{ matrix.live.path }}" |