Skip to content
CI

guard non-ascii transfer-encoding in response _is_chunked_te #19027

Sign in to view logs

guard non-ascii transfer-encoding in response _is_chunked_te

guard non-ascii transfer-encoding in response _is_chunked_te #19027

Workflow file for this run

.github/workflows/ci-cd.yml at 84a2214
name: CI
on:
merge_group:
push:
branches:
- 'master'
- '[0-9].[0-9]+' # matches to backport branches, e.g. 3.6
tags: [ 'v*' ]
pull_request:
branches:
- 'master'
- '[0-9].[0-9]+'
schedule:
- cron: '0 6 * * *' # Daily 6AM UTC build
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
env:
COLOR: yes
FORCE_COLOR: 1 # Request colored output from CLI tools supporting it
MYPY_FORCE_COLOR: 1
PY_COLORS: 1
UPSTREAM_REPOSITORY_ID: >-
13258039
permissions: {}
jobs:
pre-setup:
name: Pre-Setup global build settings
runs-on: ubuntu-latest
outputs:
upstream-repository-id: ${{ env.UPSTREAM_REPOSITORY_ID }}
release-requested: >-
${{
(
github.event_name == 'push'
&& github.ref_type == 'tag'
)
&& true
|| false
}}
steps:
- name: Dummy
if: false
run: |
echo "Pre-setup step"
lint:
permissions:
contents: read # to fetch code (actions/checkout)
pull-requests: read # to read PR metadata (verify change fragments step)
name: Linter
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: true
- name: >-
Verify that `requirements/runtime-deps.in`
is in sync with `pyproject.toml`
run: |
set -eEuo pipefail
make sync-direct-runtime-deps
git diff --exit-code -- requirements/runtime-deps.in
- name: Setup Python
uses: actions/setup-python@v7.0.0
with:
python-version: 3.11
- name: Cache PyPI
uses: actions/cache@v6.1.0
with:
key: pip-lint-${{ hashFiles('requirements/*.txt') }}
path: ~/.cache/pip
restore-keys: |
pip-lint-
- name: Install dependencies
run: |
python -m pip install -U pip wheel setuptools build twine -r requirements/lint.in -c requirements/lint.txt
- name: Install self
run: |
python -m pip install . -c requirements/runtime-deps.txt
env:
AIOHTTP_NO_EXTENSIONS: 1
- name: Run mypy
run: |
make mypy
- name: Run slotscheck
run: |
# Some extra requirements are needed to ensure all modules
# can be scanned by slotscheck.
pip install -r requirements/base.in -c requirements/base.txt
slotscheck -v -m aiohttp
- name: Verify CHANGES fragment references PR
if: github.event_name == 'pull_request'
env:
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
set -euo pipefail
# Fetch the body fresh: github.event.pull_request.body is frozen at
# the webhook payload and stale on re-runs after a body edit.
PR_BODY=$(
gh api "repos/${{ github.repository }}/pulls/${PR_NUMBER}" --jq .body
)
added=$(
gh api --paginate \
"repos/${{ github.repository }}/pulls/${PR_NUMBER}/files" \
--jq '.[] | select(.status=="added") | .filename' \
| grep '^CHANGES/' || true
)
failed=0
for f in $added; do
num=$(basename "$f" | cut -d. -f1)
[[ "$num" =~ ^[0-9]+$ ]] || continue
if [[ "$num" == "$PR_NUMBER" ]]; then continue; fi
if grep -qE "(^|[^0-9])#${num}([^0-9]|$)" <<<"$PR_BODY"; then continue; fi
echo "::error file=$f::Change fragment ($num) must reference PR #$PR_NUMBER or an issue this PR fixes"
failed=1
done
exit $failed
- name: Install spell checker
run: |
pip install -r requirements/doc-spelling.in -c requirements/doc-spelling.txt
- name: Run docs spelling
run: |
# towncrier --yes # uncomment me after publishing a release
make doc-spelling
- name: Build package
run: |
python -m build
env:
AIOHTTP_NO_EXTENSIONS: 1
- name: Run twine checker
run: |
twine check --strict dist/*
- name: Making sure that CONTRIBUTORS.txt remains sorted
run: |
LC_ALL=C sort --check --ignore-case CONTRIBUTORS.txt
gen_llhttp:
permissions:
contents: read # to fetch code (actions/checkout)
name: Generate llhttp sources
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: true
- name: Cache llhttp generated files
uses: actions/cache@v6.1.0
id: cache
with:
key: llhttp-${{ hashFiles('vendor/llhttp/package*.json', 'vendor/llhttp/src/**/*') }}
path: vendor/llhttp/build
- name: Setup NodeJS
if: steps.cache.outputs.cache-hit != 'true'
uses: actions/setup-node@v7
with:
node-version: 18
- name: Generate llhttp sources
if: steps.cache.outputs.cache-hit != 'true'
run: |
make generate-llhttp
- name: Upload llhttp generated files
uses: actions/upload-artifact@v7
with:
name: llhttp
path: vendor/llhttp/build
if-no-files-found: error
test:
permissions:
contents: read # to fetch code (actions/checkout)
name: Test
needs: gen_llhttp
strategy:
matrix:
pyver: ['3.10', '3.11', '3.12', '3.13', '3.14']
no-extensions: ['', 'Y']
os: [ubuntu, macos, windows]
experimental: [false]
exclude:
- os: macos
no-extensions: 'Y'
- os: windows
no-extensions: 'Y'
include:
- pyver: pypy-3.11
no-extensions: 'Y'
os: ubuntu
experimental: false
- os: ubuntu
pyver: "3.14t"
no-extensions: ''
experimental: false
fail-fast: true
runs-on: ${{ matrix.os }}-latest
continue-on-error: ${{ matrix.experimental }}
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: true
- name: Setup Python ${{ matrix.pyver }}
id: python-install
# important: do not use system python
env:
UV_PYTHON_PREFERENCE: only-managed
uses: astral-sh/setup-uv@v9.0.0
with:
python-version: ${{ matrix.pyver }}
activate-environment: true
enable-cache: true
- name: Install dependencies
env:
DEPENDENCY_GROUP: test${{ endsWith(matrix.pyver, 't') && '-ft' || '' }}
run: |
uv pip install -U pip wheel setuptools build twine -r requirements/${{ env.DEPENDENCY_GROUP }}.in -c requirements/${{ env.DEPENDENCY_GROUP }}.txt
- name: Set PYTHON_GIL=0 for free-threading builds
if: ${{ endsWith(matrix.pyver, 't') }}
run: echo "PYTHON_GIL=0" >> $GITHUB_ENV
- name: Restore llhttp generated files
if: ${{ matrix.no-extensions == '' }}
uses: actions/download-artifact@v8
with:
name: llhttp
path: vendor/llhttp/build/
- name: Cythonize
if: ${{ matrix.no-extensions == '' }}
run: |
make cythonize
- name: Install self
env:
AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
run: uv pip install -e .
- name: Run unittests
env:
COLOR: yes
AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
PIP_USER: 1
run: >-
pytest --junitxml=junit.xml --numprocesses=auto --cov=aiohttp/ --cov=tests/ -m 'not dev_mode and not autobahn'
shell: bash
- name: Re-run the failing tests with maximum verbosity
if: failure()
env:
COLOR: yes
AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
run: >- # `exit 1` makes sure that the job remains red with flaky runs
pytest --no-cov -vvvvv --lf && exit 1
shell: bash
- name: Run dev_mode tests
env:
COLOR: yes
AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
PIP_USER: 1
PYTHONDEVMODE: 1
run: pytest -m dev_mode --cov=aiohttp/ --cov=tests/ --cov-append
shell: bash
- name: Turn coverage into xml
env:
COLOR: 'yes'
PIP_USER: 1
run: |
python -m coverage xml
- name: Upload coverage
uses: codecov/codecov-action@v7
with:
files: ./coverage.xml
flags: >-
CI-GHA,OS-${{
runner.os
}},VM-${{
matrix.os
}},Py-${{
steps.python-install.outputs.python-version
}}
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v7
with:
files: ./junit.xml
report_type: test_results
token: ${{ secrets.CODECOV_TOKEN }}
test-mobile:
permissions:
contents: read # to fetch code (actions/checkout)
name: Test (${{ matrix.config.platform }}, ${{ matrix.pyver }}, ${{ matrix.config.os }})
runs-on: ${{ matrix.config.os }}
needs: gen_llhttp
strategy:
matrix:
pyver: ["cp313", "cp314"]
config:
- os: ubuntu-latest
platform: android
archs: x86_64
- os: macos-14
platform: ios
archs: arm64_iphonesimulator
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: true
- name: Setup Python ${{ matrix.pyver }}
id: python-install
# important: do not use system python
env:
UV_PYTHON_PREFERENCE: only-managed
uses: astral-sh/setup-uv@v9.0.0
with:
python-version: ${{ matrix.pyver }}
activate-environment: true
enable-cache: true
- name: Install build tooling and cython
run: |
uv pip install -U pip wheel setuptools build twine -r requirements/cython.in -c requirements/cython.txt
- name: Restore llhttp generated files
uses: actions/download-artifact@v8
with:
name: llhttp
path: vendor/llhttp/build/
- name: Cythonize
run: |
make cythonize
- name: Free up disk space for Android emulator
if: ${{ matrix.config.platform == 'android' }}
uses: BRAINSia/free-disk-space@v2.1.3
with:
android: false
docker-images: false
mandb: false
large-packages: false
- name: Enable KVM group perms for Android emulator
if: ${{ matrix.config.platform == 'android' }}
# This is normally done by cibuildwheel automatically, when it detects Github Actions. But by unsetting GITHUB_ACTIONS
# in the test step, we also disable that automatic setup. So we need to do it manually here.
run: |
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=kvm
- name: Install cibuildwheel
run: uv pip install cibuildwheel==3.4.1
- name: Build wheels and test
# cibuildwheel normally uses grouping in its outputs for its build/test steps. But the loading time when
# expanding large groups in GitHub Actions is very high. So by unsetting GITHUB_ACTIONS, cibuildwheel does
# not know that it is running in a GitHub Action and thus does not use groups.
run: env -u GITHUB_ACTIONS cibuildwheel
env:
CIBW_BUILD: ${{ matrix.pyver }}-*
CIBW_PLATFORM: ${{ matrix.config.platform }}
CIBW_ARCHS: ${{ matrix.config.archs }}
CIBW_TEST_REQUIRES: -r requirements/test-mobile.txt
CIBW_TEST_SOURCES: setup.cfg README.rst tests
# Currently only Android supports colored output. See https://github.com/python/cpython/issues/150932 for iOS.
CIBW_TEST_COMMAND: python -m pytest ${{ matrix.config.platform == 'android' && '--color=yes' || '' }}
autobahn:
permissions:
contents: read # to fetch code (actions/checkout)
name: Autobahn testsuite
needs: gen_llhttp
strategy:
matrix:
pyver: ['3.14']
no-extensions: ['']
os: [ubuntu]
fail-fast: true
runs-on: ${{ matrix.os }}-latest
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: true
- name: Setup Python ${{ matrix.pyver }}
id: python-install
# important: do not use system python
env:
UV_PYTHON_PREFERENCE: only-managed
uses: astral-sh/setup-uv@v9.0.0
with:
python-version: ${{ matrix.pyver }}
activate-environment: true
enable-cache: true
- name: Install dependencies
env:
DEPENDENCY_GROUP: test${{ endsWith(matrix.pyver, 't') && '-ft' || '' }}
run: |
uv pip install -U pip wheel setuptools build twine -r requirements/${{ env.DEPENDENCY_GROUP }}.in -c requirements/${{ env.DEPENDENCY_GROUP }}.txt
- name: Restore llhttp generated files
if: ${{ matrix.no-extensions == '' }}
uses: actions/download-artifact@v8
with:
name: llhttp
path: vendor/llhttp/build/
- name: Cythonize
if: ${{ matrix.no-extensions == '' }}
run: |
make cythonize
- name: Install self
env:
AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
run: uv pip install -e .
- name: Run unittests
env:
COLOR: yes
AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
PIP_USER: 1
run: >-
PATH="${HOME}/Library/Python/3.11/bin:${HOME}/.local/bin:${PATH}"
pytest --junitxml=junit.xml --cov=aiohttp/ --cov=tests/ --timeout=0 -m autobahn
shell: bash
- name: Turn coverage into xml
env:
COLOR: 'yes'
PIP_USER: 1
run: |
python -m coverage xml
- name: Upload coverage
uses: codecov/codecov-action@v7
with:
files: ./coverage.xml
flags: Autobahn
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v7
with:
files: ./junit.xml
report_type: test_results
token: ${{ secrets.CODECOV_TOKEN }}
benchmark:
name: Benchmark
needs:
- gen_llhttp
- pre-setup # transitive, for accessing settings
if: >-
needs.pre-setup.outputs.upstream-repository-id == github.repository_id
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout project
uses: actions/checkout@v7
with:
submodules: true
- name: Setup Python 3.13.2
id: python-install
uses: actions/setup-python@v7.0.0
with:
python-version: 3.13.2
cache: pip
cache-dependency-path: requirements/*.txt
- name: Install dependencies
run: |
python -m pip install -U pip wheel setuptools build twine -r requirements/test.in -c requirements/test.txt
- name: Restore llhttp generated files
uses: actions/download-artifact@v8
with:
name: llhttp
path: vendor/llhttp/build/
- name: Cythonize
run: |
make cythonize
- name: Install self
run: python -m pip install -e .
- name: Load kernel TLS module
if: runner.os == 'Linux'
run: sudo modprobe tls
- name: Show kernel and OpenSSL build information
if: runner.os == 'Linux'
run: |
lsb_release -a
uname -r
openssl version -a
- name: Run benchmarks
uses: CodSpeedHQ/action@v4.18.5
with:
mode: instrumentation
run: python -Im pytest --no-cov -vvvvv --codspeed --durations=30 --timeout=0
cython-coverage:
permissions:
contents: read # to fetch code (actions/checkout)
name: Cython coverage
needs: gen_llhttp
strategy:
matrix:
os: [ubuntu, windows]
runs-on: ${{ matrix.os }}-latest
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: true
- name: Setup Python
id: python-install
uses: actions/setup-python@v7.0.0
with:
python-version: '3.12'
- name: Install dependencies
run: |
python -Im pip install -U pip wheel setuptools build twine -r requirements/test.in -c requirements/test.txt
- name: Uninstall blocbuster
run: python -m pip uninstall blockbuster -y
- name: Restore llhttp generated files
uses: actions/download-artifact@v8
with:
name: llhttp
path: vendor/llhttp/build/
- name: Cythonize with linetrace
run: |
make cythonize CYTHON_EXTRA="-X linetrace=True"
- name: Install self
env:
AIOHTTP_CYTHON_TRACE: 1
run: python -m pip install -e .
- name: Run tests with Cython tracing
env:
COLOR: yes
PIP_USER: 1
run: >-
pytest tests/test_client_functional.py tests/test_http_parser.py tests/test_http_writer.py tests/test_web_functional.py tests/test_web_response.py tests/test_websocket_parser.py
--cov-config=.coveragerc-cython.toml --cov=aiohttp/ --cov=tests/ --numprocesses=auto
-m 'not dev_mode and not autobahn'
shell: bash
- name: Turn coverage into xml
run: |
python -m coverage xml -o cython-coverage.xml --rcfile=.coveragerc-cython.toml
- name: Upload coverage
uses: codecov/codecov-action@v7
with:
files: ./cython-coverage.xml
disable_search: true
flags: cython-coverage
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: true
check: # This job does nothing and is only used for the branch protection
if: always()
needs:
- lint
- test
- test-mobile
- autobahn
runs-on: ubuntu-latest
steps:
- name: Decide whether the needed jobs succeeded or failed
uses: re-actors/alls-green@release/v1
with:
jobs: ${{ toJSON(needs) }}
- name: Trigger codecov notification
uses: codecov/codecov-action@v7
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: true
run_command: send-notifications
pre-deploy:
name: Pre-Deploy
runs-on: ubuntu-latest
needs:
- check
- pre-setup # transitive, for accessing settings
if: fromJSON(needs.pre-setup.outputs.release-requested)
steps:
- name: Dummy
run: |
echo "Predeploy step"
build-tarball:
permissions:
contents: read # to fetch code (actions/checkout)
name: Tarball
runs-on: ubuntu-latest
needs: pre-deploy
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: true
- name: Setup Python
uses: actions/setup-python@v7.0.0
- name: Install build tooling and cython
run: >-
python -m
pip install -U pip wheel setuptools build twine -r requirements/cython.in -c requirements/cython.txt
- name: Restore llhttp generated files
uses: actions/download-artifact@v8
with:
name: llhttp
path: vendor/llhttp/build/
- name: Cythonize
run: |
make cythonize
- name: Make sdist
run: |
python -m build --sdist
- name: Upload artifacts
uses: actions/upload-artifact@v7
with:
name: dist-sdist
path: dist
build-wheels:
permissions:
contents: read # to fetch code (actions/checkout)
name: Build wheels on ${{ matrix.os }} ${{ matrix.qemu }} ${{ matrix.musl }} ${{ matrix.platform }}
runs-on: ${{ matrix.os }}
needs: pre-deploy
strategy:
matrix:
os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest", "ubuntu-24.04-arm"]
qemu: ['']
musl: [""]
platform: [""]
include:
# Split ubuntu/musl jobs for the sake of speed-up
- os: ubuntu-latest
qemu: ppc64le
musl: ""
- os: ubuntu-latest
qemu: ppc64le
musl: musllinux
- os: ubuntu-latest
qemu: riscv64
musl: ""
- os: ubuntu-latest
qemu: riscv64
musl: musllinux
- os: ubuntu-latest
qemu: s390x
musl: ""
- os: ubuntu-latest
qemu: s390x
musl: musllinux
# armv7l builds on aarch64 hosts. We still register QEMU so
# binfmt picks up the 32-bit ARM userspace handler regardless of
# whether the host kernel has CONFIG_COMPAT enabled. Even with
# emulation, aarch64-on-aarch64 hosting beats x86_64 by a wide
# margin.
- os: ubuntu-24.04-arm
qemu: armv7l
musl: ""
- os: ubuntu-24.04-arm
qemu: armv7l
musl: musllinux
- os: ubuntu-latest
musl: musllinux
- os: ubuntu-24.04-arm
musl: musllinux
- os: ubuntu-latest
platform: android
- os: macos-14
platform: ios
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: true
- name: Set up QEMU
if: ${{ matrix.qemu }}
uses: docker/setup-qemu-action@v4
with:
platforms: all
# This should be temporary
# xref https://github.com/docker/setup-qemu-action/issues/188
# xref https://github.com/tonistiigi/binfmt/issues/215
image: tonistiigi/binfmt:qemu-v8.1.5
id: qemu
- name: Prepare emulation
run: |
if [[ -n "${{ matrix.qemu }}" ]]; then
# Build emulated architectures only if QEMU is set,
# use default "auto" otherwise
echo "CIBW_ARCHS_LINUX=${{ matrix.qemu }}" >> $GITHUB_ENV
# Override pyproject.toml's `build[uv]`: the pypa odd-arch
# manylinux/musllinux containers do not ship `uv` preinstalled.
echo "CIBW_BUILD_FRONTEND=build" >> $GITHUB_ENV
fi
shell: bash
- name: Setup Python
uses: actions/setup-python@v7.0.0
with:
python-version: 3.x
- name: Install build tooling and cython
run: >-
python -m
pip install -U pip wheel setuptools build twine -r requirements/cython.in -c requirements/cython.txt
- name: Restore llhttp generated files
uses: actions/download-artifact@v8
with:
name: llhttp
path: vendor/llhttp/build/
- name: Cythonize
run: |
make cythonize
- name: Build wheels
uses: pypa/cibuildwheel@v4.1.1
with:
# `build-frontend = "build[uv]"` (pyproject.toml) requires uv to be
# available on the runner for Windows and macOS. Installing
# cibuildwheel with the `uv` extra bundles uv with it; the
# tested-arch manylinux/musllinux containers also ship uv
# preinstalled. The odd-arch containers do not, so the
# `Prepare emulation` step above sets `CIBW_BUILD_FRONTEND=build`
# for those QEMU matrix cells.
extras: uv
env:
CIBW_PLATFORM: ${{ matrix.platform || 'auto' }}
CIBW_SKIP: pp* ${{ matrix.musl == 'musllinux' && '*manylinux*' || '*musllinux*' }}
CIBW_ARCHS_MACOS: x86_64 arm64 universal2
CIBW_ARCHS_IOS: arm64_iphoneos arm64_iphonesimulator x86_64_iphonesimulator
CIBW_ARCHS_ANDROID: arm64_v8a x86_64
- name: Upload wheels
uses: actions/upload-artifact@v7
with:
name: >-
dist-${{ matrix.os }}-${{ matrix.musl }}-${{
matrix.platform
&& matrix.platform
|| matrix.qemu
&& matrix.qemu
|| 'native'
}}
path: ./wheelhouse/*.whl
deploy:
name: Deploy (${{ matrix.group }})
needs:
- build-tarball
- build-wheels
- pre-setup # transitive, for accessing settings
runs-on: ubuntu-latest
if: >-
needs.pre-setup.outputs.upstream-repository-id == github.repository_id
permissions:
contents: write # IMPORTANT: mandatory for making GitHub Releases
id-token: write # IMPORTANT: mandatory for trusted publishing & sigstore
# TAG is shared by the two release-existence steps. GITHUB_TOKEN stays scoped
# to the steps that need it rather than job-wide, so third-party actions in
# this job never see it in their environment.
env:
TAG: ${{ github.ref_name }}
# The required-reviewer pypi environment gates this job, so a human must
# approve before anything is created or published. Release creation and
# publishing all live in this one gated matrix, so a release needs a single
# approval: the groups are pending together and a reviewer approves them in
# one review (see the strategy comment below).
environment:
name: pypi
url: https://pypi.org/p/aiohttp
strategy:
# The PyPI publish and the Sigstore signing each mint one short-lived OIDC
# identity per job and reuse it for every file, so signing the whole dist
# set in a single job can outlast the token and fail partway through
# (pypa/gh-action-pypi-publish#307). Splitting the work across groups, each
# its own job with a fresh identity signing only its share, keeps every
# signing loop well under the token lifetime.
#
# The groups run in parallel and all target the pypi environment, so they
# are pending for approval at the same time and a reviewer approves them in
# a single review rather than one prompt per group. The first group
# (job-index 0) creates the GitHub Release and the others wait for it; each
# group only ever touches its own disjoint share of dists, so the
# concurrent Release asset uploads never collide. fail-fast is off and
# every step is idempotent, so a single failed group can be re-run on its
# own.
#
# Each label "N of M" self-encodes its own position and total, which is the
# only source of truth for the split. Keep `group` the sole matrix axis: an
# include/exclude entry would renumber strategy.job-index / job-total, but
# the label-derived split below stays correct as long as job-index 0 is the
# first label.
fail-fast: false
matrix:
group:
- 1 of 2
- 2 of 2
steps:
- name: Checkout
# Only the release-creating group needs the repo (create-release reads
# CHANGES.rst and aiohttp/__init__.py); the others only touch dist/.
if: ${{ strategy.job-index == 0 }}
uses: actions/checkout@v7
with:
submodules: true
- name: Login
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | gh auth login --with-token
- name: Download distributions
uses: actions/download-artifact@v8
with:
path: dist
pattern: dist-*
merge-multiple: true
- name: Select this group's distributions
# Keep only this group's share of the dists so the job signs a bounded set.
# index and count come from the "N of M" label, the single source of truth
# for the split; to add a group, extend the matrix list above (e.g.
# "1 of 3" .. "3 of 3").
#
# The split is fully deterministic: the same built dists always sort the
# same way (LC_ALL=C, byte order, independent of runner locale) and land in
# the same group, so re-running a single failed group reprocesses exactly
# its own share and never touches another group's dists.
id: group
shell: bash
env:
GROUP: ${{ matrix.group }}
run: |
set -euo pipefail
index=$(( ${GROUP%% of *} - 1 ))
count=${GROUP##* of }
shopt -s nullglob
mapfile -t all < <(printf '%s\n' dist/*.whl dist/*.tar.gz | LC_ALL=C sort)
i=0
inputs=()
for f in "${all[@]}"; do
if [ "$(( i % count ))" -eq "${index}" ]; then
inputs+=("${f}")
else
rm -f -- "${f}"
fi
i=$(( i + 1 ))
done
echo "Group ${GROUP} keeps ${#inputs[@]} of ${#all[@]} dist(s):"
printf ' %s\n' "${inputs[@]}"
echo "sigstore-inputs=${inputs[*]}" >> "${GITHUB_OUTPUT}"
- name: Check whether the GitHub Release already exists
# The first group owns Release creation. Skipping Make Release when the
# release already exists lets the job be re-run after a partial failure
# without hitting HTTP 422. Query the API and branch on the HTTP status,
# not on prose: a 404 means "create it", any other failure (auth,
# rate-limit, network) re-raises so the job fails loudly.
if: ${{ strategy.job-index == 0 }}
id: gh-release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
if gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" \
--silent 2>err; then
echo 'exists=true' >> "${GITHUB_OUTPUT}"
elif grep -q 'HTTP 404' err; then
echo 'exists=false' >> "${GITHUB_OUTPUT}"
else
cat err >&2
exit 1
fi
- name: Make Release
# The first group creates the Release and uploads its share of the
# packages; the other groups add their packages and signatures below.
if: ${{ strategy.job-index == 0 && steps.gh-release.outputs.exists != 'true' }}
uses: aio-libs/create-release@v1.6.6
with:
changes_file: CHANGES.rst
name: aiohttp
version_file: aiohttp/__init__.py
github_token: ${{ secrets.GITHUB_TOKEN }}
dist_dir: dist
fix_issue_regex: >-
:issue:`(\d+)`
fix_issue_repl: >-
#\1
- name: Wait for the GitHub Release
# The other groups do not create the Release; they wait for the first
# group to create it before they publish or upload anything, so a failure
# to create the Release blocks the irreversible PyPI upload too. Only a
# 404 counts as "not yet"; any other API failure re-raises immediately
# instead of silently retrying for the whole timeout.
if: ${{ strategy.job-index != 0 }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
for _ in $(seq 1 150); do
if gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" \
--silent 2>err; then
exit 0
fi
if ! grep -q 'HTTP 404' err; then
cat err >&2
exit 1
fi
sleep 2
done
echo "GitHub Release ${TAG} did not appear in time" >&2
exit 1
- name: Publish 🐍📦 to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
# Allow re-running after a partial PyPI upload without failing on
# dists that a prior attempt already published.
skip-existing: true
- name: Sign the dists with Sigstore
uses: sigstore/gh-action-sigstore-python@v3.4.0
with:
inputs: ${{ steps.group.outputs.sigstore-inputs }}
- name: Upload artifact signatures to GitHub Release
# Confusingly, this action also supports updating releases, not
# just creating them. This is what we want here, since the first group
# created the release above.
#
# The groups run this concurrently against the same release, which is safe:
# each group's files are a disjoint share, so asset names never collide, and
# with no body/name inputs the action preserves the existing release
# metadata (it writes back what it reads) rather than clearing it, so the
# concurrent metadata updates are identical no-ops. The Wait step above
# guarantees the release (with its notes) already exists first.
uses: softprops/action-gh-release@v3.0.2
with:
# dist/ holds this group's packages plus their Sigstore signatures.
files: dist/**