diff --git a/.github/workflows/build-and-cache-libpq.yml b/.github/workflows/build-and-cache-libpq.yml new file mode 100644 index 000000000..3b1c03981 --- /dev/null +++ b/.github/workflows/build-and-cache-libpq.yml @@ -0,0 +1,138 @@ +name: Build and cache libpq + +# Build the libpq package and cache the artifacts. +# +# Every Python version on the same architecture will use the same libpq. +# Therefore building and caching the libpq together with the binary packages +# result in multiple concurrent builds, and the github artifacts manageer very +# confused. +# +# This job builds the libpq and then caches the artifacts so that the +# packages.yml workflow will find the library in the cache. +# +# You can see the caches at https://github.com/psycopg/psycopg2/actions/caches +# +# Or from the API: +# +# curl -fsSL -X GET \ +# -H "Accept: application/vnd.github+json" \ +# -H "Authorization: Bearer $GITHUB_TOKEN" \ +# -H "X-GitHub-Api-Version: 2022-11-28" \ +# "https://api.github.com/repos/psycopg/psycopg/actions/caches" \ +# | jq -r '.actions_caches[].key' +# +# You can delete a cache using: +# +# curl -fsSL -X DELETE \ +# -H "Accept: application/vnd.github+json" \ +# -H "Authorization: Bearer $GITHUB_TOKEN" \ +# -H "X-GitHub-Api-Version: 2022-11-28" \ +# "https://api.github.com/repos/psycopg/psycopg/actions/caches?key=libpq-manylinux-ppc64le-17.2-3.4.0" +# +# ref: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-github-actions-caches-for-a-repository-using-a-cache-key + +on: + workflow_dispatch: + push: + paths: + - .github/workflows/build-and-cache-libpq.yml + - scripts/build/build_libpq.sh + +# TODO: move these env vars in an external env file in order to share them +# across workflows. + +env: + LIBPQ_VERSION: "17.6" + OPENSSL_VERSION: "3.5.4" + PQ_FLAGS: "" + +concurrency: + # Cancel older requests of the same workflow in the same branch. + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + + linux: # {{{ + runs-on: ubuntu-latest + if: true + + strategy: + fail-fast: false + matrix: + arch: [x86_64, ppc64le, aarch64, riscv64] + platform: [manylinux, musllinux] + + steps: + - uses: actions/checkout@v5 + + - name: Set up QEMU for multi-arch build + # Check https://github.com/docker/setup-qemu-action for newer versions. + uses: docker/setup-qemu-action@v3 + with: + # https://github.com/pypa/cibuildwheel/discussions/2256 + image: tonistiigi/binfmt:qemu-v8.1.5 + + - name: Cache libpq build + uses: actions/cache@v4 + with: + path: /tmp/libpq.build + key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} + + - name: Build wheels + uses: pypa/cibuildwheel@v3.2.0 + env: + CIBW_SKIP: "cp31?t-*" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_MANYLINUX_I686_IMAGE: manylinux2014 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 + CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014 + CIBW_MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + CIBW_BUILD: cp313-${{matrix.platform}}_${{matrix.arch}} + CIBW_ARCHS_LINUX: auto aarch64 ppc64le riscv64 + CIBW_BEFORE_ALL_LINUX: ./scripts/build/build_libpq.sh + CIBW_REPAIR_WHEEL_COMMAND: >- + ./scripts/build/strip_wheel.sh {wheel} + && auditwheel repair -w {dest_dir} {wheel} + CIBW_ENVIRONMENT_PASS_LINUX: LIBPQ_VERSION OPENSSL_VERSION + CIBW_ENVIRONMENT: >- + LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build + PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH" + LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64" + + + # }}} + + macos: # {{{ + runs-on: macos-latest + if: true + + strategy: + fail-fast: false + matrix: + arch: [x86_64, arm64] + + steps: + - name: Checkout repos + uses: actions/checkout@v5 + + - name: Cache libpq build + uses: actions/cache@v4 + with: + path: /tmp/libpq.build + key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} + + - name: Build wheels + uses: pypa/cibuildwheel@v3.2.0 + env: + CIBW_SKIP: "cp31?t-*" + CIBW_BUILD: cp313-macosx_${{matrix.arch}} + CIBW_ARCHS_MACOS: ${{matrix.arch}} + MACOSX_ARCHITECTURE: ${{matrix.arch}} + CIBW_BEFORE_ALL_MACOS: ./scripts/build/build_libpq.sh + CIBW_ENVIRONMENT: >- + PSYCOPG_IMPL=binary + LIBPQ_BUILD_PREFIX=/tmp/libpq.build + PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH" + + # }}} diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 0358aa61d..32dc7e214 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -1,5 +1,13 @@ --- name: Build packages + +# Note: these jobs also build and cache the libpq, but, because every Python +# version will try to build and cache the same libpq instance, there is a race +# condition and most likely the artifacts manager will refuse to cache. +# +# Please run the `build-and-cache-libpq.yml` workflow when the libpq/openssl +# version change in order to update the cache. + on: - workflow_dispatch @@ -7,6 +15,7 @@ env: PIP_BREAK_SYSTEM_PACKAGES: "1" LIBPQ_VERSION: "17.6" OPENSSL_VERSION: "3.5.4" + PQ_FLAGS: "" jobs: sdist: # {{{ @@ -65,7 +74,7 @@ jobs: fail-fast: false matrix: platform: [manylinux, musllinux] - arch: [x86_64, aarch64, ppc64le] + arch: [x86_64, aarch64, ppc64le, riscv64] pyver: [cp39, cp310, cp311, cp312, cp313, cp314] runs-on: ubuntu-latest @@ -80,7 +89,7 @@ jobs: uses: actions/cache@v4 with: path: /tmp/libpq.build - key: libpq-${{ env.LIBPQ_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }} + key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} - name: Build wheels uses: pypa/cibuildwheel@v3.2.0 @@ -90,8 +99,9 @@ jobs: CIBW_MANYLINUX_I686_IMAGE: manylinux2014 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014 + CIBW_MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 CIBW_BUILD: ${{matrix.pyver}}-${{matrix.platform}}_${{matrix.arch}} - CIBW_ARCHS_LINUX: auto aarch64 ppc64le + CIBW_ARCHS_LINUX: auto aarch64 ppc64le riscv64 CIBW_BEFORE_ALL_LINUX: ./scripts/build/wheel_linux_before_all.sh CIBW_REPAIR_WHEEL_COMMAND: >- ./scripts/build/strip_wheel.sh {wheel} @@ -151,7 +161,7 @@ jobs: uses: actions/cache@v4 with: path: /tmp/libpq.build - key: libpq-${{ env.LIBPQ_VERSION }}-macos-${{ matrix.arch }} + key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} - name: Build wheels uses: pypa/cibuildwheel@v3.2.0 diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 52627e4d1..d5d197124 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Build a modern version of libpq and depending libs from source on Centos 5, Alpine or macOS +# Build a modern version of libpq and depending libs from source on Centos 5, Rocky, Alpine or macOS set -euo pipefail @@ -54,9 +54,9 @@ fi # Install packages required to build the libpq. case "$ID" in - centos) + centos|rocky) yum update -y - yum install -y flex krb5-devel pam-devel perl-IPC-Cmd perl-Time-Piece zlib-devel + yum install -y flex krb5-devel pam-devel perl perl-IPC-Cmd perl-Time-Piece zlib-devel ;; alpine) @@ -104,7 +104,7 @@ else ) fi -if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then if [[ ! -f "${LIBPQ_BUILD_PREFIX}/openssl.cnf" ]]; then # Build openssl if needed @@ -160,7 +160,7 @@ if [ "$ID" == "macos" ]; then fi -if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libsasl2.${library_suffix}" ]]; then # Build libsasl2 if needed @@ -176,6 +176,14 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then pushd "${sasl_dir}" + if [ "$ID" == "rocky" ]; then + # Fix missing time.h include in multiple files for newer GCC versions + sed -i.bak '/#include "saslint.h"/a\ +#include ' lib/saslutil.c + sed -i.bak '/#include "plugin_common.h"/a\ +#include ' plugins/cram.c + fi + autoreconf -i ./configure "${make_configure_standard_flags[@]}" --disable-macos-framework make -s @@ -193,7 +201,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then fi -if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libldap.${library_suffix}" ]]; then # Build openldap if needed diff --git a/scripts/build/print_so_versions.sh b/scripts/build/print_so_versions.sh index a3c4ecdce..c765a1655 100755 --- a/scripts/build/print_so_versions.sh +++ b/scripts/build/print_so_versions.sh @@ -26,7 +26,7 @@ case "$ID" in done) | sort | uniq ;; - centos) + centos|rocky) echo "TODO!" ;; diff --git a/scripts/build/wheel_linux_before_all.sh b/scripts/build/wheel_linux_before_all.sh index 6be565ec5..845d6e5a1 100755 --- a/scripts/build/wheel_linux_before_all.sh +++ b/scripts/build/wheel_linux_before_all.sh @@ -35,7 +35,7 @@ case "$ID" in apt-get -y install libpq-dev ;; - centos) + centos|rocky) "${dir}/build_libpq.sh" > /dev/null ;;