From 32fcaf532c4aaca20498669e580a55cb02c43fa9 Mon Sep 17 00:00:00 2001 From: Miguel Liezun Date: Wed, 29 Oct 2025 10:06:12 +0100 Subject: [PATCH] feat: add workflow to build riscv64 binaries and cache libpq This commit introduces a new GitHub Actions workflow for building and caching the libpq package, ensuring that the libpq artifacts are available for subsequent builds. Additionally, it updates the build scripts to release riscv64 binaries for linux. --- .github/workflows/build-and-cache-libpq.yml | 138 ++++++++++++++++++++ .github/workflows/packages.yml | 18 ++- scripts/build/build_libpq.sh | 20 ++- scripts/build/print_so_versions.sh | 2 +- scripts/build/wheel_linux_before_all.sh | 2 +- 5 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/build-and-cache-libpq.yml diff --git a/.github/workflows/build-and-cache-libpq.yml b/.github/workflows/build-and-cache-libpq.yml new file mode 100644 index 000000000..3b1c03981 --- /dev/null +++ b/.github/workflows/build-and-cache-libpq.yml @@ -0,0 +1,138 @@ +name: Build and cache libpq + +# Build the libpq package and cache the artifacts. +# +# Every Python version on the same architecture will use the same libpq. +# Therefore building and caching the libpq together with the binary packages +# result in multiple concurrent builds, and the github artifacts manageer very +# confused. +# +# This job builds the libpq and then caches the artifacts so that the +# packages.yml workflow will find the library in the cache. +# +# You can see the caches at https://github.com/psycopg/psycopg2/actions/caches +# +# Or from the API: +# +# curl -fsSL -X GET \ +# -H "Accept: application/vnd.github+json" \ +# -H "Authorization: Bearer $GITHUB_TOKEN" \ +# -H "X-GitHub-Api-Version: 2022-11-28" \ +# "https://api.github.com/repos/psycopg/psycopg/actions/caches" \ +# | jq -r '.actions_caches[].key' +# +# You can delete a cache using: +# +# curl -fsSL -X DELETE \ +# -H "Accept: application/vnd.github+json" \ +# -H "Authorization: Bearer $GITHUB_TOKEN" \ +# -H "X-GitHub-Api-Version: 2022-11-28" \ +# "https://api.github.com/repos/psycopg/psycopg/actions/caches?key=libpq-manylinux-ppc64le-17.2-3.4.0" +# +# ref: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-github-actions-caches-for-a-repository-using-a-cache-key + +on: + workflow_dispatch: + push: + paths: + - .github/workflows/build-and-cache-libpq.yml + - scripts/build/build_libpq.sh + +# TODO: move these env vars in an external env file in order to share them +# across workflows. + +env: + LIBPQ_VERSION: "17.6" + OPENSSL_VERSION: "3.5.4" + PQ_FLAGS: "" + +concurrency: + # Cancel older requests of the same workflow in the same branch. + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +jobs: + + linux: # {{{ + runs-on: ubuntu-latest + if: true + + strategy: + fail-fast: false + matrix: + arch: [x86_64, ppc64le, aarch64, riscv64] + platform: [manylinux, musllinux] + + steps: + - uses: actions/checkout@v5 + + - name: Set up QEMU for multi-arch build + # Check https://github.com/docker/setup-qemu-action for newer versions. + uses: docker/setup-qemu-action@v3 + with: + # https://github.com/pypa/cibuildwheel/discussions/2256 + image: tonistiigi/binfmt:qemu-v8.1.5 + + - name: Cache libpq build + uses: actions/cache@v4 + with: + path: /tmp/libpq.build + key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} + + - name: Build wheels + uses: pypa/cibuildwheel@v3.2.0 + env: + CIBW_SKIP: "cp31?t-*" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 + CIBW_MANYLINUX_I686_IMAGE: manylinux2014 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 + CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014 + CIBW_MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + CIBW_BUILD: cp313-${{matrix.platform}}_${{matrix.arch}} + CIBW_ARCHS_LINUX: auto aarch64 ppc64le riscv64 + CIBW_BEFORE_ALL_LINUX: ./scripts/build/build_libpq.sh + CIBW_REPAIR_WHEEL_COMMAND: >- + ./scripts/build/strip_wheel.sh {wheel} + && auditwheel repair -w {dest_dir} {wheel} + CIBW_ENVIRONMENT_PASS_LINUX: LIBPQ_VERSION OPENSSL_VERSION + CIBW_ENVIRONMENT: >- + LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build + PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH" + LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64" + + + # }}} + + macos: # {{{ + runs-on: macos-latest + if: true + + strategy: + fail-fast: false + matrix: + arch: [x86_64, arm64] + + steps: + - name: Checkout repos + uses: actions/checkout@v5 + + - name: Cache libpq build + uses: actions/cache@v4 + with: + path: /tmp/libpq.build + key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} + + - name: Build wheels + uses: pypa/cibuildwheel@v3.2.0 + env: + CIBW_SKIP: "cp31?t-*" + CIBW_BUILD: cp313-macosx_${{matrix.arch}} + CIBW_ARCHS_MACOS: ${{matrix.arch}} + MACOSX_ARCHITECTURE: ${{matrix.arch}} + CIBW_BEFORE_ALL_MACOS: ./scripts/build/build_libpq.sh + CIBW_ENVIRONMENT: >- + PSYCOPG_IMPL=binary + LIBPQ_BUILD_PREFIX=/tmp/libpq.build + PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH" + + # }}} diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 0358aa61d..32dc7e214 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -1,5 +1,13 @@ --- name: Build packages + +# Note: these jobs also build and cache the libpq, but, because every Python +# version will try to build and cache the same libpq instance, there is a race +# condition and most likely the artifacts manager will refuse to cache. +# +# Please run the `build-and-cache-libpq.yml` workflow when the libpq/openssl +# version change in order to update the cache. + on: - workflow_dispatch @@ -7,6 +15,7 @@ env: PIP_BREAK_SYSTEM_PACKAGES: "1" LIBPQ_VERSION: "17.6" OPENSSL_VERSION: "3.5.4" + PQ_FLAGS: "" jobs: sdist: # {{{ @@ -65,7 +74,7 @@ jobs: fail-fast: false matrix: platform: [manylinux, musllinux] - arch: [x86_64, aarch64, ppc64le] + arch: [x86_64, aarch64, ppc64le, riscv64] pyver: [cp39, cp310, cp311, cp312, cp313, cp314] runs-on: ubuntu-latest @@ -80,7 +89,7 @@ jobs: uses: actions/cache@v4 with: path: /tmp/libpq.build - key: libpq-${{ env.LIBPQ_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }} + key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} - name: Build wheels uses: pypa/cibuildwheel@v3.2.0 @@ -90,8 +99,9 @@ jobs: CIBW_MANYLINUX_I686_IMAGE: manylinux2014 CIBW_MANYLINUX_AARCH64_IMAGE: manylinux2014 CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014 + CIBW_MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 CIBW_BUILD: ${{matrix.pyver}}-${{matrix.platform}}_${{matrix.arch}} - CIBW_ARCHS_LINUX: auto aarch64 ppc64le + CIBW_ARCHS_LINUX: auto aarch64 ppc64le riscv64 CIBW_BEFORE_ALL_LINUX: ./scripts/build/wheel_linux_before_all.sh CIBW_REPAIR_WHEEL_COMMAND: >- ./scripts/build/strip_wheel.sh {wheel} @@ -151,7 +161,7 @@ jobs: uses: actions/cache@v4 with: path: /tmp/libpq.build - key: libpq-${{ env.LIBPQ_VERSION }}-macos-${{ matrix.arch }} + key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }} - name: Build wheels uses: pypa/cibuildwheel@v3.2.0 diff --git a/scripts/build/build_libpq.sh b/scripts/build/build_libpq.sh index 52627e4d1..d5d197124 100755 --- a/scripts/build/build_libpq.sh +++ b/scripts/build/build_libpq.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Build a modern version of libpq and depending libs from source on Centos 5, Alpine or macOS +# Build a modern version of libpq and depending libs from source on Centos 5, Rocky, Alpine or macOS set -euo pipefail @@ -54,9 +54,9 @@ fi # Install packages required to build the libpq. case "$ID" in - centos) + centos|rocky) yum update -y - yum install -y flex krb5-devel pam-devel perl-IPC-Cmd perl-Time-Piece zlib-devel + yum install -y flex krb5-devel pam-devel perl perl-IPC-Cmd perl-Time-Piece zlib-devel ;; alpine) @@ -104,7 +104,7 @@ else ) fi -if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then if [[ ! -f "${LIBPQ_BUILD_PREFIX}/openssl.cnf" ]]; then # Build openssl if needed @@ -160,7 +160,7 @@ if [ "$ID" == "macos" ]; then fi -if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libsasl2.${library_suffix}" ]]; then # Build libsasl2 if needed @@ -176,6 +176,14 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then pushd "${sasl_dir}" + if [ "$ID" == "rocky" ]; then + # Fix missing time.h include in multiple files for newer GCC versions + sed -i.bak '/#include "saslint.h"/a\ +#include ' lib/saslutil.c + sed -i.bak '/#include "plugin_common.h"/a\ +#include ' plugins/cram.c + fi + autoreconf -i ./configure "${make_configure_standard_flags[@]}" --disable-macos-framework make -s @@ -193,7 +201,7 @@ if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then fi -if [ "$ID" == "centos" ] || [ "$ID" == "macos" ]; then +if [ "$ID" == "centos" ] || [ "$ID" == "rocky" ] || [ "$ID" == "macos" ]; then if [[ ! -f "${LIBPQ_BUILD_PREFIX}/lib/libldap.${library_suffix}" ]]; then # Build openldap if needed diff --git a/scripts/build/print_so_versions.sh b/scripts/build/print_so_versions.sh index a3c4ecdce..c765a1655 100755 --- a/scripts/build/print_so_versions.sh +++ b/scripts/build/print_so_versions.sh @@ -26,7 +26,7 @@ case "$ID" in done) | sort | uniq ;; - centos) + centos|rocky) echo "TODO!" ;; diff --git a/scripts/build/wheel_linux_before_all.sh b/scripts/build/wheel_linux_before_all.sh index 6be565ec5..845d6e5a1 100755 --- a/scripts/build/wheel_linux_before_all.sh +++ b/scripts/build/wheel_linux_before_all.sh @@ -35,7 +35,7 @@ case "$ID" in apt-get -y install libpq-dev ;; - centos) + centos|rocky) "${dir}/build_libpq.sh" > /dev/null ;;