Skip to content

Commit f9eb812

Browse files
mliezunmliezun
committed
feat: add workflow to build and cache libpq artifacts
1 parent bcff6bf commit f9eb812

File tree

2 files changed

+169
-2
lines changed

2 files changed

+169
-2
lines changed

‎.github/workflows/build-and-cache-libpq.yml‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
name: Build and cache libpq
2+
3+
# Build the libpq package and cache the artifacts.
4+
#
5+
# Every Python version on the same architecture will use the same libpq.
6+
# Therefore building and caching the libpq together with the binary packages
7+
# result in multiple concurrent builds, and the github artifacts manageer very
8+
# confused.
9+
#
10+
# This job builds the libpq and then caches the artifacts so that the
11+
# packages-bin.yml workflow will find the library in the cache.
12+
#
13+
# You can see the caches at https://github.com/psycopg/psycopg/actions/caches
14+
#
15+
# Or from the API:
16+
#
17+
# curl -fsSL -X GET \
18+
# -H "Accept: application/vnd.github+json" \
19+
# -H "Authorization: Bearer $GITHUB_TOKEN" \
20+
# -H "X-GitHub-Api-Version: 2022-11-28" \
21+
# "https://api.github.com/repos/psycopg/psycopg/actions/caches" \
22+
# | jq -r '.actions_caches[].key'
23+
#
24+
# You can delete a cache using:
25+
#
26+
# curl -fsSL -X DELETE \
27+
# -H "Accept: application/vnd.github+json" \
28+
# -H "Authorization: Bearer $GITHUB_TOKEN" \
29+
# -H "X-GitHub-Api-Version: 2022-11-28" \
30+
# "https://api.github.com/repos/psycopg/psycopg/actions/caches?key=libpq-manylinux-ppc64le-17.2-3.4.0"
31+
#
32+
# ref: https://docs.github.com/en/rest/actions/cache?apiVersion=2022-11-28#delete-github-actions-caches-for-a-repository-using-a-cache-key
33+
34+
on:
35+
workflow_dispatch:
36+
push:
37+
paths:
38+
- .github/workflows/build-and-cache-libpq.yml
39+
40+
# TODO: move these env vars in an external env file in order to share them
41+
# across workflows.
42+
43+
env:
44+
# Latest release: https://www.postgresql.org/ftp/source/
45+
LIBPQ_VERSION: "18.0"
46+
# Note: On windows the latest version can be found at
47+
# https://vcpkg.io/en/package/libpq
48+
# However the command line tool doesn't have a flag to specify to install
49+
# a specific version, so whatever you get you keep it.
50+
# https://github.com/microsoft/vcpkg/discussions/25622
51+
52+
# Latest release: https://www.openssl.org/source/
53+
OPENSSL_VERSION: "3.5.4"
54+
55+
# A string to differentiate build cacke keys to allow building different
56+
# flavours of libpq in different branches without mixups. Currently used to
57+
# make sure that libpq packages built for Psycopg 3.2 and 3.3 don't mix (in
58+
# order to change the gssencmode default only on 3.3). We may use it for
59+
# different reasons in the future.
60+
PQ_FLAGS: "-gssencmode-disable"
61+
62+
concurrency:
63+
# Cancel older requests of the same workflow in the same branch.
64+
group: ${{ github.workflow }}-${{ github.ref_name }}
65+
cancel-in-progress: true
66+
67+
jobs:
68+
69+
linux: # {{{
70+
runs-on: ubuntu-latest
71+
if: true
72+
73+
strategy:
74+
fail-fast: false
75+
matrix:
76+
arch: [x86_64, ppc64le, aarch64, riscv64]
77+
platform: [manylinux, musllinux]
78+
79+
steps:
80+
- uses: actions/checkout@v5
81+
82+
- name: Set up QEMU for multi-arch build
83+
# Check https://github.com/docker/setup-qemu-action for newer versions.
84+
uses: docker/setup-qemu-action@v3
85+
with:
86+
# https://github.com/pypa/cibuildwheel/discussions/2256
87+
image: tonistiigi/binfmt:qemu-v8.1.5
88+
89+
- name: Cache libpq build
90+
uses: actions/cache@v4
91+
with:
92+
path: /tmp/libpq.build
93+
key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
94+
95+
- name: Build wheels
96+
uses: pypa/[email protected]
97+
with:
98+
package-dir: psycopg_c
99+
env:
100+
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
101+
CIBW_MANYLINUX_I686_IMAGE: manylinux2014
102+
CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
103+
CIBW_MANYLINUX_PPC64LE_IMAGE: manylinux2014
104+
CIBW_MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64
105+
CIBW_BUILD: cp313-${{matrix.platform}}_${{matrix.arch}}
106+
CIBW_ARCHS_LINUX: auto aarch64 ppc64le riscv64
107+
CIBW_BEFORE_ALL_LINUX: ./tools/ci/build_libpq.sh
108+
CIBW_REPAIR_WHEEL_COMMAND: >-
109+
./tools/ci/strip_wheel.sh {wheel}
110+
&& auditwheel repair -w {dest_dir} {wheel}
111+
CIBW_ENVIRONMENT_PASS_LINUX: LIBPQ_VERSION OPENSSL_VERSION
112+
CIBW_ENVIRONMENT: >-
113+
LIBPQ_BUILD_PREFIX=/host/tmp/libpq.build
114+
PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
115+
LD_LIBRARY_PATH="$LIBPQ_BUILD_PREFIX/lib:$LIBPQ_BUILD_PREFIX/lib64"
116+
117+
118+
# }}}
119+
120+
macos: # {{{
121+
runs-on: macos-latest
122+
if: true
123+
124+
strategy:
125+
fail-fast: false
126+
matrix:
127+
arch: [x86_64, arm64]
128+
129+
steps:
130+
- name: Checkout repos
131+
uses: actions/checkout@v5
132+
133+
- name: Cache libpq build
134+
uses: actions/cache@v4
135+
with:
136+
path: /tmp/libpq.build
137+
key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
138+
139+
- name: Build wheels
140+
uses: pypa/[email protected]
141+
with:
142+
package-dir: psycopg_c
143+
env:
144+
CIBW_BUILD: cp313-macosx_${{matrix.arch}}
145+
CIBW_ARCHS_MACOS: ${{matrix.arch}}
146+
MACOSX_ARCHITECTURE: ${{matrix.arch}}
147+
CIBW_BEFORE_ALL_MACOS: ./tools/ci/build_libpq.sh
148+
CIBW_ENVIRONMENT: >-
149+
PSYCOPG_IMPL=binary
150+
LIBPQ_BUILD_PREFIX=/tmp/libpq.build
151+
PATH="$LIBPQ_BUILD_PREFIX/bin:$PATH"
152+
153+
# }}}

‎.github/workflows/packages.yml‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,26 @@
11
---
22
name: Build packages
3+
4+
# Note: these jobs also build and cache the libpq, but, because every Python
5+
# version will try to build and cache the same libpq instance, there is a race
6+
# condition and most likely the artifacts manager will refuse to cache.
7+
#
8+
# Please run the `build-and-cache-libpq.yml` workflow when the libpq/openssl
9+
# version change in order to update the cache.
10+
311
on:
412
- workflow_dispatch
513

614
env:
715
PIP_BREAK_SYSTEM_PACKAGES: "1"
816
LIBPQ_VERSION: "17.6"
917
OPENSSL_VERSION: "3.5.4"
18+
# A string to differentiate build cache keys to allow building different
19+
# flavours of libpq in different branches without mixups. Currently used to
20+
# make sure that libpq packages built for Psycopg 3.2 and 3.3 don't mix (in
21+
# order to change the gssencmode default only on 3.3). We may use it for
22+
# different reasons in the future.
23+
PQ_FLAGS: "-gssencmode-disable"
1024

1125
jobs:
1226
sdist: # {{{
@@ -80,7 +94,7 @@ jobs:
8094
uses: actions/cache@v4
8195
with:
8296
path: /tmp/libpq.build
83-
key: libpq-${{ env.LIBPQ_VERSION }}-${{ matrix.platform }}-${{ matrix.arch }}
97+
key: libpq-${{ matrix.platform }}-${{ matrix.arch }}-${{ env.LIBPQ_VERSION }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
8498

8599
- name: Build wheels
86100
uses: pypa/[email protected]
@@ -152,7 +166,7 @@ jobs:
152166
uses: actions/cache@v4
153167
with:
154168
path: /tmp/libpq.build
155-
key: libpq-${{ env.LIBPQ_VERSION }}-macos-${{ matrix.arch }}
169+
key: libpq-macos-${{ env.LIBPQ_VERSION }}-${{ matrix.arch }}-${{ env.OPENSSL_VERSION }}${{ env.PQ_FLAGS }}
156170

157171
- name: Build wheels
158172
uses: pypa/[email protected]

0 commit comments

Comments
 (0)