Skip to content
This repository was archived by the owner on Mar 26, 2024. It is now read-only.

Commit 6b69e26

Browse files
authored
Merge pull request #31 from beeper/merge-1.71
2 parents 8bdf3dd + 44bbe93 commit 6b69e26

File tree

240 files changed

+10295
-4170
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

240 files changed

+10295
-4170
lines changed

.ci/scripts/auditwheel_wrapper.py

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
#!/usr/bin/env python
2+
# Copyright 2022 The Matrix.org Foundation C.I.C.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Wraps `auditwheel repair` to first check if we're repairing a potentially abi3
17+
# compatible wheel, if so rename the wheel before repairing it.
18+
19+
import argparse
20+
import os
21+
import subprocess
22+
from typing import Optional
23+
from zipfile import ZipFile
24+
25+
from packaging.tags import Tag
26+
from packaging.utils import parse_wheel_filename
27+
from packaging.version import Version
28+
29+
30+
def check_is_abi3_compatible(wheel_file: str) -> None:
31+
"""Check the contents of the built wheel for any `.so` files that are *not*
32+
abi3 compatible.
33+
"""
34+
35+
with ZipFile(wheel_file, "r") as wheel:
36+
for file in wheel.namelist():
37+
if not file.endswith(".so"):
38+
continue
39+
40+
if not file.endswith(".abi3.so"):
41+
raise Exception(f"Found non-abi3 lib: {file}")
42+
43+
44+
def cpython(wheel_file: str, name: str, version: Version, tag: Tag) -> str:
45+
"""Replaces the cpython wheel file with a ABI3 compatible wheel"""
46+
47+
if tag.abi == "abi3":
48+
# Nothing to do.
49+
return wheel_file
50+
51+
check_is_abi3_compatible(wheel_file)
52+
53+
abi3_tag = Tag(tag.interpreter, "abi3", tag.platform)
54+
55+
dirname = os.path.dirname(wheel_file)
56+
new_wheel_file = os.path.join(
57+
dirname,
58+
f"{name}-{version}-{abi3_tag}.whl",
59+
)
60+
61+
os.rename(wheel_file, new_wheel_file)
62+
63+
print("Renamed wheel to", new_wheel_file)
64+
65+
return new_wheel_file
66+
67+
68+
def main(wheel_file: str, dest_dir: str, archs: Optional[str]) -> None:
69+
"""Entry point"""
70+
71+
# Parse the wheel file name into its parts. Note that `parse_wheel_filename`
72+
# normalizes the package name (i.e. it converts matrix_synapse ->
73+
# matrix-synapse), which is not what we want.
74+
_, version, build, tags = parse_wheel_filename(os.path.basename(wheel_file))
75+
name = os.path.basename(wheel_file).split("-")[0]
76+
77+
if len(tags) != 1:
78+
# We expect only a wheel file with only a single tag
79+
raise Exception(f"Unexpectedly found multiple tags: {tags}")
80+
81+
tag = next(iter(tags))
82+
83+
if build:
84+
# We don't use build tags in Synapse
85+
raise Exception(f"Unexpected build tag: {build}")
86+
87+
# If the wheel is for cpython then convert it into an abi3 wheel.
88+
if tag.interpreter.startswith("cp"):
89+
wheel_file = cpython(wheel_file, name, version, tag)
90+
91+
# Finally, repair the wheel.
92+
if archs is not None:
93+
# If we are given archs then we are on macos and need to use
94+
# `delocate-listdeps`.
95+
subprocess.run(["delocate-listdeps", wheel_file], check=True)
96+
subprocess.run(
97+
["delocate-wheel", "--require-archs", archs, "-w", dest_dir, wheel_file],
98+
check=True,
99+
)
100+
else:
101+
subprocess.run(["auditwheel", "repair", "-w", dest_dir, wheel_file], check=True)
102+
103+
104+
if __name__ == "__main__":
105+
parser = argparse.ArgumentParser(description="Tag wheel as abi3 and repair it.")
106+
107+
parser.add_argument(
108+
"--wheel-dir",
109+
"-w",
110+
metavar="WHEEL_DIR",
111+
help="Directory to store delocated wheels",
112+
required=True,
113+
)
114+
115+
parser.add_argument(
116+
"--require-archs",
117+
metavar="archs",
118+
default=None,
119+
)
120+
121+
parser.add_argument(
122+
"wheel_file",
123+
metavar="WHEEL_FILE",
124+
)
125+
126+
args = parser.parse_args()
127+
128+
wheel_file = args.wheel_file
129+
wheel_dir = args.wheel_dir
130+
archs = args.require_archs
131+
132+
main(wheel_file, wheel_dir, archs)

.ci/scripts/calculate_jobs.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,13 @@
1818
import json
1919
import os
2020

21+
22+
def set_output(key: str, value: str):
23+
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
24+
with open(os.environ["GITHUB_OUTPUT"], "at") as f:
25+
print(f"{key}={value}", file=f)
26+
27+
2128
IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
2229

2330
# First calculate the various trial jobs.
@@ -39,7 +46,7 @@
3946
"database": "sqlite",
4047
"extras": "all",
4148
}
42-
for version in ("3.8", "3.9", "3.10")
49+
for version in ("3.8", "3.9", "3.10", "3.11")
4350
)
4451

4552

@@ -55,7 +62,7 @@
5562
if not IS_PR:
5663
trial_postgres_tests.append(
5764
{
58-
"python-version": "3.10",
65+
"python-version": "3.11",
5966
"database": "postgres",
6067
"postgres-version": "14",
6168
"extras": "all",
@@ -81,7 +88,7 @@
8188
test_matrix = json.dumps(
8289
trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests
8390
)
84-
print(f"::set-output name=trial_test_matrix::{test_matrix}")
91+
set_output("trial_test_matrix", test_matrix)
8592

8693

8794
# First calculate the various sytest jobs.
@@ -125,4 +132,4 @@
125132
print("::endgroup::")
126133

127134
test_matrix = json.dumps(sytest_tests)
128-
print(f"::set-output name=sytest_test_matrix::{test_matrix}")
135+
set_output("sytest_test_matrix", test_matrix)

.ci/scripts/setup_complement_prerequisites.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ endblock
2121

2222
block Install Complement Dependencies
2323
sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev
24-
go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
24+
go get -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
2525
endblock
2626

2727
block Install custom gotestfmt template

.flake8

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,11 @@
88
# E203: whitespace before ':' (which is contrary to pep8?)
99
# E731: do not assign a lambda expression, use a def
1010
# E501: Line too long (black enforces this for us)
11-
ignore=W503,W504,E203,E731,E501
11+
#
12+
# flake8-bugbear runs extra checks. Its error codes are described at
13+
# https://github.com/PyCQA/flake8-bugbear#list-of-warnings
14+
# B019: Use of functools.lru_cache or functools.cache on methods can lead to memory leaks
15+
# B023: Functions defined inside a loop must not use variables redefined in the loop
16+
# B024: Abstract base class with no abstract method.
17+
18+
ignore=W503,W504,E203,E731,E501,B019,B023,B024

.github/workflows/docs.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ jobs:
2020
- uses: actions/checkout@v3
2121

2222
- name: Setup mdbook
23-
uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
23+
uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
2424
with:
2525
mdbook-version: '0.4.17'
2626

@@ -54,11 +54,11 @@ jobs:
5454
esac
5555
5656
# finally, set the 'branch-version' var.
57-
echo "::set-output name=branch-version::$branch"
57+
echo "branch-version=$branch" >> "$GITHUB_OUTPUT"
5858
5959
# Deploy to the target directory.
6060
- name: Deploy to gh pages
61-
uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
61+
uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935 # v3.9.0
6262
with:
6363
github_token: ${{ secrets.GITHUB_TOKEN }}
6464
publish_dir: ./book

.github/workflows/latest_deps.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ jobs:
7676
-e POSTGRES_PASSWORD=postgres \
7777
-e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
7878
postgres:${{ matrix.postgres-version }}
79-
- uses: actions/setup-python@v2
79+
- uses: actions/setup-python@v4
8080
with:
8181
python-version: "3.x"
8282
- run: pip install .[all,test]

.github/workflows/release-artifacts.yml

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,17 @@ jobs:
2626
runs-on: ubuntu-latest
2727
steps:
2828
- uses: actions/checkout@v3
29-
- uses: actions/setup-python@v2
29+
- uses: actions/setup-python@v4
30+
with:
31+
python-version: '3.x'
3032
- id: set-distros
3133
run: |
3234
# if we're running from a tag, get the full list of distros; otherwise just use debian:sid
3335
dists='["debian:sid"]'
3436
if [[ $GITHUB_REF == refs/tags/* ]]; then
3537
dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
3638
fi
37-
echo "::set-output name=distros::$dists"
39+
echo "distros=$dists" >> "$GITHUB_OUTPUT"
3840
# map the step outputs to job outputs
3941
outputs:
4042
distros: ${{ steps.set-distros.outputs.distros }}
@@ -69,7 +71,9 @@ jobs:
6971
${{ runner.os }}-buildx-
7072
7173
- name: Set up python
72-
uses: actions/setup-python@v2
74+
uses: actions/setup-python@v4
75+
with:
76+
python-version: '3.x'
7377

7478
- name: Build the packages
7579
# see https://github.com/docker/build-push-action/issues/252
@@ -91,39 +95,63 @@ jobs:
9195
path: debs/*
9296

9397
build-wheels:
94-
name: Build wheels on ${{ matrix.os }}
98+
name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
9599
runs-on: ${{ matrix.os }}
96100
strategy:
97101
matrix:
98-
os: [ubuntu-20.04, macos-10.15]
102+
os: [ubuntu-20.04, macos-11]
103+
arch: [x86_64, aarch64]
104+
# is_pr is a flag used to exclude certain jobs from the matrix on PRs.
105+
# It is not read by the rest of the workflow.
99106
is_pr:
100107
- ${{ startsWith(github.ref, 'refs/pull/') }}
101108

102109
exclude:
103110
# Don't build macos wheels on PR CI.
104111
- is_pr: true
105-
os: "macos-10.15"
112+
os: "macos-11"
113+
# Don't build aarch64 wheels on mac.
114+
- os: "macos-11"
115+
arch: aarch64
116+
# Don't build aarch64 wheels on PR CI.
117+
- is_pr: true
118+
arch: aarch64
106119

107120
steps:
108121
- uses: actions/checkout@v3
109122

110-
- uses: actions/setup-python@v3
123+
- uses: actions/setup-python@v4
124+
with:
125+
# setup-python@v4 doesn't impose a default python version. Need to use 3.x
126+
# here, because `python` on osx points to Python 2.7.
127+
python-version: "3.x"
111128

112129
- name: Install cibuildwheel
113130
run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0
114131

115-
# Only build a single wheel in CI.
116-
- name: Set env vars.
117-
run: |
118-
echo "CIBW_BUILD="cp37-manylinux_x86_64"" >> $GITHUB_ENV
132+
- name: Set up QEMU to emulate aarch64
133+
if: matrix.arch == 'aarch64'
134+
uses: docker/setup-qemu-action@v2
135+
with:
136+
platforms: arm64
137+
138+
- name: Build aarch64 wheels
139+
if: matrix.arch == 'aarch64'
140+
run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV
141+
142+
- name: Only build a single wheel on PR
119143
if: startsWith(github.ref, 'refs/pull/')
144+
run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
120145

121146
- name: Build wheels
122147
run: python -m cibuildwheel --output-dir wheelhouse
123148
env:
124149
# Skip testing for platforms which various libraries don't have wheels
125150
# for, and so need extra build deps.
126151
CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx*
152+
# Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583
153+
CARGO_NET_GIT_FETCH_WITH_CLI: true
154+
CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
127155

128156
- uses: actions/upload-artifact@v3
129157
with:

.github/workflows/tests.yml

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
runs-on: ubuntu-latest
3333
steps:
3434
- uses: actions/checkout@v3
35-
- uses: actions/setup-python@v2
35+
- uses: actions/setup-python@v4
3636
- uses: matrix-org/setup-python-poetry@v1
3737
with:
3838
extras: "all"
@@ -43,7 +43,7 @@ jobs:
4343
runs-on: ubuntu-latest
4444
steps:
4545
- uses: actions/checkout@v3
46-
- uses: actions/setup-python@v2
46+
- uses: actions/setup-python@v4
4747
- run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
4848
- run: scripts-dev/check_schema_delta.py --force-colors
4949

@@ -67,7 +67,7 @@ jobs:
6767
with:
6868
ref: ${{ github.event.pull_request.head.sha }}
6969
fetch-depth: 0
70-
- uses: actions/setup-python@v2
70+
- uses: actions/setup-python@v4
7171
- run: "pip install 'towncrier>=18.6.0rc1'"
7272
- run: scripts-dev/check-newsfragment.sh
7373
env:
@@ -142,7 +142,7 @@ jobs:
142142
runs-on: ubuntu-latest
143143
steps:
144144
- uses: actions/checkout@v3
145-
- uses: actions/setup-python@v2
145+
- uses: actions/setup-python@v4
146146
- id: get-matrix
147147
run: .ci/scripts/calculate_jobs.py
148148
outputs:
@@ -167,6 +167,14 @@ jobs:
167167
-e POSTGRES_PASSWORD=postgres \
168168
-e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
169169
postgres:${{ matrix.job.postgres-version }}
170+
171+
- name: Install Rust
172+
uses: actions-rs/toolchain@v1
173+
with:
174+
toolchain: 1.58.1
175+
override: true
176+
- uses: Swatinem/rust-cache@v2
177+
170178
- uses: matrix-org/setup-python-poetry@v1
171179
with:
172180
python-version: ${{ matrix.job.python-version }}
@@ -385,7 +393,7 @@ jobs:
385393
- python-version: "3.7"
386394
postgres-version: "10"
387395

388-
- python-version: "3.10"
396+
- python-version: "3.11"
389397
postgres-version: "14"
390398

391399
services:

0 commit comments

Comments
 (0)