Skip to content

Commit f6c2a19

Browse files
nico-famedlynico-famedly
committed
ci: Basic code coverage setup
Currently only uploads coverage for the unittests run via trial. In theory we could calculate coverage also from sytest and complement. The builtin coverage of trial tends to error out when trying to calculate the coverage of the html files, because it doesn't distinguish which files got opened. For this reason we are using coverage.py, but that doesn't work when using multiple trial jobs in parallel (unless putting in additional work), so this currently uses no concurrency. I disabled some jobs around the rust parts of synapse for now as those would take more work to set up.
1 parent d0f7f08 commit f6c2a19

File tree

5 files changed

+462
-16
lines changed

5 files changed

+462
-16
lines changed

‎.codecov.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
coverage:
2+
status:
3+
project:
4+
default:
5+
# Do not pass if PR reduces coverage by more than 5%
6+
target: auto
7+
threshold: 5
8+
comment:
9+
layout: "header, diff, components, files, footer"

‎.codecov.yml‎

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎.github/workflows/famedly-tests.yml‎

Lines changed: 373 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,373 @@
1+
name: Tests
2+
3+
on:
4+
push:
5+
pull_request:
6+
merge_group:
7+
workflow_dispatch:
8+
9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
check-sampleconfig:
15+
runs-on: ubuntu-latest
16+
17+
steps:
18+
- uses: actions/checkout@v4
19+
- uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
20+
- uses: matrix-org/setup-python-poetry@v1
21+
with:
22+
python-version: "3.x"
23+
poetry-version: "1.3.2"
24+
extras: "all"
25+
- run: poetry run scripts-dev/generate_sample_config.sh --check
26+
- run: poetry run scripts-dev/config-lint.sh
27+
28+
# We don't update develop currently
29+
#check-schema-delta:
30+
# runs-on: ubuntu-latest
31+
32+
# steps:
33+
# - uses: actions/checkout@v4
34+
# - uses: actions/setup-python@v5
35+
# with:
36+
# python-version: "3.x"
37+
# - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
38+
# - run: scripts-dev/check_schema_delta.py --force-colors
39+
40+
check-lockfile:
41+
runs-on: ubuntu-latest
42+
steps:
43+
- uses: actions/checkout@v4
44+
- uses: actions/setup-python@v5
45+
with:
46+
python-version: "3.x"
47+
- run: .ci/scripts/check_lockfile.py
48+
49+
lint:
50+
runs-on: ubuntu-latest
51+
52+
steps:
53+
- name: Checkout repository
54+
uses: actions/checkout@v4
55+
56+
- name: Setup Poetry
57+
uses: matrix-org/setup-python-poetry@v1
58+
with:
59+
install-project: "false"
60+
61+
- name: Run ruff check
62+
run: poetry run ruff check --output-format=github .
63+
64+
- name: Run ruff format
65+
run: poetry run ruff format --check .
66+
67+
lint-mypy:
68+
runs-on: ubuntu-latest
69+
name: Typechecking
70+
71+
steps:
72+
- name: Checkout repository
73+
uses: actions/checkout@v4
74+
75+
- uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
76+
77+
- name: Setup Poetry
78+
uses: matrix-org/setup-python-poetry@v1
79+
with:
80+
# We want to make use of type hints in optional dependencies too.
81+
extras: all
82+
# We have seen odd mypy failures that were resolved when we started
83+
# installing the project again:
84+
# https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
85+
# To make CI green, err towards caution and install the project.
86+
install-project: "true"
87+
88+
# Cribbed from
89+
# https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
90+
- name: Restore/persist mypy's cache
91+
uses: actions/cache@v4
92+
with:
93+
path: |
94+
.mypy_cache
95+
key: mypy-cache-${{ github.context.sha }}
96+
restore-keys: mypy-cache-
97+
98+
- name: Run mypy
99+
run: poetry run mypy
100+
101+
lint-crlf:
102+
runs-on: ubuntu-latest
103+
steps:
104+
- uses: actions/checkout@v4
105+
- name: Check line endings
106+
run: scripts-dev/check_line_terminators.sh
107+
108+
lint-pydantic:
109+
runs-on: ubuntu-latest
110+
111+
steps:
112+
- uses: actions/checkout@v4
113+
with:
114+
ref: ${{ github.event.pull_request.head.sha }}
115+
- uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
116+
- uses: matrix-org/setup-python-poetry@v1
117+
with:
118+
poetry-version: "1.3.2"
119+
extras: "all"
120+
- run: poetry run scripts-dev/check_pydantic_models.py
121+
122+
#lint-clippy:
123+
# runs-on: ubuntu-latest
124+
125+
# steps:
126+
# - uses: actions/checkout@v4
127+
128+
# - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
129+
130+
# - run: cargo clippy -- -D warnings
131+
132+
#lint-rustfmt:
133+
# runs-on: ubuntu-latest
134+
135+
# steps:
136+
# - uses: actions/checkout@v4
137+
138+
# - name: Install Rust
139+
# uses: dtolnay/rust-toolchain@master
140+
# with:
141+
# # We use nightly so that it correctly groups together imports
142+
# toolchain: nightly-2022-12-01
143+
# components: rustfmt
144+
# - uses: Swatinem/rust-cache@v2
145+
146+
# - run: cargo fmt --check
147+
148+
# This is to detect issues with the rst file, which can otherwise cause issues
149+
# when uploading packages to PyPi.
150+
lint-readme:
151+
runs-on: ubuntu-latest
152+
steps:
153+
- uses: actions/checkout@v4
154+
- uses: actions/setup-python@v5
155+
with:
156+
python-version: "3.x"
157+
- run: "pip install rstcheck"
158+
- run: "rstcheck --report-level=WARNING README.rst"
159+
160+
# Dummy step to gate other tests on without repeating the whole list
161+
linting-done:
162+
if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
163+
needs:
164+
- lint
165+
- lint-mypy
166+
- lint-crlf
167+
- lint-pydantic
168+
- check-sampleconfig
169+
#- check-schema-delta
170+
- check-lockfile
171+
#- lint-clippy
172+
#- lint-clippy-nightly
173+
#- lint-rustfmt
174+
- lint-readme
175+
runs-on: ubuntu-latest
176+
steps:
177+
- run: echo "done"
178+
179+
calculate-test-jobs:
180+
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
181+
needs: linting-done
182+
runs-on: ubuntu-latest
183+
steps:
184+
- uses: actions/checkout@v4
185+
- uses: actions/setup-python@v5
186+
with:
187+
python-version: "3.x"
188+
- id: get-matrix
189+
run: .ci/scripts/calculate_jobs.py
190+
outputs:
191+
trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
192+
sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
193+
194+
trial:
195+
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
196+
needs:
197+
- calculate-test-jobs
198+
runs-on: ubuntu-latest
199+
strategy:
200+
matrix:
201+
job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}
202+
env:
203+
TOP: ${{ github.workspace }}
204+
205+
steps:
206+
- uses: actions/checkout@v4
207+
- run: sudo apt-get -qq install xmlsec1
208+
- name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
209+
if: ${{ matrix.job.postgres-version }}
210+
# 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer.
211+
# 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections.
212+
run: |
213+
docker run -d -p 5432:5432 \
214+
--tmpfs /var/lib/postgres:rw,size=6144m \
215+
--mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \
216+
-e POSTGRES_PASSWORD=postgres \
217+
-e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
218+
postgres:${{ matrix.job.postgres-version }}
219+
220+
- uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
221+
222+
- uses: matrix-org/setup-python-poetry@v1
223+
with:
224+
python-version: ${{ matrix.job.python-version }}
225+
poetry-version: "1.3.2"
226+
extras: ${{ matrix.job.extras }}
227+
- name: Await PostgreSQL
228+
if: ${{ matrix.job.postgres-version }}
229+
timeout-minutes: 2
230+
run: until pg_isready -h localhost; do sleep 1; done
231+
- run: python -m pip install --user coverage
232+
- run: poetry run coverage run -m twisted.trial tests
233+
# can't run parallel jobs as that breaks coverage
234+
env:
235+
SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }}
236+
SYNAPSE_POSTGRES_HOST: /var/run/postgresql
237+
SYNAPSE_POSTGRES_USER: postgres
238+
SYNAPSE_POSTGRES_PASSWORD: postgres
239+
- name: Dump logs
240+
# Logs are most useful when the command fails, always include them.
241+
if: ${{ always() }}
242+
# Note: Dumps to workflow logs instead of using actions/upload-artifact
243+
# This keeps logs colocated with failing jobs
244+
# It also ignores find's exit code; this is a best effort affair
245+
run: >-
246+
find _trial_temp -name '*.log'
247+
-exec echo "::group::{}" \;
248+
-exec cat {} \;
249+
-exec echo "::endgroup::" \;
250+
|| true
251+
- run: poetry run coverage combine
252+
- name: Codecov - Upload coverage
253+
uses: codecov/codecov-action@v4
254+
with:
255+
token: ${{secrets.CODECOV_TOKEN}}
256+
257+
sytest:
258+
if: ${{ !failure() && !cancelled() }}
259+
needs:
260+
- calculate-test-jobs
261+
runs-on: ubuntu-latest
262+
container:
263+
image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
264+
volumes:
265+
- ${{ github.workspace }}:/src
266+
env:
267+
# If this is a pull request to a release branch, use that branch as default branch for sytest, else use develop
268+
# This works because the release script always create a branch on the sytest repo with the same name as the release branch
269+
SYTEST_DEFAULT_BRANCH: ${{ startsWith(github.base_ref, 'release-') && github.base_ref || 'develop' }}
270+
SYTEST_BRANCH: ${{ github.head_ref }}
271+
POSTGRES: ${{ matrix.job.postgres && 1}}
272+
MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
273+
ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
274+
WORKERS: ${{ matrix.job.workers && 1 }}
275+
BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
276+
TOP: ${{ github.workspace }}
277+
278+
strategy:
279+
fail-fast: false
280+
matrix:
281+
job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}
282+
283+
steps:
284+
- uses: actions/checkout@v4
285+
- name: Prepare test blacklist
286+
run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
287+
288+
- uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
289+
290+
- name: Run SyTest
291+
run: /bootstrap.sh synapse
292+
working-directory: /src
293+
- name: Summarise results.tap
294+
if: ${{ always() }}
295+
run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
296+
- name: Upload SyTest logs
297+
uses: actions/upload-artifact@v4
298+
if: ${{ always() }}
299+
with:
300+
name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
301+
path: |
302+
/logs/results.tap
303+
/logs/**/*.log*
304+
305+
complement:
306+
if: "${{ !failure() && !cancelled() }}"
307+
needs:
308+
- linting-done
309+
runs-on: ubuntu-latest
310+
311+
strategy:
312+
fail-fast: false
313+
matrix:
314+
include:
315+
- arrangement: monolith
316+
database: SQLite
317+
318+
- arrangement: monolith
319+
database: Postgres
320+
321+
- arrangement: workers
322+
database: Postgres
323+
324+
steps:
325+
- name: Run actions/checkout@v4 for synapse
326+
uses: actions/checkout@v4
327+
with:
328+
path: synapse
329+
330+
- uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
331+
332+
- name: Prepare Complement's Prerequisites
333+
run: synapse/.ci/scripts/setup_complement_prerequisites.sh
334+
335+
- uses: actions/setup-go@v5
336+
with:
337+
cache-dependency-path: complement/go.sum
338+
go-version-file: complement/go.mod
339+
340+
# use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once.
341+
- run: |
342+
set -o pipefail
343+
COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt
344+
shell: bash
345+
env:
346+
POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
347+
WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
348+
name: Run Complement Tests
349+
350+
cargo-test:
351+
runs-on: ubuntu-latest
352+
needs:
353+
- linting-done
354+
355+
steps:
356+
- uses: actions/checkout@v4
357+
358+
- uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352
359+
360+
- run: cargo test
361+
362+
# a job which marks all the other jobs as complete, thus allowing PRs to be merged.
363+
tests-done:
364+
if: ${{ always() }}
365+
needs:
366+
- trial
367+
- sytest
368+
- complement
369+
- cargo-test
370+
- linting-done
371+
runs-on: ubuntu-latest
372+
steps:
373+
- run: echo "done"

0 commit comments

Comments
 (0)