Skip to content

Commit 6f15ad5

Browse files
nshirleynshirley
authored
test(e2e): run integration and e2e tests with pytest (#1697)
- Update docker compose steps for mysql and spanner to use pytest - Add infra and configuration for pytest to run tests - Remove old "run.py" test setup Closes STOR-235
1 parent 4ba7b32 commit 6f15ad5

15 files changed

+550
-560
lines changed

‎.circleci/config.yml‎

Lines changed: 39 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ commands:
101101
steps:
102102
- run:
103103
name: Install test dependencies
104-
command: cargo install cargo-nextest cargo-llvm-cov
104+
command: cargo install --locked cargo-nextest cargo-llvm-cov
105105

106106
make-test-dir:
107107
steps:
@@ -145,34 +145,18 @@ commands:
145145
make run_token_server_integration_tests
146146
environment:
147147
SYNCSTORAGE_RS_IMAGE: app:build
148-
run-e2e-mysql-tests:
149-
steps:
150-
- run:
151-
name: e2e tests (syncstorage mysql)
152-
command: >
153-
/usr/local/bin/docker-compose
154-
-f docker-compose.mysql.yaml
155-
-f docker-compose.e2e.mysql.yaml
156-
up
157-
--exit-code-from mysql-e2e-tests
158-
--abort-on-container-exit
159-
environment:
160-
SYNCSTORAGE_RS_IMAGE: app:build
161-
162-
163-
run-e2e-spanner-tests:
148+
run-e2e-tests:
149+
parameters:
150+
db:
151+
type: enum
152+
enum: ["mysql", "spanner"]
164153
steps:
165154
- run:
166-
name: e2e tests (syncstorage spanner)
155+
name: e2e tests (syncstorage << parameters.db >>)
167156
command: >
168-
/usr/local/bin/docker-compose
169-
-f docker-compose.spanner.yaml
170-
-f docker-compose.e2e.spanner.yaml
171-
up
172-
--exit-code-from spanner-e2e-tests
173-
--abort-on-container-exit
157+
make docker_run_<< parameters.db >>_e2e_tests
174158
environment:
175-
SYNCSTORAGE_RS_IMAGE: app:build
159+
SYNCSTORAGE_RS_IMAGE: app:build
176160

177161
upload-to-gcs:
178162
parameters:
@@ -283,7 +267,6 @@ jobs:
283267
# if the above tests don't run tokenserver-db tests (i.e. using --workspace)
284268
# then run-tokenserver-scripts-tests will fail. These tests expect the db to be
285269
# configured already, and it appears unit-tests modify the db to the expected state
286-
- run-tokenserver-integration-tests
287270
- store-test-results
288271
- upload-to-gcs:
289272
source: workflow/test-results
@@ -324,11 +307,13 @@ jobs:
324307
- run:
325308
name: Save docker-compose config
326309
command: cp docker-compose*mysql.yaml /home/circleci/cache
310+
- run:
311+
name: Save Makefile to cache
312+
command: cp Makefile /home/circleci/cache
327313
- save_cache:
328314
key: mysql-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }}
329315
paths:
330316
- /home/circleci/cache
331-
332317
build-spanner-image:
333318
docker:
334319
- image: cimg/rust:1.86 # RUST_VER
@@ -361,6 +346,9 @@ jobs:
361346
- run:
362347
name: Save docker-compose config
363348
command: cp docker-compose*spanner.yaml /home/circleci/cache
349+
- run:
350+
name: Save Makefile to cache
351+
command: cp Makefile /home/circleci/cache
364352
- save_cache:
365353
key: spanner-{{ .Branch }}-{{ .Environment.CIRCLE_SHA1 }}-{{ epoch }}
366354
paths:
@@ -422,7 +410,7 @@ jobs:
422410

423411
mysql-e2e-tests:
424412
docker:
425-
- image: docker/compose:1.24.0
413+
- image: cimg/base:2025.04
426414
auth:
427415
username: $DOCKER_USER
428416
password: $DOCKER_PASS
@@ -434,14 +422,24 @@ jobs:
434422
- run:
435423
name: Restore Docker image cache
436424
command: docker load -i /home/circleci/cache/docker.tar
425+
- run:
426+
name: Restore Makefile from save_cache
427+
command: cp /home/circleci/cache/Makefile .
437428
- run:
438429
name: Restore docker-compose config
439430
command: cp /home/circleci/cache/docker-compose*.yaml .
440-
- run-e2e-mysql-tests
431+
- make-test-dir
432+
- run-e2e-tests:
433+
db: mysql
434+
- store-test-results
435+
- upload-to-gcs:
436+
source: workflow/test-results
437+
destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
438+
extension: xml
441439

442440
spanner-e2e-tests:
443441
docker:
444-
- image: docker/compose:1.24.0
442+
- image: cimg/base:2025.04
445443
auth:
446444
username: $DOCKER_USER
447445
password: $DOCKER_PASS
@@ -453,10 +451,20 @@ jobs:
453451
- run:
454452
name: Restore Docker image cache
455453
command: docker load -i /home/circleci/cache/docker.tar
454+
- run:
455+
name: Restore Makefile from save_cache
456+
command: cp /home/circleci/cache/Makefile .
456457
- run:
457458
name: Restore docker-compose config
458459
command: cp /home/circleci/cache/docker-compose*.yaml .
459-
- run-e2e-spanner-tests
460+
- make-test-dir
461+
- run-e2e-tests:
462+
db: spanner
463+
- store-test-results
464+
- upload-to-gcs:
465+
source: workflow/test-results
466+
destination: gs://ecosystem-test-eng-metrics/syncstorage-rs/junit
467+
extension: xml
460468

461469
deploy:
462470
docker:

‎Makefile‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,13 @@ TEST_PROFILE := $(if $(CIRCLECI),ci,default)
2020
TEST_FILE_PREFIX := $(if $(CIRCLECI),$(CIRCLE_BUILD_NUM)__$(EPOCH_TIME)__$(CIRCLE_PROJECT_REPONAME)__$(WORKFLOW)__)
2121
UNIT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__results.xml
2222
UNIT_COVERAGE_JSON := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)unit__coverage.json
23-
INTEGRATION_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)integration__results.xml
23+
24+
SPANNER_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)spanner_integration__results.xml
25+
SPANNER_NO_JWK_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)spanner_no_oauth_integration__results.xml
26+
MYSQL_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)mysql_integration__results.xml
27+
MYSQL_NO_JWK_INT_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)mysql_no_oauth_integration__results.xml
28+
29+
LOCAL_INTEGRATION_JUNIT_XML := $(TEST_RESULTS_DIR)/$(TEST_FILE_PREFIX)local_integration__results.xml
2430
SYNC_SYNCSTORAGE__DATABASE_URL ?= mysql://sample_user:sample_password@localhost/syncstorage_rs
2531
SYNC_TOKENSERVER__DATABASE_URL ?= mysql://sample_user:sample_password@localhost/tokenserver_rs
2632

@@ -40,22 +46,48 @@ clean:
4046
rm -r venv
4147

4248
docker_start_mysql:
43-
docker-compose -f docker-compose.mysql.yaml up -d
49+
docker compose -f docker-compose.mysql.yaml up -d
4450

4551
docker_start_mysql_rebuild:
46-
docker-compose -f docker-compose.mysql.yaml up --build -d
52+
docker compose -f docker-compose.mysql.yaml up --build -d
4753

4854
docker_stop_mysql:
49-
docker-compose -f docker-compose.mysql.yaml down
55+
docker compose -f docker-compose.mysql.yaml down
5056

5157
docker_start_spanner:
52-
docker-compose -f docker-compose.spanner.yaml up -d
58+
docker compose -f docker-compose.spanner.yaml up -d
5359

5460
docker_start_spanner_rebuild:
55-
docker-compose -f docker-compose.spanner.yaml up --build -d
61+
docker compose -f docker-compose.spanner.yaml up --build -d
5662

5763
docker_stop_spanner:
58-
docker-compose -f docker-compose.spanner.yaml down
64+
docker compose -f docker-compose.spanner.yaml down
65+
66+
.ONESHELL:
67+
docker_run_mysql_e2e_tests:
68+
docker compose \
69+
-f docker-compose.mysql.yaml \
70+
-f docker-compose.e2e.mysql.yaml \
71+
up \
72+
--exit-code-from mysql-e2e-tests \
73+
--abort-on-container-exit;
74+
exit_code=$$?;
75+
docker cp mysql-e2e-tests:/mysql_integration_results.xml ${MYSQL_INT_JUNIT_XML};
76+
docker cp mysql-e2e-tests:/mysql_no_jwk_integration_results.xml ${MYSQL_NO_JWK_INT_JUNIT_XML};
77+
exit $$exit_code;
78+
79+
.ONESHELL:
80+
docker_run_spanner_e2e_tests:
81+
docker compose \
82+
-f docker-compose.spanner.yaml \
83+
-f docker-compose.e2e.spanner.yaml \
84+
up \
85+
--exit-code-from spanner-e2e-tests \
86+
--abort-on-container-exit;
87+
exit_code=$$?;
88+
docker cp spanner-e2e-tests:/spanner_integration_results.xml ${SPANNER_INT_JUNIT_XML};
89+
docker cp spanner-e2e-tests:/spanner_no_jwk_integration_results.xml ${SPANNER_NO_JWK_INT_JUNIT_XML};
90+
exit $$exit_code;
5991

6092
python:
6193
python3 -m venv venv
@@ -104,4 +136,4 @@ merge_coverage_results:
104136
.ONESHELL:
105137
run_token_server_integration_tests:
106138
pip3 install -r tools/tokenserver/requirements.txt
107-
pytest tools/tokenserver --junit-xml=${INTEGRATION_JUNIT_XML}
139+
pytest tools/tokenserver --junit-xml=${INTEGRATION_JUNIT_XML}

‎docker-compose.e2e.mysql.yaml‎

Lines changed: 18 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,24 @@
11
version: "3"
22
services:
3-
sync-db:
4-
tokenserver-db:
5-
syncserver:
6-
depends_on:
7-
- sync-db
8-
- tokenserver-db
9-
# TODO: either syncserver should retry the db connection
10-
# itself a few times or should include a wait-for-it.sh script
11-
# inside its docker that would do this for us.
12-
entrypoint: >
13-
/bin/sh -c "
14-
sleep 15;
15-
/app/bin/syncserver;
16-
"
173
mysql-e2e-tests:
4+
container_name: mysql-e2e-tests
185
depends_on:
19-
- mock-fxa-server
20-
- syncserver
6+
sync-db:
7+
condition: service_healthy
8+
mock-fxa-server:
9+
condition: service_started
10+
tokenserver-db:
11+
condition: service_healthy
12+
# this depend is to avoid migration collisions.
13+
# the syncserver isn't actually used for the tests,
14+
# but collisions can happen particularly in CI.
15+
syncserver:
16+
condition: service_started
2117
image: app:build
2218
privileged: true
2319
user: root
2420
environment:
21+
JWK_CACHE_DISABLED: false
2522
MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000
2623
SYNC_HOST: 0.0.0.0
2724
SYNC_MASTER_SECRET: secret0
@@ -43,5 +40,9 @@ services:
4340
TOKENSERVER_HOST: http://localhost:8000
4441
entrypoint: >
4542
/bin/sh -c "
46-
sleep 28; python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
43+
exit_code=0;
44+
pytest /app/tools/integration_tests/ --junit-xml=/mysql_integration_results.xml || exit_code=$$?;
45+
export JWK_CACHE_DISABLED=true;
46+
pytest /app/tools/integration_tests/ --junit-xml=/mysql_no_jwk_integration_results.xml || exit_code=$$?;
47+
exit $$exit_code;
4748
"

‎docker-compose.e2e.spanner.yaml‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,23 @@
11
version: "3"
22
services:
3-
sync-db:
4-
sync-db-setup:
5-
tokenserver-db:
6-
syncserver:
7-
depends_on:
8-
- sync-db-setup
9-
# TODO: either syncserver should retry the db connection
10-
# itself a few times or should include a wait-for-it.sh script
11-
# inside its docker that would do this for us.
12-
entrypoint: >
13-
/bin/sh -c "
14-
sleep 15;
15-
/app/bin/syncserver;
16-
"
173
spanner-e2e-tests:
4+
container_name: spanner-e2e-tests
185
depends_on:
19-
- mock-fxa-server
20-
- syncserver
6+
mock-fxa-server:
7+
condition: service_started
8+
syncserver:
9+
condition: service_started
10+
tokenserver-db:
11+
condition: service_healthy
2112
image: app:build
2213
privileged: true
2314
user: root
2415
environment:
16+
# Some tests can run without the `FXA_OAUTH...` vars.
17+
# Setting this to false will delete any of those keys before starting
18+
# the syncserver and startging the test. This can be set/passed
19+
# in from CircleCI when calling `docker-compose -f docker-compose.e2e.spanner.yaml`
20+
JWK_CACHE_DISABLED: false
2521
MOCK_FXA_SERVER_URL: http://mock-fxa-server:6000
2622
SYNC_HOST: 0.0.0.0
2723
SYNC_MASTER_SECRET: secret0
@@ -44,5 +40,9 @@ services:
4440
TOKENSERVER_HOST: http://localhost:8000
4541
entrypoint: >
4642
/bin/sh -c "
47-
sleep 28; python3 /app/tools/integration_tests/run.py 'http://localhost:8000#secret0'
43+
exit_code=0;
44+
pytest /app/tools/integration_tests/ --junit-xml=/spanner_integration_results.xml || exit_code=$$?;
45+
export JWK_CACHE_DISABLED=true;
46+
pytest /app/tools/integration_tests/ --junit-xml=/spanner_no_jwk_integration_results.xml || exit_code=$$?;
47+
exit $$exit_code;
4848
"

‎docker-compose.mysql.yaml‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,12 @@ services:
2424
MYSQL_DATABASE: syncstorage
2525
MYSQL_USER: test
2626
MYSQL_PASSWORD: test
27+
healthcheck:
28+
test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"]
29+
interval: 2s
30+
retries: 10
31+
start_period: 20s
32+
timeout: 2s
2733

2834
tokenserver-db:
2935
image: docker.io/library/mysql:5.7
@@ -39,6 +45,12 @@ services:
3945
MYSQL_DATABASE: tokenserver
4046
MYSQL_USER: test
4147
MYSQL_PASSWORD: test
48+
healthcheck:
49+
test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"]
50+
interval: 2s
51+
retries: 10
52+
start_period: 20s
53+
timeout: 2s
4254

4355
mock-fxa-server:
4456
image: app:build
@@ -58,8 +70,10 @@ services:
5870
ports:
5971
- "8000:8000"
6072
depends_on:
61-
- sync-db
62-
- tokenserver-db
73+
sync-db:
74+
condition: service_healthy
75+
tokenserver-db:
76+
condition: service_healthy
6377
environment:
6478
SYNC_HOST: 0.0.0.0
6579
SYNC_MASTER_SECRET: secret0

‎docker-compose.spanner.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ services:
3535
MYSQL_DATABASE: tokenserver
3636
MYSQL_USER: test
3737
MYSQL_PASSWORD: test
38+
healthcheck:
39+
test: ["CMD-SHELL", "mysqladmin -uroot -p$${MYSQL_ROOT_PASSWORD} version"]
40+
interval: 2s
41+
retries: 10
42+
start_period: 20s
43+
timeout: 2s
3844
mock-fxa-server:
3945
image: app:build
4046
restart: "no"

0 commit comments

Comments
 (0)