Skip to content

Commit 7a922a1

Browse files
blink1073blink1073
committed
PYTHON-5216 Convert perf tests to use new scripts
1 parent 4353278 commit 7a922a1

File tree

11 files changed

+167
-126
lines changed

11 files changed

+167
-126
lines changed

‎.evergreen/config.yml‎

Lines changed: 0 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -304,17 +304,6 @@ functions:
304304
args:
305305
- ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
306306

307-
"run perf tests":
308-
- command: subprocess.exec
309-
type: test
310-
params:
311-
working_dir: "src"
312-
binary: bash
313-
include_expansions_in_env: [SUB_TEST_NAME]
314-
args:
315-
- .evergreen/scripts/run-with-env.sh
316-
- .evergreen/scripts/run-perf-tests.sh
317-
318307
"attach benchmark test results":
319308
- command: attach.results
320309
params:
@@ -461,84 +450,6 @@ tasks:
461450
commands:
462451
- func: "download and merge coverage"
463452

464-
- name: "perf-6.0-standalone"
465-
tags: ["perf"]
466-
commands:
467-
- func: "run server"
468-
vars:
469-
VERSION: "v6.0-perf"
470-
- func: "run perf tests"
471-
vars:
472-
SUB_TEST_NAME: "sync"
473-
- func: "attach benchmark test results"
474-
- func: "send dashboard data"
475-
476-
- name: "perf-6.0-standalone-ssl"
477-
tags: ["perf"]
478-
commands:
479-
- func: "run server"
480-
vars:
481-
VERSION: "v6.0-perf"
482-
SSL: "ssl"
483-
- func: "run perf tests"
484-
vars:
485-
SUB_TEST_NAME: "sync"
486-
- func: "attach benchmark test results"
487-
- func: "send dashboard data"
488-
489-
- name: "perf-8.0-standalone"
490-
tags: ["perf"]
491-
commands:
492-
- func: "run server"
493-
vars:
494-
VERSION: "8.0"
495-
- func: "run perf tests"
496-
vars:
497-
SUB_TEST_NAME: "sync"
498-
- func: "attach benchmark test results"
499-
- func: "send dashboard data"
500-
501-
- name: "perf-6.0-standalone-async"
502-
tags: [ "perf" ]
503-
commands:
504-
- func: "run server"
505-
vars:
506-
VERSION: "v6.0-perf"
507-
TOPOLOGY: "server"
508-
- func: "run perf tests"
509-
vars:
510-
SUB_TEST_NAME: "async"
511-
- func: "attach benchmark test results"
512-
- func: "send dashboard data"
513-
514-
- name: "perf-6.0-standalone-ssl-async"
515-
tags: [ "perf" ]
516-
commands:
517-
- func: "run server"
518-
vars:
519-
VERSION: "v6.0-perf"
520-
TOPOLOGY: "server"
521-
SSL: "ssl"
522-
- func: "run perf tests"
523-
vars:
524-
SUB_TEST_NAME: "async"
525-
- func: "attach benchmark test results"
526-
- func: "send dashboard data"
527-
528-
- name: "perf-8.0-standalone-async"
529-
tags: [ "perf" ]
530-
commands:
531-
- func: "run server"
532-
vars:
533-
VERSION: "8.0"
534-
TOPOLOGY: "server"
535-
- func: "run perf tests"
536-
vars:
537-
SUB_TEST_NAME: "async"
538-
- func: "attach benchmark test results"
539-
- func: "send dashboard data"
540-
541-
542453
- name: "check-import-time"
543454
tags: ["pr"]
544455
commands:
@@ -610,15 +521,3 @@ buildvariants:
610521
- rhel8.7-small
611522
tasks:
612523
- name: "backport-pr"
613-
614-
- name: "perf-tests"
615-
display_name: "Performance Benchmarks"
616-
batchtime: 10080 # 7 days
617-
run_on: rhel90-dbx-perf-large
618-
tasks:
619-
- name: "perf-6.0-standalone"
620-
- name: "perf-6.0-standalone-ssl"
621-
- name: "perf-8.0-standalone"
622-
- name: "perf-6.0-standalone-async"
623-
- name: "perf-6.0-standalone-ssl-async"
624-
- name: "perf-8.0-standalone-async"

‎.evergreen/generated_configs/tasks.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1151,6 +1151,112 @@ tasks:
11511151
SUB_TEST_NAME: gke
11521152
tags: [auth_oidc, auth_oidc_remote]
11531153

1154+
# Perf tests
1155+
- name: perf-6.0-standalone-ssl
1156+
commands:
1157+
- func: run server
1158+
vars:
1159+
VERSION: v6.0-perf
1160+
SSL: ssl
1161+
- func: run tests
1162+
vars:
1163+
TEST_NAME: perf
1164+
SUB_TEST_NAME: sync
1165+
- func: attach benchmark test results
1166+
- func: send dashboard data
1167+
tags: [perf]
1168+
- name: perf-6.0-standalone-ssl-async
1169+
commands:
1170+
- func: run server
1171+
vars:
1172+
VERSION: v6.0-perf
1173+
SSL: ssl
1174+
- func: run tests
1175+
vars:
1176+
TEST_NAME: perf
1177+
SUB_TEST_NAME: async
1178+
- func: attach benchmark test results
1179+
- func: send dashboard data
1180+
tags: [perf]
1181+
- name: perf-6.0-standalone
1182+
commands:
1183+
- func: run server
1184+
vars:
1185+
VERSION: v6.0-perf
1186+
SSL: nossl
1187+
- func: run tests
1188+
vars:
1189+
TEST_NAME: perf
1190+
SUB_TEST_NAME: sync
1191+
- func: attach benchmark test results
1192+
- func: send dashboard data
1193+
tags: [perf]
1194+
- name: perf-6.0-standalone-async
1195+
commands:
1196+
- func: run server
1197+
vars:
1198+
VERSION: v6.0-perf
1199+
SSL: nossl
1200+
- func: run tests
1201+
vars:
1202+
TEST_NAME: perf
1203+
SUB_TEST_NAME: async
1204+
- func: attach benchmark test results
1205+
- func: send dashboard data
1206+
tags: [perf]
1207+
- name: perf-8.0-standalone-ssl
1208+
commands:
1209+
- func: run server
1210+
vars:
1211+
VERSION: v8.0-perf
1212+
SSL: ssl
1213+
- func: run tests
1214+
vars:
1215+
TEST_NAME: perf
1216+
SUB_TEST_NAME: sync
1217+
- func: attach benchmark test results
1218+
- func: send dashboard data
1219+
tags: [perf]
1220+
- name: perf-8.0-standalone-ssl-async
1221+
commands:
1222+
- func: run server
1223+
vars:
1224+
VERSION: v8.0-perf
1225+
SSL: ssl
1226+
- func: run tests
1227+
vars:
1228+
TEST_NAME: perf
1229+
SUB_TEST_NAME: async
1230+
- func: attach benchmark test results
1231+
- func: send dashboard data
1232+
tags: [perf]
1233+
- name: perf-8.0-standalone
1234+
commands:
1235+
- func: run server
1236+
vars:
1237+
VERSION: v8.0-perf
1238+
SSL: nossl
1239+
- func: run tests
1240+
vars:
1241+
TEST_NAME: perf
1242+
SUB_TEST_NAME: sync
1243+
- func: attach benchmark test results
1244+
- func: send dashboard data
1245+
tags: [perf]
1246+
- name: perf-8.0-standalone-async
1247+
commands:
1248+
- func: run server
1249+
vars:
1250+
VERSION: v8.0-perf
1251+
SSL: nossl
1252+
- func: run tests
1253+
vars:
1254+
TEST_NAME: perf
1255+
SUB_TEST_NAME: async
1256+
- func: attach benchmark test results
1257+
- func: send dashboard data
1258+
tags: [perf]
1259+
11541260
# Server tests
11551261
- name: test-4.0-standalone-auth-ssl-sync
11561262
commands:

‎.evergreen/generated_configs/variants.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -928,6 +928,15 @@ buildvariants:
928928
- windows-64-vsMulti-small
929929
batchtime: 10080
930930

931+
# Perf tests
932+
- name: performance-benchmarks
933+
tasks:
934+
- name: .perf
935+
display_name: Performance Benchmarks
936+
run_on:
937+
- rhel90-dbx-perf-large
938+
batchtime: 10080
939+
931940
# Pyopenssl tests
932941
- name: pyopenssl-macos-python3.9
933942
tasks:

‎.evergreen/run-perf-tests.sh‎

Lines changed: 0 additions & 19 deletions
This file was deleted.

‎.evergreen/scripts/generate_config.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ class Host:
6969
HOSTS["ubuntu20"] = Host("ubuntu20", "ubuntu2004-small", "Ubuntu-20", dict())
7070
HOSTS["ubuntu22"] = Host("ubuntu22", "ubuntu2204-small", "Ubuntu-22", dict())
7171
HOSTS["rhel7"] = Host("rhel7", "rhel79-small", "RHEL7", dict())
72+
HOSTS["perf"] = Host("perf", "rhel90-dbx-perf-large", "", dict())
7273
DEFAULT_HOST = HOSTS["rhel8"]
7374

7475
# Other hosts
@@ -722,6 +723,13 @@ def create_atlas_connect_variants():
722723
]
723724

724725

726+
def create_perf_variants():
727+
host = HOSTS["perf"]
728+
return [
729+
create_variant([".perf"], "Performance Benchmarks", host=host, batchtime=BATCHTIME_WEEK)
730+
]
731+
732+
725733
def create_aws_auth_variants():
726734
variants = []
727735

@@ -942,6 +950,26 @@ def create_enterprise_auth_tasks():
942950
return [EvgTask(name=task_name, tags=tags, commands=[server_func, assume_func, test_func])]
943951

944952

953+
def create_perf_tasks():
954+
tasks = []
955+
for version, ssl, sync in product(["6.0", "8.0"], ["ssl", "nossl"], ["sync", "async"]):
956+
vars = dict(VERSION=f"v{version}-perf", SSL=ssl)
957+
server_func = FunctionCall(func="run server", vars=vars)
958+
vars = dict(TEST_NAME="perf", SUB_TEST_NAME=sync)
959+
test_func = FunctionCall(func="run tests", vars=vars)
960+
attach_func = FunctionCall(func="attach benchmark test results")
961+
send_func = FunctionCall(func="send dashboard data")
962+
task_name = f"perf-{version}-standalone"
963+
if ssl == "ssl":
964+
task_name += "-ssl"
965+
if sync == "async":
966+
task_name += "-async"
967+
tags = ["perf"]
968+
commands = [server_func, test_func, attach_func, send_func]
969+
tasks.append(EvgTask(name=task_name, tags=tags, commands=commands))
970+
return tasks
971+
972+
945973
def create_ocsp_tasks():
946974
tasks = []
947975
tests = [

‎.evergreen/scripts/run-perf-tests.sh‎

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎.evergreen/scripts/setup_tests.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,14 @@ def handle_test_env() -> None:
362362
write_env("DISABLE_CONTEXT")
363363

364364
if test_name == "perf":
365+
run_command("git clone --depth 1 https://github.com/mongodb/specifications.git")
366+
data_dir = ROOT / "specifications/source/benchmarking/data"
367+
run_command("tar xf extended_bson.tgz", cwd=data_dir)
368+
run_command("tar xf parallel.tgz", cwd=data_dir)
369+
run_command("tar xf single_and_multi_document.tgz", cwd=data_dir)
370+
write_env("TEST_PATH", str(data_dir))
371+
write_env("OUTPUT_FILE", str(ROOT / "results.json"))
372+
365373
# PYTHON-4769 Run perf_test.py directly otherwise pytest's test collection negatively
366374
# affects the benchmark results.
367375
if sub_test_name == "sync":

‎.evergreen/scripts/teardown_tests.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
from __future__ import annotations
22

33
import os
4+
import shutil
45
import sys
6+
from pathlib import Path
57

6-
from utils import DRIVERS_TOOLS, LOGGER, run_command
8+
from utils import DRIVERS_TOOLS, LOGGER, ROOT, run_command
79

810
TEST_NAME = os.environ.get("TEST_NAME", "unconfigured")
911
SUB_TEST_NAME = os.environ.get("SUB_TEST_NAME")
@@ -44,6 +46,11 @@
4446
elif TEST_NAME == "auth_aws" and sys.platform != "darwin":
4547
run_command(f"bash {DRIVERS_TOOLS}/.evergreen/auth_aws/teardown.sh")
4648

49+
# Tear down perf if applicable.
50+
elif TEST_NAME == "perf":
51+
shutil.rmtree(ROOT / "specifications", ignore_errors=True)
52+
Path(os.environ["OUTPUT_FILE"]).unlink(missing_ok=True)
53+
4754
# Tear down mog_wsgi if applicable.
4855
elif TEST_NAME == "mod_wsgi":
4956
from mod_wsgi_tester import teardown_mod_wsgi

‎.evergreen/scripts/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ class Distro:
5050
}
5151

5252
# Tests that require a sub test suite.
53-
SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi"]
53+
SUB_TEST_REQUIRED = ["auth_aws", "auth_oidc", "kms", "mod_wsgi", "perf"]
5454

5555
EXTRA_TESTS = ["mod_wsgi"]
5656

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ expansion.yml
2727
*expansions.yml
2828
.evergreen/scripts/env.sh
2929
.evergreen/scripts/test-env.sh
30+
specifications/
3031

3132
# Lambda temp files
3233
test/lambda/.aws-sam

0 commit comments

Comments
 (0)