INTPYTHON-847 Add performance tests

.evergreen/config.yml

@@ -11,89 +11,8 @@ pre_timeout_secs: 1800 # 5 minutes
 post_error_fails_task: true
 post_timeout_secs: 1800 # 5 minutes
 
-functions:
-  "setup":
-    - command: git.get_project
-      params:
-        directory: src
-    - command: subprocess.exec
-      params:
-        binary: bash
-        working_dir: "src"
-        add_expansions_to_env: true
-        args:
-          - ./.evergreen/setup.sh
-    - command: expansions.update
-      params:
-        file: src/expansion.yml
-
-  "bootstrap mongo-orchestration":
-    - command: subprocess.exec
-      params:
-        binary: bash
-        add_expansions_to_env: true
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
-    - command: expansions.update
-      params:
-        file: mo-expansion.yml
-
-  "run unit tests":
-    - command: subprocess.exec
-      type: test
-      params:
-        binary: bash
-        working_dir: "src"
-        include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI"]
-        args:
-          - ./.evergreen/run-tests.sh
-
-  "teardown":
-    - command: subprocess.exec
-      params:
-        binary: bash
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/teardown.sh
-
-  # Encryption-specific functions
-  "start csfle servers":
-    - command: ec2.assume_role
-      params:
-        role_arn: ${aws_test_secrets_role}
-    - command: subprocess.exec
-      params:
-        binary: bash
-        include_expansions_in_env: [
-            "AWS_SECRET_ACCESS_KEY",
-            "AWS_ACCESS_KEY_ID",
-            "AWS_SESSION_TOKEN",
-        ]
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh
-
-  "teardown csfle":
-    - command: subprocess.exec
-      params:
-        binary: bash
-        args:
-          - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh
-
-  "run encryption tests":
-    - command: subprocess.exec
-      type: test
-      params:
-        binary: bash
-        working_dir: "src"
-        include_expansions_in_env: [
-            "AWS_KMS_ARN",
-            "DRIVERS_TOOLS",
-            "MONGODB_URI",
-            "DJANGO_SETTINGS_MODULE",
-            "CRYPT_SHARED_LIB_PATH",
-        ]
-        args:
-          - ./.evergreen/run-tests.sh
-          - encryption
+include:
+  - filename: .evergreen/functions.yml
 
 pre:
   - func: setup
@@ -106,12 +25,16 @@ tasks:
   - name: run-tests
     commands:
       - func: "run unit tests"
-
   - name: run-encryption-tests
     commands:
       - func: "start csfle servers"
       - func: "run encryption tests"
       - func: "teardown csfle"
+  - name: perf-tests
+    commands:
+      - func: "run performance tests"
+      - func: "attach performance test results"
+      - func: "send dashboard data"
 
 buildvariants:
   - name: tests-7-noauth-nossl
@@ -177,3 +100,11 @@ buildvariants:
       DJANGO_SETTINGS_MODULE: "encrypted_aws_settings"
     tasks:
       - name: run-encryption-tests
+
+  - name: performance-benchmarks
+    display_name: Performance Benchmarks
+    run_on:
+      - rhel90-dbx-perf-large
+    batchtime: 1440
+    tasks:
+      - name: perf-tests

.evergreen/functions.yml

@@ -0,0 +1,142 @@
+functions:
+  "setup":
+    - command: git.get_project
+      params:
+        directory: src
+    - command: subprocess.exec
+      params:
+        binary: bash
+        working_dir: "src"
+        add_expansions_to_env: true
+        args:
+          - ./.evergreen/setup.sh
+    - command: expansions.update
+      params:
+        file: src/expansion.yml
+
+  "bootstrap mongo-orchestration":
+    - command: subprocess.exec
+      params:
+        binary: bash
+        add_expansions_to_env: true
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
+    - command: expansions.update
+      params:
+        file: mo-expansion.yml
+
+  "run unit tests":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: "src"
+        include_expansions_in_env: ["DRIVERS_TOOLS", "MONGODB_URI"]
+        args:
+          - ./.evergreen/run-tests.sh
+
+  "teardown":
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/teardown.sh
+
+  # Encryption-specific functions
+  "start csfle servers":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
+    - command: subprocess.exec
+      params:
+        binary: bash
+        include_expansions_in_env: [
+          "AWS_SECRET_ACCESS_KEY",
+          "AWS_ACCESS_KEY_ID",
+          "AWS_SESSION_TOKEN",
+        ]
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/setup.sh
+
+  "teardown csfle":
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/teardown.sh
+
+  "run encryption tests":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: "src"
+        include_expansions_in_env: [
+          "AWS_KMS_ARN",
+          "DRIVERS_TOOLS",
+          "MONGODB_URI",
+          "DJANGO_SETTINGS_MODULE",
+          "CRYPT_SHARED_LIB_PATH",
+        ]
+        args:
+          - ./.evergreen/run-tests.sh
+          - encryption
+
+  # Performance test functions
+  "run performance tests":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        working_dir: "src"
+        include_expansions_in_env: [ "DRIVERS_TOOLS", "MONGODB_URI" ]
+        args:
+          - ./.evergreen/run-perf-tests.sh
+
+  "attach performance test results":
+    - command: attach.results
+      params:
+        file_location: src/report.json
+
+  "send dashboard data":
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/perf-submission-setup.sh
+        working_dir: src
+        include_expansions_in_env: [
+          "requester",
+          "revision_order_id",
+          "project_id",
+          "version_id",
+          "build_variant",
+          "parsed_order_id",
+          "task_name",
+          "task_id",
+          "execution",
+          "is_mainline"
+        ]
+      type: test
+    - command: expansions.update
+      params:
+        file: src/expansion.yml
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/perf-submission.sh
+        working_dir: src
+        include_expansions_in_env: [
+          "requester",
+          "revision_order_id",
+          "project_id",
+          "version_id",
+          "build_variant",
+          "parsed_order_id",
+          "task_name",
+          "task_id",
+          "execution",
+          "is_mainline"
+        ]
+      type: test

.evergreen/perf-submission-setup.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+# We use the requester expansion to determine whether the data is from a mainline evergreen run or not
+
+set -eu
+
+# shellcheck disable=SC2154
+if [ "${requester}" == "commit" ]; then
+  echo "is_mainline: true" >> expansion.yml
+else
+  echo "is_mainline: false" >> expansion.yml
+fi
+
+# We parse the username out of the order_id as patches append that in and SPS does not need that information
+# shellcheck disable=SC2154
+echo "parsed_order_id: $(echo "${revision_order_id}" | awk -F'_' '{print $NF}')"  >> expansion.yml

.evergreen/perf-submission.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -eu
+
+# Submit the performance data to the SPS endpoint
+# shellcheck disable=SC2154
+response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \
+  "https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=${parsed_order_id}&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=${is_mainline}" \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d @results.json)
+
+http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}')
+response_body=$(echo "$response" | sed '/HTTP_STATUS/d')
+
+# Throw an error if the data was not successfully submitted
+if [ "$http_status" -ne 200 ]; then
+  echo "Error: Received HTTP status $http_status"
+  echo "Response Body: $response_body"
+  exit 1
+fi
+
+echo "Response Body: $response_body"
+echo "HTTP Status: $http_status"

.evergreen/run-perf-tests.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/bash
+
+set -eux
+
+export OUTPUT_FILE="results.json"
+
+# Install django-mongodb-backend
+/opt/python/3.12/bin/python3 -m venv venv
+. venv/bin/activate
+python -m pip install -U pip
+pip install -e .
+
+python .evergreen/run_perf_test.py
+mv performance_tests/$OUTPUT_FILE $OUTPUT_FILE
+mv performance_tests/report.json report.json

.evergreen/run_perf_test.py

@@ -0,0 +1,69 @@
+import json
+import logging
+import os
+import shlex
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+LOGGER = logging.getLogger("test")
+logging.basicConfig(level=logging.INFO, format="%(levelname)-8s %(message)s")
+OUTPUT_FILE = os.environ.get("OUTPUT_FILE", "results.json")
+
+
+def format_output(start_time: datetime):
+    """Formats the output from the performance tests into a report.json file."""
+    end_time = datetime.now()
+    elapsed_secs = (end_time - start_time).total_seconds()
+    with open(OUTPUT_FILE) as fid:  # noqa: PTH123
+        results = json.load(fid)
+    LOGGER.info("results.json:\n%s", json.dumps(results, indent=2))
+
+    results = {
+        "status": "PASS",
+        "exit_code": 0,
+        "test_file": "BenchmarkTests",
+        "start": int(start_time.timestamp()),
+        "end": int(end_time.timestamp()),
+        "elapsed": elapsed_secs,
+    }
+    report = {"results": [results]}
+
+    LOGGER.info("report.json\n%s", json.dumps(report, indent=2))
+
+    with open("report.json", "w", newline="\n") as fid:  # noqa: PTH123
+        json.dump(report, fid)
+
+
+def run_command(cmd: str | list[str], **kwargs) -> None:
+    """Runs a shell command. Exits on failure."""
+    if isinstance(cmd, list):
+        cmd = " ".join(cmd)
+    LOGGER.info("Running command '%s'...", cmd)
+    kwargs.setdefault("check", True)
+    try:
+        subprocess.run(shlex.split(cmd), **kwargs)  # noqa: PLW1510, S603
+    except subprocess.CalledProcessError as e:
+        LOGGER.error(e.output)
+        LOGGER.error(str(e))
+        sys.exit(e.returncode)
+    LOGGER.info("Running command '%s'... done.", cmd)
+
+
+start_time = datetime.now()
+ROOT = Path(__file__).absolute().parent.parent
+data_dir = ROOT / "specifications/source/benchmarking/odm-data"
+if not data_dir.exists():
+    run_command("git clone --depth 1 https://github.com/mongodb/specifications.git")
+    run_command("tar xf flat_models.tgz", cwd=data_dir)
+    run_command("tar xf nested_models.tgz", cwd=data_dir)
+
+os.chdir("performance_tests")
+start_time = datetime.now()
+run_command(
+    "python manage.py test",
+    env=os.environ
+    | {"DJANGO_MONGODB_PERFORMANCE_TEST_DATA_PATH": str(data_dir), "OUTPUT_FILE": "results.json"},
+)
+format_output(start_time)