standard-tests: add benchmarks (#31302)

Co-authored-by: Sydney Runkle <[email protected]>

Author: ccurme

.github/scripts/check_diff.py

@@ -119,7 +119,9 @@ def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:
     if job == "test-pydantic":
         return _get_pydantic_test_configs(dir_)
 
-    if dir_ == "libs/core":
+    if job == "codspeed":
+        py_versions = ["3.12"]  # 3.13 is not yet supported
+    elif dir_ == "libs/core":
         py_versions = ["3.9", "3.10", "3.11", "3.12", "3.13"]
     # custom logic for specific directories
     elif dir_ == "libs/partners/milvus":
@@ -210,6 +212,8 @@ def _get_configs_for_multi_dirs(
         )
     elif job == "extended-tests":
         dirs = list(dirs_to_run["extended-test"])
+    elif job == "codspeed":
+        dirs = list(dirs_to_run["codspeed"])
     else:
         raise ValueError(f"Unknown job: {job}")
 
@@ -225,6 +229,7 @@ def _get_configs_for_multi_dirs(
         "lint": set(),
         "test": set(),
         "extended-test": set(),
+        "codspeed": set(),
     }
     docs_edited = False
 
@@ -248,6 +253,8 @@ def _get_configs_for_multi_dirs(
             dirs_to_run["extended-test"].update(LANGCHAIN_DIRS)
             dirs_to_run["lint"].add(".")
 
+        if file.startswith("libs/core"):
+            dirs_to_run["codspeed"].add(f"libs/core")
         if any(file.startswith(dir_) for dir_ in LANGCHAIN_DIRS):
             # add that dir and all dirs after in LANGCHAIN_DIRS
             # for extended testing
@@ -286,6 +293,7 @@ def _get_configs_for_multi_dirs(
                 if not filename.startswith(".")
             ] != ["README.md"]:
                 dirs_to_run["test"].add(f"libs/partners/{partner_dir}")
+                dirs_to_run["codspeed"].add(f"libs/partners/{partner_dir}")
             # Skip if the directory was deleted or is just a tombstone readme
         elif file == "libs/packages.yml":
             continue
@@ -311,6 +319,7 @@ def _get_configs_for_multi_dirs(
             "compile-integration-tests",
             "dependencies",
             "test-pydantic",
+            "codspeed",
         ]
     }
     map_job_to_configs["test-doc-imports"] = (

.github/workflows/check_diffs.yml

@@ -152,6 +152,7 @@ jobs:
           # grep will exit non-zero if the target message isn't found,
           # and `set -e` above will cause the step to fail.
           echo "$STATUS" | grep 'nothing to commit, working tree clean'
+
   ci_success:
     name: "CI Success"
     needs: [build, lint, test, compile-integration-tests, extended-tests, test-doc-imports, test-pydantic]

.github/workflows/codspeed.yml

@@ -7,38 +7,73 @@ on:
   pull_request:
     paths:
       - 'libs/core/**'
-  # `workflow_dispatch` allows CodSpeed to trigger backtest
-  # performance analysis in order to generate initial data.
+      - 'libs/partners/**'
+
   workflow_dispatch:
 
+env:
+  AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
+  AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
+  AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
+  AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LEGACY_CHAT_DEPLOYMENT_NAME }}
+  DEEPSEEK_API_KEY: foo
+  FIREWORKS_API_KEY: foo
+
 jobs:
+  prepare_matrix:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.12"
+      - id: files
+        uses: Ana06/[email protected]
+      - id: set-matrix
+        run: |
+          uv venv
+          uv pip install packaging requests
+          uv run .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT
+    outputs:
+      codspeed: ${{ steps.set-matrix.outputs.codspeed }}
   codspeed:
     name: Run benchmarks
-    if: (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-codspeed-benchmarks')) || github.event_name == 'workflow_dispatch' || github.event_name == 'push'
+    needs: [ prepare_matrix ]
+    if: ${{ needs.prepare_matrix.outputs.codspeed != '[]' }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        job-configs: ${{ fromJson(needs.prepare_matrix.outputs.codspeed) }}
+      fail-fast: false
+
     steps:
       - uses: actions/checkout@v4
 
-      # We have to use 3.12, 3.13 is not yet supported
+      # We have to use 3.12 as 3.13 is not yet supported
       - name: Install uv
         uses: astral-sh/setup-uv@v6
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.job-configs.python-version }}
 
-      # Using this action is still necessary for CodSpeed to work
       - uses: actions/setup-python@v5
         with:
-          python-version: "3.12"
+          python-version: ${{ matrix.job-configs.python-version }}
 
-      - name: install deps
+      - name: Install dependencies
         run: uv sync --group test
-        working-directory: ./libs/core
+        working-directory: ${{ matrix.job-configs.working-directory }}
 
-      - name: Run benchmarks
+      - name: Run benchmarks ${{ matrix.job-configs.working-directory }}
         uses: CodSpeedHQ/action@v3
         with:
           token: ${{ secrets.CODSPEED_TOKEN }}
           run: |
-            cd libs/core
-            uv run --no-sync pytest ./tests/benchmarks --codspeed
-          mode: walltime
+            cd ${{ matrix.job-configs.working-directory }}
+            if [ "${{ matrix.job-configs.working-directory }}" = "libs/core" ]; then
+              uv run --no-sync pytest ./tests/benchmarks --codspeed
+            else
+              uv run --no-sync pytest ./tests/ --codspeed
+            fi
+          mode: ${{ matrix.job-configs.working-directory == 'libs/core' && 'walltime' || 'instrumentation' }}
+