include pre-commit to lint the config.yml file

Author: Jibola

.evergreen/config.yml

@@ -310,30 +310,32 @@ buildvariants:
       - name: test-langgraph-python-remote
         batchtime: 10080  # 1 week
 
-  - name: test-chatgpt-retrieval-plugin-rhel
-    display_name: ChatGPT Retrieval Plugin
-    tags: [python]
-    expansions:
-      DIR: chatgpt-retrieval-plugin
-    run_on:
-      - rhel87-small
-    tasks:
-      - name: test-chatgpt-retrieval-plugin-local
-      - name: test-chatgpt-retrieval-plugin-remote
-        batchtime: 10080  # 1 week
-
-  - name: test-llama-index-vectorstore-rhel
-    display_name: LlamaIndex RHEL Vector Store
-    tags: [python]
-    expansions:
-      DIR: llama-index-python-vectorstore
-    run_on:
-      - rhel87-small
-    tasks:
-      - name: test-llama-index-local
-      # TODO: INTPYTHON-440
-      # - name: test-llama-index-remote
-      #   batchtime: 10080  # 1 week
+  # TODO: INTPYTHON-668
+  # - name: test-chatgpt-retrieval-plugin-rhel
+  #   display_name: ChatGPT Retrieval Plugin
+  #   tags: [python]
+  #   expansions:
+  #     DIR: chatgpt-retrieval-plugin
+  #   run_on:
+  #     - rhel87-small
+  #   tasks:
+  #     - name: test-chatgpt-retrieval-plugin-local
+  #     - name: test-chatgpt-retrieval-plugin-remote
+  #       batchtime: 10080  # 1 week
+
+  # TODO: INTPYTHON-669
+  # - name: test-llama-index-vectorstore-rhel
+  #   display_name: LlamaIndex RHEL Vector Store
+  #   tags: [python]
+  #   expansions:
+  #     DIR: llama-index-python-vectorstore
+  #   run_on:
+  #     - rhel87-small
+  #   tasks:
+  #     - name: test-llama-index-local
+  #     # TODO: INTPYTHON-440
+  #     # - name: test-llama-index-remote
+  #     #   batchtime: 10080  # 1 week
 
   - name: test-docarray-rhel
     display_name: DocArray RHEL
@@ -391,13 +393,14 @@ buildvariants:
     tasks:
       - name: test-langchaingo-local
 
-  - name: test-langchain-js-ubuntu
-    display_name: LangchainJS Ubuntu2204
-    tags: [javascript]
-    expansions:
-      DIR: langchain-js
-    run_on:
-      - ubuntu2204-small
-    tasks:
-      - name: test-langchain-js-local
-      - name: test-langchain-js-remote
+  # TODO: INTPYTHON-667
+  # - name: test-langchain-js-ubuntu
+  #   display_name: LangchainJS Ubuntu2204
+  #   tags: [javascript]
+  #   expansions:
+  #     DIR: langchain-js
+  #   run_on:
+  #     - ubuntu2204-small
+  #   tasks:
+  #     - name: test-langchain-js-local
+  #     - name: test-langchain-js-remote

.evergreen/execute-tests.sh

@@ -5,12 +5,12 @@ set -eu
 SCRIPT_DIR=$(realpath "$(dirname ${BASH_SOURCE[0]})")
 ROOT_DIR=$(dirname $SCRIPT_DIR)
 
-
 # Source the configuration.
 cd ${ROOT_DIR}/${DIR}
 set -a
 source config.env
 set +a
 
 cd ${REPO_NAME}
+
 bash ${ROOT_DIR}/${DIR}/run.sh

.evergreen/lint_config.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Pre-commit hook to check if buildvariant tasks contain required language tags.
+"""
+
+import logging
+import sys
+import yaml
+from pathlib import Path
+from typing import List, Dict, Any
+
+logging.basicConfig()
+logger = logging.getLogger(__file__)
+logger.setLevel(logging.DEBUG)
+
+
+CURRENT_DIR = Path(__file__).parent.resolve()
+CONFIG_YML = CURRENT_DIR / "config.yml"
+VALID_LANGUAGES = {"python", "golang", "javascript", "csharp"}
+
+
+def load_yaml_file(file_path: str) -> Dict[Any, Any]:
+    """Load and parse a YAML file."""
+    with open(file_path, "r", encoding="utf-8") as file:
+        return yaml.safe_load(file) or {}
+
+
+def check_buildvariant_tags(data: Dict[Any, Any]) -> List[str]:
+    """
+    Check if buildvariant tasks contain at least one required language tag.
+
+    Example Buildvariant structure in YAML:
+    buildvariants:
+    - name: test-semantic-kernel-python-rhel
+        display_name: Semantic-Kernel RHEL Python
+        tags: [python]
+        expansions:
+        DIR: semantic-kernel-python
+        run_on:
+        - rhel87-small
+        tasks:
+        - name: test-semantic-kernel-python-local
+        - name: test-semantic-kernel-python-remote
+          batchtime: 10080  # 1 week
+
+    Args:
+        data: Parsed YAML data
+
+    Returns:
+        List of error messages for tasks missing required tags
+    """
+    errors = []
+
+    buildvariants = data.get("buildvariants", [])
+    if not isinstance(buildvariants, list):
+        return ["'buildvariants' should be a list"]
+
+    for i, buildvariant in enumerate(buildvariants):
+        if not isinstance(buildvariant, dict):
+            errors.append(f"buildvariants[{i}] should be a dictionary")
+            continue
+
+        buildvariant_name = buildvariant.get("display_name", f"buildvariant_{i}")
+        tags = buildvariant.get("tags", [])
+
+        if not isinstance(tags, list) or len(tags) != 1:
+            errors.append(
+                f"'tags' in buildvariant '{buildvariant_name}' should be a list of size 1"
+            )
+            continue
+
+        if tags[0] not in VALID_LANGUAGES:
+            errors.append(
+                f"buildvariant '{buildvariant_name}' has invalid tag '{tags[0]}'. "
+                f"Valid tags are: {', '.join(VALID_LANGUAGES)}"
+            )
+    return errors
+
+
+def main():
+    """Main function for the pre-commit hook."""
+    total_errors = 0
+
+    data = load_yaml_file(CONFIG_YML)
+    if not data:
+        raise FileNotFoundError(f"Failed to load or parse {CONFIG_YML}")
+
+    errors = check_buildvariant_tags(data)
+
+    if errors:
+        logger.error("❌ Errors found in %s:", CONFIG_YML)
+        for error in errors:
+            logger.error("  - %s", error)
+        total_errors += len(errors)
+
+    if total_errors > 0:
+        logger.error("❌ Total errors found: %s", total_errors)
+        return 1
+    else:
+        logger.info("✅ %s passed AI/ML testing pipeline validation", CONFIG_YML)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.evergreen/provision-atlas.sh

@@ -18,6 +18,8 @@ source secrets-export.sh
 
 # Create the env file
 echo "export VOYAGEAI_S3_BUCKET=$VOYAGEAI_S3_BUCKET" >> env.sh
-echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh
+echo "export AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> env.sh
+echo "export AZURE_OPENAI_API_KEY=$AZURE_OPENAI_API_KEY" >> env.sh
+echo "export OPENAI_API_VERSION=$OPENAI_API_VERSION" >> env.sh
 echo "export MONGODB_URI=$CONN_STRING" >> env.sh
 echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh

.evergreen/setup-remote.sh

@@ -53,9 +53,11 @@ export MONGODB_URI
 
 # Create the env file
 echo "export VOYAGEAI_S3_BUCKET=$VOYAGEAI_S3_BUCKET" >> env.sh
-echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh
-echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh
+echo "export AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> env.sh
+echo "export AZURE_OPENAI_API_KEY=$AZURE_OPENAI_API_KEY" >> env.sh
+echo "export OPENAI_API_VERSION=$OPENAI_API_VERSION" >> env.sh
 echo "export MONGODB_URI=$MONGODB_URI" >> env.sh
+echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh
 
 # Ensure the remote database is populated.
 . .evergreen/utils.sh

.pre-commit-config.yaml

@@ -53,3 +53,12 @@ repos:
     rev: "v2.2.6"
     hooks:
       - id: codespell
+
+  - repo: local
+    hooks:
+      - id: check-buildvariant-tags
+        name: Check buildvariant language tags
+        entry: python3 .evergreen/lint_config.py
+        language: system
+        files: .evergreen/config.yml
+        args: ['--languages=python,golang,javascript,csharp']

README.md

@@ -189,7 +189,7 @@ evergreen patch -p ai-ml-pipeline-testing --param REPO_ORG=caseyclements --param
 
 Tests are run periodically (nightly) and any failures will propagate into both the `dbx-ai-ml-testing-pipline-notifications` and `dbx-ai-ml-testing-pipeline-notifications-{language}` channel. Repo owners of this `ai-ml-testing-pipeline` library are required to join the `dbx-ai-ml-testing-pipeline-notifications`. Pipeline specific implementers must **at least** join `dbx-ai-ml-testing-pipline-notifications-{language}` (e.g. whomever implemented `langchain-js` must at least be a member of `dbx-ai-ml-testing-pipeline-notifications-js`).
 
-If tests are found to be failing, and cannot be addressed quickly, the responsible team MUST create a JIRA ticket, and disable the relevant tests
+If tests are found to be failing, and cannot be addressed quickly, the responsible team MUST create a JIRA ticket within their team's project (e.g. a python failure should generate an `INTPYTHON` ticket), and disable the relevant tests
 in the `config.yml` file, with a comment about the JIRA ticket that will address it.
 
 This policy will help ensure that a single failing integration does not cause noise in the `dbx-ai-ml-testing-pipeline-notifications` or `dbx-ai-ml-testing-pipeline-notifications-{language}` that would mask other

langchain-python/run.sh

@@ -22,9 +22,6 @@ pip install uv rust-just
 
 just install
 
-export MONGODB_URI=$MONGODB_URI
-export OPENAI_API_KEY=$OPENAI_API_KEY
-
-just tests
+just unit_tests
 
 just integration_tests

langchaingo-golang/run.sh

@@ -9,4 +9,7 @@ cd vectorstores/mongovector
 export PATH="$PATH:/opt/golang/$GO_VERSION/bin"
 export GOROOT="/opt/golang/$GO_VERSION"
 
-go test -v -failfast -race -shuffle=on
+# TODO(GODRIVER-3606): Update expected error in LangChainGo test for
+# non-tokenized filter.
+go test -v -failfast -race -shuffle=on \
+  -skip "TestStore_SimilaritySearch_NonExactQuery/with_non-tokenized_filter"

langgraph-python/run.sh

@@ -22,9 +22,6 @@ pip install uv rust-just
 
 just install
 
-export MONGODB_URI=$MONGODB_URI
-export OPENAI_API_KEY=$OPENAI_API_KEY
-
-just tests
+just unit_tests
 
 just integration_tests