Merge branch 'main' into INTPYTHON-715-Add-langgraph-store-mongodb-to-ai-ml-testing

Author: caseyclements

.evergreen/config.yml

@@ -289,9 +289,24 @@ tasks:
         params:
           file: src/langchain-js/langchainjs/libs/langchain-mongodb/results.xml
 
+  - name: test-mem0-python-local
+    tags: [ local ]
+    commands:
+      - func: "fetch repo"
+      - func: "setup local atlas"
+      - func: "execute tests"
+
+  - name: test-mem0-python-remote
+    tags: [ remote ]
+    commands:
+      - func: "fetch repo"
+      - func: "setup remote atlas"
+      - func: "execute tests"
+
 buildvariants:
   - name: test-semantic-kernel-python-rhel
     display_name: Semantic-Kernel RHEL Python
+    tags: [python]
     expansions:
       DIR: semantic-kernel-python
     run_on:
@@ -304,6 +319,7 @@ buildvariants:
 
   - name: test-semantic-kernel-csharp-rhel
     display_name: Semantic-Kernel RHEL CSharp
+    tags: [csharp]
     expansions:
       DIR: semantic-kernel-csharp
     run_on:
@@ -314,6 +330,7 @@ buildvariants:
 
   - name: test-langchain-python-rhel
     display_name: Langchain RHEL Python
+    tags: [python]
     expansions:
       DIR: langchain-python
     run_on:
@@ -325,6 +342,7 @@ buildvariants:
 
   - name: test-langgraph-python-rhel
     display_name: Langgraph RHEL Python
+    tags: [python]
     expansions:
       DIR: langgraph-python
     run_on:
@@ -346,8 +364,9 @@ buildvariants:
         batchtime: 10080  # 1 week
 
   # TODO: INTPYTHON-668
-  # - name: test-chatgpt-retrieval-plugin-rhel
+  # - name: test-chatgpt-retrieval-plugin-python-rhel
   #   display_name: ChatGPT Retrieval Plugin
+  #   tags: [python]
   #   expansions:
   #     DIR: chatgpt-retrieval-plugin
   #   run_on:
@@ -358,8 +377,9 @@ buildvariants:
   #       batchtime: 10080  # 1 week
 
   # TODO: INTPYTHON-669
-  # - name: test-llama-index-vectorstore-rhel
+  # - name: test-llama-index-vectorstore-python-rhel
   #   display_name: LlamaIndex RHEL Vector Store
+  #   tags: [python]
   #   expansions:
   #     DIR: llama-index-python-vectorstore
   #   run_on:
@@ -370,8 +390,9 @@ buildvariants:
   #     # - name: test-llama-index-remote
   #     #   batchtime: 10080  # 1 week
 
-  - name: test-docarray-rhel
+  - name: test-docarray-python-rhel
     display_name: DocArray RHEL
+    tags: [python]
     expansions:
       DIR: docarray
     run_on:
@@ -380,8 +401,9 @@ buildvariants:
       - name: test-docarray-local
       - name: test-docarray-remote
 
-  - name: test-pymongo-voyageai-rhel
+  - name: test-pymongo-voyageai-python-rhel
     display_name: PyMongo-VoyageAI RHEL
+    tags: [python]
     expansions:
       DIR: pymongo-voyageai
     run_on:
@@ -390,8 +412,9 @@ buildvariants:
       - name: test-pymongo-voyageai-local
       - name: test-pymongo-voyageai-remote
 
-  - name: test-crewai-tools-rhel
+  - name: test-crewai-tools-python-rhel
     display_name: CrewAI-Tools Ubuntu
+    tags: [python]
     expansions:
       DIR: crewai-tools
     run_on:
@@ -400,8 +423,9 @@ buildvariants:
       - name: test-crewai-tools-local
       - name: test-crewai-tools-remote
 
-  - name: test-haystack-embeddings-rhel
+  - name: test-haystack-embeddings-python-rhel
     display_name: Haystack Embeddings RHEL
+    tags: [python]
     expansions:
       DIR: haystack-embeddings
     run_on:
@@ -411,8 +435,9 @@ buildvariants:
       # TODO: INTPYTHON-465
       # - name: test-haystack-embeddings-remote
 
-  - name: test-haystack-fulltext-rhel
+  - name: test-haystack-fulltext-python-rhel
     display_name: Haystack FullText RHEL
+    tags: [python]
     expansions:
       DIR: haystack-fulltext
     run_on:
@@ -422,22 +447,35 @@ buildvariants:
       # TODO: INTPYTHON-465
       # - name: test-haystack-fulltext-remote
 
-  - name: test-langchaingo-ubuntu
+  - name: test-langchaingo-golang-ubuntu
     display_name: LangchainGo Ubuntu2204
+    tags: [golang]
     expansions:
       DIR: langchaingo-golang
     run_on:
       - ubuntu2204-small
     tasks:
       - name: test-langchaingo-local
 
-  # TODO: INTPYTHON-667
-  # - name: test-langchain-js-ubuntu
-  #   display_name: LangchainJS Ubuntu2204
-  #   expansions:
-  #     DIR: langchain-js
-  #   run_on:
-  #     - ubuntu2204-small
-  #   tasks:
-  #     - name: test-langchain-js-local
-  #     - name: test-langchain-js-remote
+  - name: test-langchain-javascript-ubuntu
+    display_name: LangchainJS Ubuntu2204
+    tags: [javascript]
+    expansions:
+      DIR: langchain-js
+    run_on:
+      - ubuntu2204-small
+    tasks:
+      - name: test-langchain-js-local
+      - name: test-langchain-js-remote
+
+  - name: test-mem0-python-rhel
+    display_name: mem0 RHEL Python
+    tags: [python]
+    expansions:
+      DIR: mem0-python
+    run_on:
+      - rhel87-small
+    tasks:
+      - name: test-mem0-python-local
+      - name: test-mem0-python-remote
+        batchtime: 10080  # 1 week

.evergreen/lint_config.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""
+Pre-commit hook to check if buildvariant tasks contain required language tags.
+"""
+
+import logging
+import sys
+import yaml
+from pathlib import Path
+from typing import List, Dict, Any
+
+logging.basicConfig()
+logger = logging.getLogger(__file__)
+logger.setLevel(logging.DEBUG)
+
+
+CURRENT_DIR = Path(__file__).parent.resolve()
+CONFIG_YML = CURRENT_DIR / "config.yml"
+VALID_LANGUAGES = {"python", "golang", "javascript", "csharp"}
+
+
+def load_yaml_file(file_path: str) -> Dict[Any, Any]:
+    """Load and parse a YAML file."""
+    with open(file_path, "r", encoding="utf-8") as file:
+        return yaml.safe_load(file) or {}
+
+
+def check_buildvariants(data: Dict[Any, Any]) -> List[str]:
+    """
+    Check if buildvariant tasks contain at least one required language tag
+    as well as the language within the buildvariant name.
+
+    Example Buildvariant structure in YAML:
+    buildvariants:
+    - name: test-semantic-kernel-python-rhel
+        display_name: Semantic-Kernel RHEL Python
+        tags: [python]
+        expansions:
+        DIR: semantic-kernel-python
+        run_on:
+        - rhel87-small
+        tasks:
+        - name: test-semantic-kernel-python-local
+        - name: test-semantic-kernel-python-remote
+          batchtime: 10080  # 1 week
+
+    Args:
+        data: Parsed YAML data
+
+    Returns:
+        List of error messages for tasks missing required tags
+    """
+    errors = []
+
+    buildvariants = data.get("buildvariants", [])
+    if not isinstance(buildvariants, list):
+        return ["'buildvariants' should be a list"]
+
+    for i, buildvariant in enumerate(buildvariants):
+        if not isinstance(buildvariant, dict):
+            errors.append(f"buildvariants[{i}] should contain sub-fields")
+            continue
+
+        buildvariant_name = buildvariant.get("name", "")
+        if not buildvariant_name:
+            errors.append(f"buildvariants[{i}] is missing 'name'")
+            continue
+        else:
+            if all([f"-{lang}-" not in buildvariant_name for lang in VALID_LANGUAGES]):
+                errors.append(
+                    f"buildvariant '{buildvariant_name}' should contain one"
+                    f" '-[{', '.join(VALID_LANGUAGES)}]-' in its name"
+                    f"got: {buildvariant_name}",
+                )
+
+        buildvariant_display_name = buildvariant.get("display_name", buildvariant_name)
+
+        tags = buildvariant.get("tags", [])
+
+        if not isinstance(tags, list) or len(tags) != 1:
+            errors.append(
+                f"'tags' in buildvariant '{buildvariant_display_name}' should be a list of size 1"
+            )
+            continue
+
+        if tags[0] not in VALID_LANGUAGES:
+            errors.append(
+                f"buildvariant '{buildvariant_display_name}' has invalid tag '{tags[0]}'. "
+                f"Valid tags are: {', '.join(VALID_LANGUAGES)}"
+            )
+    return errors
+
+
+def main():
+    """Main function for the pre-commit hook."""
+    total_errors = 0
+
+    data = load_yaml_file(CONFIG_YML)
+    if not data:
+        raise FileNotFoundError(f"Failed to load or parse {CONFIG_YML}")
+
+    errors = check_buildvariants(data)
+
+    if errors:
+        logger.error("❌ Errors found in %s:", CONFIG_YML)
+        for error in errors:
+            logger.error("  - %s", error)
+        total_errors += len(errors)
+
+    if total_errors > 0:
+        logger.error("❌ Total errors found: %s", total_errors)
+        return 1
+    else:
+        logger.info("✅ %s passed AI/ML testing pipeline validation", CONFIG_YML)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.evergreen/setup-remote.sh

@@ -47,6 +47,9 @@ case $DIR in
     langchain-js)
         MONGODB_URI=$LANGCHAIN_MONGODB_URI
     ;;
+    mem0-python)
+        MONGODB_URI=$MEM0_URI
+    ;;
     *)
         echo "Missing config in setup-remote.sh for DIR: $DIR"
         exit 1

.github/workflows/lint.yml

@@ -23,7 +23,7 @@ jobs:
           python-version: '3.10'
       - name: Install Python dependencies
         run: |
-          python -m pip install -U pip pre-commit
+          python -m pip install -U pip pre-commit pyyaml
       - name: Run linters
         run: |
           pre-commit run --hook-stage=manual --all-files

.pre-commit-config.yaml

@@ -53,3 +53,12 @@ repos:
     rev: "v2.2.6"
     hooks:
       - id: codespell
+
+  - repo: local
+    hooks:
+      - id: check-buildvariant-tags
+        name: Check buildvariant language tags
+        entry: python3 .evergreen/lint_config.py
+        language: system
+        files: .evergreen/config.yml
+        args: ['--languages=python,golang,javascript,csharp']

README.md

@@ -110,6 +110,8 @@ Test execution flow is defined in `.evergreen/config.yml`. The test pipeline's c
 - `run_on` -- Specified platform to run on. `rhel87-small` or `ubuntu2204-small` should be used by default. Any other distro may fail Atlas CLI setup.
 - `tasks` -- Tasks to run. See below for more details
 - `cron` -- The tests are run via a cron job on a nightly cadence. This can be modified by setting a different cadence. Cron jobs can be scheduled using [cron syntax](https://crontab.guru/#0_0_*_*_*)
+- `tags` -- This should include the language where the AI/ML is run. i.e. `[python, csharp, golang, javascript]` Any tagged language will populate the
+appropriate language-specific slack channel.
 
 **[Tasks](https://docs.devprod.prod.corp.mongodb.com/evergreen/Project-Configuration/Project-Configuration-Files#tasks)** -- These are the "building blocks" of our runs. Here is where we consolidate the specific set of functions. The basic parameters to add are shown below
 
@@ -185,8 +187,10 @@ evergreen patch -p ai-ml-pipeline-testing --param REPO_ORG=caseyclements --param
 
 ### Handling Failing Tests
 
-If tests are found to be failing, and cannot be addressed quickly, the responsible team MUST create a JIRA ticket, and disable the relevant tests
+Tests are run periodically (nightly) and any failures will propagate into both the `dbx-ai-ml-testing-pipline-notifications` and `dbx-ai-ml-testing-pipeline-notifications-{language}` channel. Repo owners of this `ai-ml-testing-pipeline` library are required to join the `dbx-ai-ml-testing-pipeline-notifications`. Pipeline specific implementers must **at least** join `dbx-ai-ml-testing-pipline-notifications-{language}` (e.g. whomever implemented `langchain-js` must at least be a member of `dbx-ai-ml-testing-pipeline-notifications-js`).
+
+If tests are found to be failing, and cannot be addressed quickly, the responsible team MUST create a JIRA ticket within their team's project (e.g. a python failure should generate an `INTPYTHON` ticket), and disable the relevant tests
 in the `config.yml` file, with a comment about the JIRA ticket that will address it.
 
-This policy will help ensure that a single failing integration does not cause noise in the `dbx-ai-ml-testing-pipeline-notifications` that would mask other
+This policy will help ensure that a single failing integration does not cause noise in the `dbx-ai-ml-testing-pipeline-notifications` or `dbx-ai-ml-testing-pipeline-notifications-{language}` that would mask other
 failures.

crewai-tools/test_mongodb_vector_search_tool.py

@@ -2,7 +2,6 @@
 from crewai import Agent
 from crewai import Task
 from crewai import Crew, Process, LLM
-from crewai.cli.constants import DEFAULT_LLM_MODEL
 from crewai_tools import MongoDBVectorSearchTool, MongoDBVectorSearchConfig
 from langchain_community.document_loaders import PyPDFLoader
 import time
@@ -55,9 +54,9 @@
     role="AI Accuracy Researcher",
     goal="Find and extract key information from a technical document",
     backstory="You're specialized in analyzing technical content to extract insights and answers",
-    verbose=False,
+    verbose=True,
     tools=[tool],
-    llm=LLM(model=f"azure/{DEFAULT_LLM_MODEL}"),
+    llm=LLM(model="azure/gpt-4o", seed=12345),
 )
 research_task = Task(
     description="Research information in a technical document",
@@ -68,12 +67,12 @@
     agents=[researcher],
     tasks=[research_task],
     process=Process.sequential,
-    verbose=False,
+    verbose=True,
 )
 
 # Get the result and assert something about the results
 print("Running the crew...")
 result = crew.kickoff()
 text = result.raw.lower()
-assert "advancements" in text or "improvements" in text, text
+assert "limitations" in text, text
 assert "GPT-4" in result.raw

langchain-js/run.sh

@@ -37,7 +37,11 @@ setup_langchain_integration() {
 
     yarn add --dev jest-junit
     export JEST_JUNIT_OUTPUT_NAME=results.xml
-
+    # Trim trailing slashes since lanchainjs is doing string manipulationn, not
+    # using the URI class.
+    AZURE_OPENAI_BASE_PATH=$(echo "$AZURE_OPENAI_ENDPOINT" | sed 's:/*$::')
+    export AZURE_OPENAI_BASE_PATH
+    export AZURE_OPENAI_API_VERSION=$OPENAI_API_VERSION
     # optionally enable to debug local atlas in CI.
     # export DEBUG=testcontainers*
 }

mem0-python/config.env

@@ -0,0 +1,3 @@
+REPO_NAME=mem0
+REPO_ORG=mem0ai
+DATABASE=mem0_test_db