INTPYTHON-580 Add CrewAI Integration Tests (#71)

Author: blink1073

.evergreen/config.yml

@@ -208,6 +208,20 @@ tasks:
       - func: "setup remote atlas"
       - func: "execute tests"
 
+  - name: test-crewai-tools-local
+    tags: [local]
+    commands:
+      - func: "fetch repo"
+      - func: "setup local atlas"
+      - func: "execute tests"
+
+  - name: test-crewai-tools-remote
+    tags: [remote]
+    commands:
+      - func: "fetch repo"
+      - func: "setup remote atlas"
+      - func: "execute tests"
+
   - name: test-haystack-embeddings-local
     tags: [local]
     commands:
@@ -351,6 +365,16 @@ buildvariants:
       - name: test-pymongo-voyageai-local
       - name: test-pymongo-voyageai-remote
 
+  - name: test-crewai-tools-rhel
+    display_name: CrewAI-Tools Ubuntu
+    expansions:
+      DIR: crewai-tools
+    run_on:
+      - ubuntu2404-small
+    tasks:
+      - name: test-crewai-tools-local
+      - name: test-crewai-tools-remote
+
   - name: test-haystack-embeddings-rhel
     display_name: Haystack Embeddings RHEL
     expansions:

.evergreen/setup-remote.sh

@@ -41,6 +41,9 @@ case $DIR in
     pymongo-voyageai)
         MONGODB_URI=$VOYAGEAI_MONGODB_URI
     ;;
+    crewai-tools)
+        MONGODB_URI=$CREWAI_TOOLS_URI
+    ;;
     langchain-js)
         MONGODB_URI=$LANGCHAIN_MONGODB_URI
     ;;

crewai-tools/config.env

@@ -0,0 +1,3 @@
+REPO_NAME=crewAI-tools
+REPO_ORG=crewAIInc
+DATABASE=crewai_test_db

crewai-tools/run.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eu
+
+# Get the MONGODB_URI and OPENAI_API_KEY.
+SCRIPT_DIR=$(realpath "$(dirname ${BASH_SOURCE[0]})")
+ROOT_DIR=$(dirname $SCRIPT_DIR)
+. $ROOT_DIR/env.sh
+
+. $ROOT_DIR/.evergreen/utils.sh
+
+PYTHON_BINARY=$(find_python3)
+
+$PYTHON_BINARY -m venv venv_pipeline
+source venv_pipeline/bin/activate
+
+pip install uv
+
+uv sync --extra mongodb
+uv run pytest -v tests/tools/test*mongodb*.py
+
+mv ../test_mongodb_vector_search_tool.py .
+uv run python test_mongodb_vector_search_tool.py

crewai-tools/test_mongodb_vector_search_tool.py

@@ -0,0 +1,79 @@
+import os
+from crewai import Agent
+from crewai import Task
+from crewai import Crew, Process, LLM
+from crewai.cli.constants import DEFAULT_LLM_MODEL
+from crewai_tools import MongoDBVectorSearchTool, MongoDBVectorSearchConfig
+from langchain_community.document_loaders import PyPDFLoader
+import time
+
+# Set environment as LiteLLM expects
+os.environ["AZURE_API_KEY"] = os.environ["AZURE_OPENAI_API_KEY"]
+os.environ["AZURE_API_BASE"] = os.environ["AZURE_OPENAI_ENDPOINT"]
+os.environ["AZURE_API_VERSION"] = os.environ["OPENAI_API_VERSION"]
+
+# Pre-populate a collection and an index
+print("Creating collection...")
+conn_string = os.environ.get(
+    "MONGODB_URI", "mongodb://localhost:27017?directConnection=true"
+)
+database_name = "crewai_test_db"
+collection_name = "vector_test"
+
+tool = MongoDBVectorSearchTool(
+    connection_string=conn_string,
+    database_name=database_name,
+    collection_name=collection_name,
+)
+coll = tool._coll
+coll.delete_many({})
+
+# Insert documents from a pdf.
+print("Loading documents...")
+loader = PyPDFLoader("https://arxiv.org/pdf/2303.08774.pdf")
+tool.add_texts([i.page_content for i in loader.load()])
+
+print("Creating vector index...")
+if not any([ix["name"] == "vector_index" for ix in coll.list_search_indexes()]):
+    tool.create_vector_search_index(dimensions=1536, auto_index_timeout=60)
+
+# Create the MongoDB tool
+print("Creating tool and waiting for index to be complete...")
+
+# Wait for index to be complete.
+n_docs = coll.count_documents({})
+tool.query_config = MongoDBVectorSearchConfig(limit=n_docs, oversampling_factor=1)
+start = time.monotonic()
+while time.monotonic() - start <= 60:
+    if len(tool._run("sandwich")) == n_docs:
+        break
+    else:
+        time.sleep(1)
+
+# Assemble a crew
+researcher = Agent(
+    role="AI Accuracy Researcher",
+    goal="Find and extract key information from a technical document",
+    backstory="You're specialized in analyzing technical content to extract insights and answers",
+    verbose=False,
+    tools=[tool],
+    llm=LLM(model=f"azure/{DEFAULT_LLM_MODEL}"),
+)
+research_task = Task(
+    description="Research information in a technical document",
+    expected_output="A summary of the accuracy of GPT-4",
+    agent=researcher,
+)
+crew = Crew(
+    agents=[researcher],
+    tasks=[research_task],
+    process=Process.sequential,
+    verbose=False,
+)
+
+# Get the result and assert something about the results
+print("Running the crew...")
+result = crew.kickoff()
+text = result.raw.lower()
+assert "advancements" in text or "improvements" in text, text
+assert "GPT-4" in result.raw