Merge branch 'main' of github.com:mongodb-labs/ai-ml-pipeline-testing into INTPYTHON-580

Author: blink1073

.evergreen/config.yml

@@ -119,18 +119,10 @@ tasks:
       - func: "setup remote atlas"
       - func: "execute tests"
 
-  - name: test-semantic-kernel-csharp-local
+  - name: test-semantic-kernel-csharp
     tags: [local]
     commands:
       - func: "fetch repo"
-      - func: "setup local atlas"
-      - func: "execute tests"
-
-  - name: test-semantic-kernel-csharp-remote
-    tags: [remote]
-    commands:
-      - func: "fetch repo"
-      - func: "setup remote atlas"
       - func: "execute tests"
 
   - name: test-langchain-python-local
@@ -264,6 +256,25 @@ tasks:
       - func: "fetch repo"
       - func: "execute tests"
 
+  - name: test-langchain-js-local
+    tags: [local]
+    commands:
+      - func: "fetch repo"
+      - func: "execute tests"
+      - command: attach.xunit_results
+        params:
+          file: src/langchain-js/langchainjs/libs/langchain-mongodb/results.xml
+
+  - name: test-langchain-js-remote
+    tags: [remote]
+    commands:
+      - func: "fetch repo"
+      - func: "setup remote atlas"
+      - func: "execute tests"
+      - command: attach.xunit_results
+        params:
+          file: src/langchain-js/langchainjs/libs/langchain-mongodb/results.xml
+
 buildvariants:
   - name: test-semantic-kernel-python-rhel
     display_name: Semantic-Kernel RHEL Python
@@ -284,8 +295,7 @@ buildvariants:
     run_on:
       - rhel87-small
     tasks:
-      - name: test-semantic-kernel-csharp-local
-      - name: test-semantic-kernel-csharp-remote
+      - name: test-semantic-kernel-csharp
         batchtime: 10080  # 1 week
 
   - name: test-langchain-python-rhel
@@ -310,28 +320,30 @@ buildvariants:
       - name: test-langgraph-python-remote
         batchtime: 10080  # 1 week
 
-  - name: test-chatgpt-retrieval-plugin-rhel
-    display_name: ChatGPT Retrieval Plugin
-    expansions:
-      DIR: chatgpt-retrieval-plugin
-    run_on:
-      - rhel87-small
-    tasks:
-      - name: test-chatgpt-retrieval-plugin-local
-      - name: test-chatgpt-retrieval-plugin-remote
-        batchtime: 10080  # 1 week
-
-  - name: test-llama-index-vectorstore-rhel
-    display_name: LlamaIndex RHEL Vector Store
-    expansions:
-      DIR: llama-index-python-vectorstore
-    run_on:
-      - rhel87-small
-    tasks:
-      - name: test-llama-index-local
-      # TODO: INTPYTHON-440
-      # - name: test-llama-index-remote
-      #   batchtime: 10080  # 1 week
+  # TODO: INTPYTHON-668
+  # - name: test-chatgpt-retrieval-plugin-rhel
+  #   display_name: ChatGPT Retrieval Plugin
+  #   expansions:
+  #     DIR: chatgpt-retrieval-plugin
+  #   run_on:
+  #     - rhel87-small
+  #   tasks:
+  #     - name: test-chatgpt-retrieval-plugin-local
+  #     - name: test-chatgpt-retrieval-plugin-remote
+  #       batchtime: 10080  # 1 week
+
+  # TODO: INTPYTHON-669
+  # - name: test-llama-index-vectorstore-rhel
+  #   display_name: LlamaIndex RHEL Vector Store
+  #   expansions:
+  #     DIR: llama-index-python-vectorstore
+  #   run_on:
+  #     - rhel87-small
+  #   tasks:
+  #     - name: test-llama-index-local
+  #     # TODO: INTPYTHON-440
+  #     # - name: test-llama-index-remote
+  #     #   batchtime: 10080  # 1 week
 
   - name: test-docarray-rhel
     display_name: DocArray RHEL
@@ -393,3 +405,14 @@ buildvariants:
       - ubuntu2204-small
     tasks:
       - name: test-langchaingo-local
+
+  # TODO: INTPYTHON-667
+  # - name: test-langchain-js-ubuntu
+  #   display_name: LangchainJS Ubuntu2204
+  #   expansions:
+  #     DIR: langchain-js
+  #   run_on:
+  #     - ubuntu2204-small
+  #   tasks:
+  #     - name: test-langchain-js-local
+  #     - name: test-langchain-js-remote

.evergreen/execute-tests.sh

@@ -5,12 +5,12 @@ set -eu
 SCRIPT_DIR=$(realpath "$(dirname ${BASH_SOURCE[0]})")
 ROOT_DIR=$(dirname $SCRIPT_DIR)
 
-
 # Source the configuration.
 cd ${ROOT_DIR}/${DIR}
 set -a
 source config.env
 set +a
 
 cd ${REPO_NAME}
+
 bash ${ROOT_DIR}/${DIR}/run.sh

.evergreen/provision-atlas.sh

@@ -18,6 +18,8 @@ source secrets-export.sh
 
 # Create the env file
 echo "export VOYAGEAI_S3_BUCKET=$VOYAGEAI_S3_BUCKET" >> env.sh
-echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh
+echo "export AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> env.sh
+echo "export AZURE_OPENAI_API_KEY=$AZURE_OPENAI_API_KEY" >> env.sh
+echo "export OPENAI_API_VERSION=$OPENAI_API_VERSION" >> env.sh
 echo "export MONGODB_URI=$CONN_STRING" >> env.sh
 echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh

.evergreen/setup-remote.sh

@@ -44,6 +44,9 @@ case $DIR in
     crewai-tools)
         MONGODB_URI=$CREWAI_TOOLS_URI
     ;;
+    langchain-js)
+        MONGODB_URI=$LANGCHAIN_MONGODB_URI
+    ;;
     *)
         echo "Missing config in setup-remote.sh for DIR: $DIR"
         exit 1
@@ -53,9 +56,11 @@ export MONGODB_URI
 
 # Create the env file
 echo "export VOYAGEAI_S3_BUCKET=$VOYAGEAI_S3_BUCKET" >> env.sh
-echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh
-echo "export OPENAI_API_KEY=$OPENAI_API_KEY" >> env.sh
+echo "export AZURE_OPENAI_ENDPOINT=$AZURE_OPENAI_ENDPOINT" >> env.sh
+echo "export AZURE_OPENAI_API_KEY=$AZURE_OPENAI_API_KEY" >> env.sh
+echo "export OPENAI_API_VERSION=$OPENAI_API_VERSION" >> env.sh
 echo "export MONGODB_URI=$MONGODB_URI" >> env.sh
+echo "export VOYAGEAI_API_KEY=$VOYAGEAI_API_KEY" >> env.sh
 
 # Ensure the remote database is populated.
 . .evergreen/utils.sh

README.md

@@ -107,7 +107,7 @@ Test execution flow is defined in `.evergreen/config.yml`. The test pipeline's c
 
   - `DIR` -- The subdirectory where the tasks will run
 
-- `run_on` -- Specified platform to run on. `rhel87-small` should be used by default. Any other distro may fail Atlas CLI setup.
+- `run_on` -- Specified platform to run on. `rhel87-small` or `ubuntu2204-small` should be used by default. Any other distro may fail Atlas CLI setup.
 - `tasks` -- Tasks to run. See below for more details
 - `cron` -- The tests are run via a cron job on a nightly cadence. This can be modified by setting a different cadence. Cron jobs can be scheduled using [cron syntax](https://crontab.guru/#0_0_*_*_*)
 
@@ -175,5 +175,18 @@ as they often have to build for a more broad set of scenarios than the original
 Rather than making a new branch and modifying a `config.env` file, you can run a patch build as follows:
 
 ```bash
-evergreen patch -p ai-ml-pipelin-testing --param REPO_ORG="<my-org>" --param REPO_BRANCH="<my-branch>" -y "<my-message>"
+evergreen patch -p ai-ml-pipeline-testing --param REPO_ORG="<my-org>" --param REPO_BRANCH="<my-branch>" -y -d "<my-message>"
 ```
+
+For example
+```bash
+evergreen patch -p ai-ml-pipeline-testing --param REPO_ORG=caseyclements --param REPO_NAME="langchain-mongodb" --param REPO_BRANCH="INTPYTHON-629" -y -d "Increased retries to 4."
+```
+
+### Handling Failing Tests
+
+If tests are found to be failing, and cannot be addressed quickly, the responsible team MUST create a JIRA ticket, and disable the relevant tests
+in the `config.yml` file, with a comment about the JIRA ticket that will address it.
+
+This policy will help ensure that a single failing integration does not cause noise in the `dbx-ai-ml-testing-pipeline-notifications` that would mask other
+failures.

docarray/run.sh

@@ -30,6 +30,6 @@ set +x
 
 # Run tests. Sensitive variables in Evergreen come from Evergeen project: ai-ml-pipeline-testing/
 # shellcheck disable=SC2154
-MONGODB_URI="$MONGODB_URI" \
-MONGODB_DATABASE="docarray_test_db" \
-pytest -v tests/index/mongo_atlas
+export MONGODB_URI="$MONGODB_URI"
+export MONGODB_DATABASE="docarray_test_db"
+pytest -v tests/index/mongo_atlas || pytest --lf

haystack-embeddings/run.sh

@@ -22,4 +22,4 @@ PYTHON_BINARY=$(which python)
 $PYTHON_BINARY -m pip install -U pip hatch
 
 # Run tests.
-MONGO_CONNECTION_STRING="$MONGODB_URI" hatch run test
+MONGO_CONNECTION_STRING="$MONGODB_URI" hatch run test:all -v

haystack-fulltext/run.sh

@@ -22,4 +22,4 @@ PYTHON_BINARY=$(which python)
 $PYTHON_BINARY -m pip install -U pip hatch
 
 # Run tests.
-MONGO_CONNECTION_STRING_2="$MONGODB_URI" hatch run test
+MONGO_CONNECTION_STRING_2="$MONGODB_URI" hatch run test:all -v

langchain-js/config.env

@@ -0,0 +1,3 @@
+REPO_NAME=langchainjs
+REPO_ORG=langchain-ai
+DATABASE=langchain_test_db

langchain-js/indexes/langchain_test.json

@@ -0,0 +1,18 @@
+{
+    "name": "default",
+    "type": "search",
+    "mappings": {
+        "fields": {
+            "e": {
+                "type": "number"
+            },
+            "embedding": {
+                "dimensions": 1536,
+                "similarity": "euclidean",
+                "type": "knnVector"
+            }
+        }
+    },
+    "database": "langchain",
+    "collectionName": "test"
+}