PYTHON-5217 Add helper scripts and test for atlas data lake (#636)

Author: blink1073

.evergreen/atlas_data_lake/pull-mongohouse-image.sh

@@ -8,7 +8,9 @@ SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
 . $SCRIPT_DIR/../handle-paths.sh
 pushd $SCRIPT_DIR
 REPO="904697982180.dkr.ecr.us-east-1.amazonaws.com/atlas-query-engine-test"
-. ../secrets_handling/setup-secrets.sh drivers/adl
+if [ ! -f secrets-export.sh ]; then
+  . ../secrets_handling/setup-secrets.sh drivers/adl
+fi
 source secrets-export.sh
 unset AWS_SESSION_TOKEN
 aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $REPO

.evergreen/atlas_data_lake/run-mongohouse-image.sh

@@ -11,4 +11,9 @@ SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
 IMAGE=904697982180.dkr.ecr.us-east-1.amazonaws.com/atlas-query-engine-test
 USE_TTY=""
 test -t 1 && USE_TTY="-t"
-docker run -d -p 27017:27017 -v "$DRIVERS_TOOLS/.evergreen/atlas_data_lake:/src" -i $USE_TTY $IMAGE --config ./testdata/config/external/drivers/config.yaml
+if command -v podman &> /dev/null; then
+    DOCKER="podman --storage-opt ignore_chown_errors=true"
+else
+    DOCKER=docker
+fi
+$DOCKER run -d -p 27017:27017 --platform linux/amd64 --name atlas-data-lake -v "$DRIVERS_TOOLS/.evergreen/atlas_data_lake:/src" -i $USE_TTY $IMAGE --config ./testdata/config/external/drivers/config.yaml

.evergreen/atlas_data_lake/setup.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+#
+# This script sets Atlas Data Lake tests.
+#
+set -eu
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../handle-paths.sh
+
+bash $SCRIPT_DIR/pull-mongohouse-image.sh
+bash $SCRIPT_DIR/run-mongohouse-image.sh

.evergreen/atlas_data_lake/teardown.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# This script tears down Atlas Data Lake tests.
+#
+set -eu
+
+if command -v podman &> /dev/null; then
+    DOCKER="podman --storage-opt ignore_chown_errors=true"
+else
+    DOCKER=docker
+fi
+$DOCKER kill atlas-data-lake || true
+$DOCKER rm atlas-data-lake

.evergreen/config.yml

@@ -520,6 +520,14 @@ functions:
         include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN]
         args: [src/.evergreen/tests/test-aws.sh]
 
+  "run data lake test":
+    - command: subprocess.exec
+      type: test
+      params:
+        binary: bash
+        include_expansions_in_env: [AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN]
+        args: [src/.evergreen/tests/test-data-lake.sh]
+
   "run csfle test":
     - command: ec2.assume_role
       params:
@@ -1037,12 +1045,18 @@ tasks:
       commands:
         - func: "run aws test"
 
+    - name: "test-atlas-data-lake"
+      tags: ["pr", "data_lake"]
+      commands:
+        - func: "run data lake test"
+
     - name: "test-cli-full"
       tags: ["pr"]
       commands:
         - func: "run cli test full"
 
     - name: "test-ocsp"
+      tags: ["pr", "ocsp"]
       commands:
         - func: "run ocsp test"
 
@@ -1427,20 +1441,15 @@ buildvariants:
   tasks:
     - ".releng" # Run all tasks with the "releng" tag
 
-# Tests relating to docker images
-- name: tests-docker
-  display_name: Docker
-  run_on:
-    - ubuntu2204-small
-  tasks:
-    - ".docker"  # Run all tasks with the "docker" tag
-
-- name: tests-aws
-  display_name: Auth AWS
+- name: tests-misc
+  display_name: Other tests
   run_on:
     - ubuntu2204-small
   tasks:
     - ".aws"
+    - ".data_lake"
+    - ".ocsp"
+    - ".docker"
 
 - name: tests-oidc
   display_name: Auth OIDC

.evergreen/docker/run-server.sh

@@ -31,7 +31,9 @@ fi
 pushd $DRIVERS_TOOLS
 cp .gitignore .dockerignore
 USER="--build-arg USER_ID=$(id -u) --build-arg GROUP_ID=$(id -g)"
-$DOCKER build $PLATFORM -t $NAME -f $SCRIPT_DIR/$IMAGE/Dockerfile $USER .
+ARGS="$PLATFORM -t $NAME -f $SCRIPT_DIR/$IMAGE/Dockerfile $USER ."
+# Allow a single failure due to transient issues with apt-get.
+$DOCKER build $ARGS || $DOCKER build $ARGS
 popd
 
 # Handle environment variables.

.evergreen/tests/test-data-lake.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Test aws setup function for different inputs.
+set -eu
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../handle-paths.sh
+
+pushd $SCRIPT_DIR/..
+DOWNLOAD_DIR=$SCRIPT_DIR/dl_test
+bash install-cli.sh .
+./mongosh-dl --version 2.1.1 --out ${DOWNLOAD_DIR}/bin --strip-path-components 2 --retries 5
+popd
+
+pushd $SCRIPT_DIR/../atlas_data_lake
+bash ./setup.sh
+source secrets-export.sh
+${DOWNLOAD_DIR}/bin/mongosh "mongodb://$ADL_USERNAME:$ADL_PASSWORD@localhost:27017" --eval "db.runCommand({\"ping\":1})"
+bash ./teardown.sh
+popd
+
+rm -rf "${SCRIPT_DIR:?}/${DOWNLOAD_DIR}"
+
+make -C ${DRIVERS_TOOLS} test

.gitignore

@@ -97,6 +97,7 @@ test-env.sh
 # Temporary files created during operation.
 /uri.txt
 /results.json
+/test-results.json
 /mongodb
 node_index.tab
 mongodb-binaries.tgz